Intproxy logs in tests. (#2686)

* Add intproxy logs for tests

* create dir

* create dir all

* fix : in name

* replace

* better looking logs

* intproxy logs for macos

* changelog

* add docs // handle creation at init_tracing

* fix macos init

* different dirs for macos and linux

* fix macos

* modify action for separate dirs

* different names

.github/workflows/ci.yaml

@@ -356,6 +356,14 @@ jobs:
         run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-operator --features "crd, client"
       - name: mirrord cli UT
         run: cargo test --target x86_64-unknown-linux-gnu -p mirrord
+      - name: save intproxy logs
+        continue-on-error: true
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          # Name of the artifact to upload.
+          name: intproxy_logs_linux
+          path: /tmp/intproxy_logs/linux
 
   macos_tests:
     runs-on: macos-13
@@ -469,6 +477,14 @@ jobs:
       - run: cargo build --target=x86_64-apple-darwin -p mirrord-layer # Build layer lib. The tests load it into the apps.
       - name: mirrord layer tests
         run: cargo test --target=x86_64-apple-darwin -p mirrord-layer
+      - name: save intproxy logs
+        continue-on-error: true
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          # Name of the artifact to upload.
+          name: intproxy_logs_macos
+          path: /tmp/intproxy_logs/macos
 
   e2e:
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -159,6 +159,16 @@ And then in order to use that dylib in the tests, run the tests like this:
 MIRRORD_TEST_USE_EXISTING_LIB=../../target/universal-apple-darwin/debug/libmirrord_layer.dylib cargo test -p mirrord-layer
 ```
 
+### Integration Tests logs and you
+
+These tests will try writing the `mirrord-intproxy` logs to a file in `/tmp/intproxy_logs`
+(the dir will be created if it doesn't exist), the file name should be the same as the test name,
+e.g. `/tmp/intproxy_logs/node_close_application_1_Application__NodeFileOps.log`.
+If log file creation fails, then you should see the logs in `stderr`.
+
+When running these in CI, an artifact is produced (scroll to `Artifacts` which is under
+the `Actions` -> `Summary` page) with all the test log files that could be created.
+
 ## Testing mirrord manually with a sample app.
 
 From the root directory of the mirrord repository, create a new testing deployment and service:

changelog.d/+intproxy-logs-for-tests.internal.md

@@ -0,0 +1 @@
+Adds intproxy logs for the integration tests in CI.

mirrord/layer/tests/common/mod.rs

@@ -2,9 +2,12 @@ use std::{
     assert_matches::assert_matches,
     collections::HashMap,
     fmt::Debug,
+    fs::File,
+    io,
     path::{Path, PathBuf},
     process::Stdio,
     str::FromStr,
+    sync::Arc,
     time::Duration,
 };
 
@@ -28,12 +31,70 @@ use tokio::{
     net::{TcpListener, TcpStream},
     process::Command,
 };
+use tracing::subscriber::DefaultGuard;
+use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter};
 
 /// Configuration for [`Application::RustOutgoingTcp`] and [`Application::RustOutgoingUdp`].
 pub const RUST_OUTGOING_PEERS: &str = "1.1.1.1:1111,2.2.2.2:2222,3.3.3.3:3333";
 /// Configuration for [`Application::RustOutgoingTcp`] and [`Application::RustOutgoingUdp`].
 pub const RUST_OUTGOING_LOCAL: &str = "4.4.4.4:4444";
 
+/// Initializes tracing for the current thread, allowing us to have multiple tracing subscribers
+/// writin logs to different files.
+///
+/// We take advantage of how Rust's thread naming scheme for tests to create the log files,
+/// and if we have no thread name, then we just write the logs to `stderr`.
+pub fn init_tracing() -> Result<DefaultGuard, Box<dyn std::error::Error>> {
+    let subscriber = tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::new("mirrord=trace"))
+        .without_time()
+        .with_ansi(false)
+        .with_file(true)
+        .with_line_number(true)
+        .with_span_events(FmtSpan::ACTIVE)
+        .pretty();
+
+    // Sets the default subscriber for the _current thread_, returning a guard that unsets
+    // the default subscriber when it is dropped.
+    match std::thread::current()
+        .name()
+        .map(|name| name.replace(':', "_"))
+    {
+        Some(test_name) => {
+            let mut logs_file = PathBuf::from_str("/tmp/intproxy_logs")?;
+
+            #[cfg(target_os = "macos")]
+            logs_file.push("macos");
+            #[cfg(not(target_os = "macos"))]
+            logs_file.push("linux");
+
+            let _ = std::fs::create_dir_all(&logs_file).ok();
+
+            logs_file.push(&test_name);
+            match File::create(logs_file) {
+                Ok(file) => {
+                    let subscriber = subscriber.with_writer(Arc::new(file)).finish();
+
+                    // Writes the logs to a file.
+                    Ok(tracing::subscriber::set_default(subscriber))
+                }
+                Err(_) => {
+                    let subscriber = subscriber.with_writer(io::stderr).finish();
+
+                    // File creation failure makes the output go to `stderr`.
+                    Ok(tracing::subscriber::set_default(subscriber))
+                }
+            }
+        }
+        None => {
+            let subscriber = subscriber.with_writer(io::stderr).finish();
+
+            // No thread name makes the output go to `stderr`.
+            Ok(tracing::subscriber::set_default(subscriber))
+        }
+    }
+}
+
 pub struct TestIntProxy {
     codec: Framed<TcpStream, DaemonCodec>,
     num_connections: u64,

mirrord/layer/tests/fileops.rs

@@ -44,6 +44,8 @@ async fn self_open(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(dylib_path, vec![], None)
         .await;
@@ -63,6 +65,8 @@ async fn self_open(
 #[tokio::test]
 #[timeout(Duration::from_secs(20))]
 async fn read_from_mirrord_bin(dylib_path: &Path) {
+    let _tracing = init_tracing().unwrap();
+
     let contents = "please don't flake";
     let temp_dir = env::temp_dir();
     let file_path = temp_dir.join("mirrord-test-read-from-mirrord-bin");
@@ -104,6 +108,8 @@ async fn read_from_mirrord_bin(dylib_path: &Path) {
 #[tokio::test]
 #[timeout(Duration::from_secs(60))]
 async fn pwrite(#[values(Application::RustFileOps)] application: Application, dylib_path: &Path) {
+    let _tracing = init_tracing().unwrap();
+
     // add rw override for the specific path
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(
@@ -222,6 +228,8 @@ async fn node_close(
     #[values(Application::NodeFileOps)] application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(
             dylib_path,
@@ -287,6 +295,8 @@ async fn go_stat(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     // add rw override for the specific path
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(
@@ -348,6 +358,8 @@ async fn go_dir(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(
             dylib_path,
@@ -466,6 +478,8 @@ async fn go_dir_on_linux(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(
             dylib_path,
@@ -561,6 +575,8 @@ async fn go_dir_bypass(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let tmp_dir = temp_dir().join("go_dir_bypass_test");
     std::fs::create_dir_all(tmp_dir.clone()).unwrap();
     std::fs::write(tmp_dir.join("a"), "").unwrap();
@@ -600,6 +616,8 @@ async fn read_go(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(dylib_path, vec![("MIRRORD_FILE_MODE", "read")], None)
         .await;
@@ -640,6 +658,8 @@ async fn write_go(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut layer_connection) = application
         .start_process_with_layer(dylib_path, get_rw_test_file_env_vars(), None)
         .await;
@@ -667,6 +687,8 @@ async fn lseek_go(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(dylib_path, get_rw_test_file_env_vars(), None)
         .await;
@@ -700,6 +722,8 @@ async fn faccessat_go(
     application: Application,
     dylib_path: &Path,
 ) {
+    let _tracing = init_tracing().unwrap();
+
     let (mut test_process, mut intproxy) = application
         .start_process_with_layer(dylib_path, get_rw_test_file_env_vars(), None)
         .await;