test: keep e2e tests artifacts on failure (#4214)

Author: gilcu3

crates/e2e-tests/README.md

@@ -172,7 +172,8 @@ indexer state may be corrupt).
 The entry point for tests. `MpcCluster::start(config)` does everything:
 
 1. Create `TestPorts` via `TestPorts::e2e_tests(config.port_seed)`.
-2. Create a per-test temp directory.
+2. Create a per-test artifact directory (`TestDir`, kept on failure, see
+   [Debugging a failure](#debugging-a-failure)).
 3. Start the `NearSandbox`.
 4. Build a `NearBlockchain` signed as the sandbox root.
 5. Generate deterministic signing keys for each node (near signer, p2p, operator,
@@ -213,8 +214,9 @@ The returned cluster exposes:
   `wait_for_foreign_chains_registrations`, `wait_for_available_foreign_chains`
 - **User accounts:** `user_client`, `default_user_account`.
 
-`Drop` kills all running nodes; the temp directory is held via `test_dir` and
-removed when the cluster is dropped.
+`Drop` kills all running nodes; the artifact directory is held via `test_dir` and
+removed when the cluster is dropped, unless the test failed (see
+[Debugging a failure](#debugging-a-failure)).
 
 ```rust
 pub struct MpcClusterConfig {
@@ -361,6 +363,30 @@ CI runs the same task via the `mpc-e2e-tests` job.
 
 ---
 
+## Debugging a failure
+
+Each test gets an artifact directory (`/tmp/mpc-e2e-<random>` by default) with a
+`node<i>/` subdirectory per node holding its config, secrets, RocksDB data, the
+embedded neard home, `stdout.log` (the node's tracing output) and `stderr.log`
+(panics only). `TestDir`
+(`src/test_dir.rs`) keeps it when the test failed and prints the path to the
+test's stderr; a startup failure also carries it in the error:
+
+```
+failed to start cluster: cluster artifacts preserved in /tmp/mpc-e2e-AbC123:
+mpc-node 0 exited early, check /tmp/mpc-e2e-AbC123/node0 (stdout.log holds its
+logs, stderr.log any panic)
+```
+
+| Variable | Effect |
+|---|---|
+| `E2E_KEEP_TMP=1` | Keep artifacts for passing tests too. `0`, `false`, `no` or `off` (any case) deletes them even for failing ones. |
+| `E2E_HOME_BASE=<dir>` | Parent of the artifact directories, created if missing. Same as `MpcClusterConfig::home_base`, without editing the test. |
+| `MPC_NODE_LOG=<filter>` | `RUST_LOG` for the spawned mpc-node processes (default `DEBUG`). |
+| `MPC_NODE_BACKTRACE=<0\|1\|full>` | `RUST_BACKTRACE` for the spawned mpc-node processes (default `1`). |
+
+---
+
 ## Test layout conventions
 
 - Follow the `<subject>__should_<assertion>` or `<subject>__<scenario>` naming
@@ -370,8 +396,8 @@ CI runs the same task via the `mpc-e2e-tests` job.
 - Prefer `common::must_setup_cluster` over calling `MpcCluster::start` directly;
   it initialises `tracing_subscriber` and waits for presignatures.
 - Tests must be deterministic across parallel execution. Use the port
-  allocator, the per-cluster temp directory, and the deterministic key
-  generation rather than creating state outside the cluster.
+  allocator, the per-cluster artifact directory (`cluster.test_dir`), and the
+  deterministic key generation rather than creating state outside the cluster.
 - Arithmetic in tests uses raw `+`/`-`/`*`/`/`; overflow panics are the
   desired failure mode (see `CLAUDE.md`).
 

crates/e2e-tests/src/cluster.rs

@@ -30,6 +30,7 @@ use crate::blockchain::{DeployedContract, NearBlockchain};
 use crate::caller::{CallMpc, WithWaitLevel};
 use crate::mpc_node::{MpcNode, MpcNodeSetup, MpcNodeSetupArgs, NodePorts};
 use crate::near_sandbox::NearSandbox;
+use crate::test_dir::TestDir;
 use test_port_allocator::TestPorts;
 
 const SANDBOX_ROOT_ACCOUNT: &str = "sandbox";
@@ -97,7 +98,8 @@ pub struct MpcClusterConfig {
     pub presignatures_to_buffer: usize,
     /// Version of the `near-sandbox` binary (e.g. `"2.6.3"`, `"2.10.4"`).
     pub sandbox_version: String,
-    /// Root directory for all test artifacts (logs, configs, DB). If `None`, a temp dir is created.
+    /// Parent of the directory holding all test artifacts (logs, configs, DB).
+    /// If `None`, `E2E_HOME_BASE` is used, else the system temp dir.
     pub home_base: Option<PathBuf>,
     /// Indices (into the node array) of nodes that are initial participants.
     /// An empty vec means all nodes are participants. Set to a subset to start
@@ -250,8 +252,7 @@ pub struct MpcCluster {
     pub threshold: usize,
     pub user_accounts: HashMap<AccountId, SigningKey>,
     pub ports: TestPorts,
-    /// Held to keep the temp directory alive for the lifetime of the cluster.
-    pub test_dir: tempfile::TempDir,
+    pub test_dir: TestDir,
 }
 
 impl MpcCluster {
@@ -263,9 +264,19 @@ impl MpcCluster {
     /// binaries, and wait for Running state.
     pub async fn start(config: MpcClusterConfig) -> anyhow::Result<Self> {
         config.validate()?;
+        let test_dir = TestDir::new(config.home_base.as_deref())?;
+        Self::start_in(&test_dir, config).await.map_err(|error| {
+            test_dir.keep();
+            error.context(format!(
+                "cluster artifacts preserved in {}",
+                test_dir.path().display()
+            ))
+        })
+    }
+
+    async fn start_in(test_dir: &TestDir, config: MpcClusterConfig) -> anyhow::Result<Self> {
         let threshold = config.threshold;
         let ports = TestPorts::e2e_tests(config.port_seed);
-        let test_dir = create_test_dir(&config.home_base)?;
 
         let sandbox = NearSandbox::start(&ports, &config.sandbox_version).await?;
         let root_secret_key: near_kit::SecretKey = SANDBOX_ROOT_SECRET_KEY
@@ -371,7 +382,7 @@ impl MpcCluster {
             threshold,
             user_accounts,
             ports,
-            test_dir,
+            test_dir: test_dir.clone(),
         })
     }
 
@@ -1192,16 +1203,6 @@ impl MpcNodeState {
     }
 }
 
-fn create_test_dir(home_base: &Option<PathBuf>) -> anyhow::Result<tempfile::TempDir> {
-    match home_base {
-        Some(base) => {
-            std::fs::create_dir_all(base)?;
-            Ok(tempfile::tempdir_in(base)?)
-        }
-        None => Ok(tempfile::tempdir()?),
-    }
-}
-
 fn generate_signing_keys(
     num_nodes: u64,
 ) -> (
@@ -1637,8 +1638,9 @@ async fn ensure_nodes_alive(nodes: &mut [MpcNodeState]) -> anyhow::Result<()> {
         if let MpcNodeState::Running(n) = node {
             anyhow::ensure!(
                 !n.has_exited(),
-                "mpc-node {i} exited early — check {}/{}",
+                "mpc-node {i} exited early, check {} ({} holds its logs, {} any panic)",
                 n.setup().home_dir().display(),
+                crate::mpc_node::STDOUT_LOG,
                 crate::mpc_node::STDERR_LOG
             );
         }

crates/e2e-tests/src/lib.rs

@@ -6,6 +6,7 @@ pub mod foreign_chain_mock;
 pub mod metrics;
 pub mod mpc_node;
 pub mod near_sandbox;
+pub mod test_dir;
 
 pub use blockchain::{DeployedContract, NearBlockchain};
 pub use caller::NearKitCaller;
@@ -14,4 +15,5 @@ pub use cluster::{
     MpcClusterConfig, MpcNodeState,
 };
 pub use near_sandbox::NearSandbox;
+pub use test_dir::TestDir;
 pub use test_port_allocator::{E2eTestPorts, TestPorts};

crates/e2e-tests/src/mpc_node.rs

@@ -150,6 +150,7 @@ impl MpcNode {
     }
 }
 
+pub const STDOUT_LOG: &str = "stdout.log";
 pub const STDERR_LOG: &str = "stderr.log";
 
 /// Guard that kills the child process on drop.
@@ -362,9 +363,9 @@ impl MpcNodeSetup {
             "starting mpc-node"
         );
 
-        let stdout_file = std::fs::File::create(self.home_dir.join("stdout.log"))
+        let stdout_file = std::fs::File::create(self.home_dir.join(STDOUT_LOG))
             .context("failed to create stdout log")?;
-        let stderr_file = std::fs::File::create(self.home_dir.join("stderr.log"))
+        let stderr_file = std::fs::File::create(self.home_dir.join(STDERR_LOG))
             .context("failed to create stderr log")?;
 
         let child = Command::new(&self.binary_path)