fix: catalyst teardown timeout — nvidia RM takes ~160s on GV100

The warm_swap phase (unbinding nvsov from Titan V) triggers nvidia-470
RM teardown which takes ~160s for HBM2 dealloc + falcon halt. The
previous 10s UNBIND_TIMEOUT killed the child process prematurely,
causing the rebind to race with still-running kernel teardown.

- Add CATALYST_TEARDOWN_TIMEOUT (200s) for nvidia RM teardown
- Use extended timeout for catalyst warm_swap unbind
- Raise RPC timeout from 180s to 420s
- Raise HANDOFF_DEADLINE from 150s to 400s

Validated: full catalyst pipeline completes end-to-end on Titan V —
insmod (400ms), BAR0 capture (78,627 registers), warm_swap (~160s),
snapshot persisted, frozen .ko archived, rmmod clean (100ms).

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: ecoPrimal

crates/core/cylinder/src/vfio/guarded_sysfs.rs

@@ -37,10 +37,14 @@ pub const UNBIND_TIMEOUT: Duration = Duration::from_secs(10);
 pub const INSMOD_TIMEOUT: Duration = Duration::from_secs(15);
 /// Default timeout for `rmmod` operations.
 pub const RMMOD_TIMEOUT: Duration = Duration::from_secs(10);
+/// Extended timeout for nvidia RM teardown during catalyst unbind.
+/// nvidia-470's RM on GV100 takes ~160s to fully teardown (HBM2 dealloc,
+/// falcon shutdown, FECS/GPCCS halt). Must exceed this or the child gets
+/// killed and the probe/rebind races with still-running kernel teardown.
+pub const CATALYST_TEARDOWN_TIMEOUT: Duration = Duration::from_secs(200);
 /// Default overall handoff deadline.
-/// 150s: catalyst cold-boot on Volta needs ~30s deferred probe delay,
-/// 15s settle, and 30-60s RM cold init (HBM2 + falcon + FECS).
-pub const HANDOFF_DEADLINE: Duration = Duration::from_secs(150);
+/// 400s for catalyst: 15s settle + 160s RM teardown + 30s BAR0 capture.
+pub const HANDOFF_DEADLINE: Duration = Duration::from_secs(400);
 
 /// Errors from guarded sysfs operations.
 #[derive(Debug, thiserror::Error)]

crates/core/cylinder/src/vfio/sovereign_handoff.rs

@@ -1030,15 +1030,20 @@ pub fn execute_handoff(
     let t = Instant::now();
     if let Some(ref current) = guarded_sysfs::read_current_driver(&config.bdf) {
         let unbind_path = crate::linux_paths::sysfs_pci_driver_unbind(current);
+        let unbind_timeout = if is_catalyst {
+            guarded_sysfs::CATALYST_TEARDOWN_TIMEOUT
+        } else {
+            guarded_sysfs::UNBIND_TIMEOUT
+        };
         if let Err(e) = guarded_sysfs::sysfs_write_guarded(
-            &unbind_path, &config.bdf, guarded_sysfs::UNBIND_TIMEOUT,
+            &unbind_path, &config.bdf, unbind_timeout,
         ) {
             steps.push(HandoffStep {
                 name: "warm_swap".into(), ok: false,
                 detail: Some(format!("guarded unbind {current} failed: {e}")),
                 duration_ms: t.elapsed().as_millis() as u64,
             });
-    
+
             return halt_result(&config.bdf, "warm_swap", steps, patch_result,
                                module_loaded, false, overall, &sibling_state,
                                &config.module_name, needs_device_rollback);

crates/server/src/pure_jsonrpc/handler/dispatch/mod.rs

@@ -1077,9 +1077,10 @@ impl DispatchHandler {
         // Wrapped in tokio::time::timeout to prevent indefinite RPC hangs.
         // The handoff itself has internal deadlines via guarded_sysfs, but
         // this outer timeout is the last line of defense.
-        // 180s: catalyst cold-boot on Volta requires ~30s PCI probe delay
-        // + 15s settle + 30-60s RM init (HBM2 training, falcon boot, FECS).
-        let rpc_timeout = std::time::Duration::from_secs(180);
+        // 420s: catalyst teardown on GV100 needs ~160s for nvidia RM
+        // shutdown (HBM2 dealloc, falcon halt) + 15s settle + 30s probe
+        // + 30s BAR0 capture margin.
+        let rpc_timeout = std::time::Duration::from_secs(420);
         let blocking_future = tokio::task::spawn_blocking(move || {
             execute_handoff(&config, None)
         });