feat: persist daemon termination evidence (#8170)

Author: chubes4

src/commands/daemon.rs

@@ -73,6 +73,14 @@ enum DaemonCommand {
         #[arg(long, default_value = daemon::DEFAULT_ADDR)]
         addr: String,
     },
+    /// Supervise one daemon child and persist its termination evidence.
+    #[command(hide = true)]
+    Supervise {
+        #[arg(long, default_value = daemon::DEFAULT_ADDR)]
+        addr: String,
+        #[arg(long)]
+        startup_token: String,
+    },
     /// Stop the background daemon recorded in the state file
     Stop,
     /// Show daemon state and selected local address
@@ -163,6 +171,10 @@ pub fn run(args: DaemonArgs, _global: &crate::commands::GlobalArgs) -> CmdResult
             0,
         )),
         DaemonCommand::Serve { addr } => serve(&addr),
+        DaemonCommand::Supervise { addr, startup_token } => {
+            daemon::supervise(&addr, &startup_token)?;
+            Ok((DaemonOutput::Serve(DaemonStartResult { pid: std::process::id(), address: addr, state_path: String::new(), lease_id: String::new() }), 0))
+        }
         DaemonCommand::Stop => Ok((DaemonOutput::Stop(daemon::stop()?), 0)),
         DaemonCommand::Status => Ok((DaemonOutput::Status(daemon::read_status()?), 0)),
         DaemonCommand::BrokerConfig {

src/commands/runner/doctor/remote.rs

@@ -283,6 +283,13 @@ pub(super) fn disconnected_report(
             if let Some(evidence) = &recovery.ownership_evidence {
                 details.insert("ownership_evidence".to_string(), evidence.clone());
             }
+            if let Some(evidence) = &recovery.termination_evidence {
+                details.insert(
+                    "termination_evidence".to_string(),
+                    serde_json::to_string(evidence)
+                        .unwrap_or_else(|_| "unavailable: serialization failed".to_string()),
+                );
+            }
             (
                 "Disconnected runner was checked through bounded remote lease recovery".to_string(),
                 recovery.adoption_command.clone(),

src/commands/runner/doctor/tests/daemon.rs

@@ -127,6 +127,7 @@ fn disconnected_lab_doctor_reuses_daemon_recovery_envelope() {
         daemon_build_identity: Some("homeboy 0.284.0+live".to_string()),
         runtime_paths: None,
         active_jobs: 1,
+        termination_evidence: None,
         repair_plan: Vec::new(),
     };
     let runner = Runner {

src/core/daemon.rs

@@ -37,7 +37,7 @@ pub use artifact_download::ArtifactDownload;
 pub use broker_config::{render_broker_config, BrokerConfig, BrokerConfigOptions, ServiceIdentity};
 pub use control::{
     adopt_orphaned_lease, artifact_content_url, ensure_running, fetch_artifact_to_path,
-    reconcile_leaseless_orphans, recover_missing_lease_state, start_background,
+    reconcile_leaseless_orphans, recover_missing_lease_state, start_background, supervise,
     ArtifactFetchOutcome,
 };
 use patch_capture::{capture_baseline, capture_patch_report};
@@ -88,6 +88,43 @@ pub struct DaemonStatus {
     /// operator can distinguish another runner from an attributable owner.
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub process_candidates: Vec<DaemonProcessCandidate>,
+    /// Launcher-owned evidence is best effort. A missing record explicitly does
+    /// not imply an OS-level cause for a dead daemon.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub termination_evidence: Option<DaemonTerminationEvidence>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum DaemonTerminationClassification {
+    CleanStop,
+    UnexpectedExit,
+}
+
+/// Durable, bounded evidence from the process that launched the daemon.
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
+pub struct DaemonTerminationEvidence {
+    pub classification: DaemonTerminationClassification,
+    pub observed_at: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub lease_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pid: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub binary_identity: Option<String>,
+    pub active_jobs: usize,
+    pub resource_evidence: String,
+    pub os_evidence: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub exit_code: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub signal: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub stdout: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub stderr: Option<String>,
+    #[serde(default)]
+    pub stop_requested: bool,
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
@@ -168,6 +205,8 @@ pub struct DaemonFreshnessReport {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub runtime_paths: Option<DaemonRuntimeSnapshot>,
     pub active_jobs: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub termination_evidence: Option<DaemonTerminationEvidence>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub repair_plan: Vec<DaemonRepairStep>,
 }
@@ -497,6 +536,44 @@ pub fn read_status() -> Result<DaemonStatus> {
         state_path,
         state_identity,
         process_candidates: control::daemon_process_candidates(&jobs_path)?,
+        termination_evidence: (!validation.running)
+            .then(read_termination_evidence)
+            .transpose()?
+            .flatten(),
+    })
+}
+
+fn read_termination_evidence() -> Result<Option<DaemonTerminationEvidence>> {
+    let path = paths::daemon_termination_file()?;
+    match fs::read_to_string(&path) {
+        Ok(body) => serde_json::from_str(&body).map(Some).map_err(|error| {
+            Error::internal_json(error.to_string(), Some(format!("parse {}", path.display())))
+        }),
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(error) => Err(Error::internal_io(
+            error.to_string(),
+            Some(format!("read {}", path.display())),
+        )),
+    }
+}
+
+pub(crate) fn write_termination_evidence(evidence: &DaemonTerminationEvidence) -> Result<()> {
+    let path = paths::daemon_termination_file()?;
+    let parent = path.parent().expect("daemon termination path has parent");
+    fs::create_dir_all(parent).map_err(|error| {
+        Error::internal_io(
+            error.to_string(),
+            Some(format!("create {}", parent.display())),
+        )
+    })?;
+    let body = serde_json::to_vec_pretty(evidence).map_err(|error| {
+        Error::internal_json(
+            error.to_string(),
+            Some("serialize daemon termination evidence".to_string()),
+        )
+    })?;
+    fs::write(&path, body).map_err(|error| {
+        Error::internal_io(error.to_string(), Some(format!("write {}", path.display())))
     })
 }
 
@@ -672,6 +749,12 @@ fn freshness_report_from_validation(
         daemon_build_identity: state.map(|state| state.build_identity.display.clone()),
         runtime_paths: state.map(|state| state.runtime_paths.clone()),
         active_jobs,
+        termination_evidence: (!validation.running)
+            .then(read_termination_evidence)
+            .transpose()
+            .ok()
+            .flatten()
+            .flatten(),
         repair_plan,
     }
 }
@@ -757,6 +840,26 @@ fn stop_unlocked() -> Result<DaemonStopResult> {
         )));
     }
 
+    if pid_is_running(pid) {
+        write_termination_evidence(&DaemonTerminationEvidence {
+            classification: DaemonTerminationClassification::CleanStop,
+            observed_at: chrono::Utc::now().to_rfc3339(),
+            lease_id: Some(state.lease_id.clone()),
+            pid: Some(pid),
+            binary_identity: Some(state.build_identity.display.clone()),
+            active_jobs: JobStore::active_count_at_path(paths::daemon_jobs_file()?)?,
+            resource_evidence: "unavailable: launcher does not collect OS resource snapshots"
+                .to_string(),
+            os_evidence:
+                "unavailable: no OS termination evidence collected before operator-requested stop"
+                    .to_string(),
+            exit_code: None,
+            signal: None,
+            stdout: None,
+            stderr: None,
+            stop_requested: true,
+        })?;
+    }
     let stopped = if pid_is_running(pid) {
         terminate_pid(pid)?;
         true

src/core/daemon/control.rs

@@ -22,7 +22,8 @@ use super::{
     read_status, repair_legacy_lease_for_start, stop_unlocked, try_acquire_daemon_owner_lock,
     DaemonLeaselessOrphanReconciliationResult, DaemonLeaselessRecoveryResult,
     DaemonOrphanAdoptionResult, DaemonProcessCandidate, DaemonProcessOwnership,
-    DaemonStaleReasonCode, DaemonStartResult, DAEMON_STARTUP_TOKEN_ENV,
+    DaemonStaleReasonCode, DaemonStartResult, DaemonTerminationClassification,
+    DaemonTerminationEvidence, DAEMON_STARTUP_TOKEN_ENV,
 };
 
 /// Enumerate foreground daemon processes without inferring ownership from a
@@ -53,6 +54,81 @@ pub(super) fn daemon_process_candidates(jobs_path: &Path) -> Result<Vec<DaemonPr
         .collect())
 }
 
+/// Supervise one daemon child and persist its bounded termination evidence.
+/// This is shared by local and SSH launches because SSH invokes the same CLI.
+pub fn supervise(addr: &str, startup_token: &str) -> Result<()> {
+    let exe = std::env::current_exe().map_err(|error| {
+        Error::internal_io(
+            error.to_string(),
+            Some("resolve current executable".to_string()),
+        )
+    })?;
+    let child = Command::new(exe)
+        .args(["daemon", "serve", "--addr", addr])
+        .env(DAEMON_STARTUP_TOKEN_ENV, startup_token)
+        .stdin(Stdio::null())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|error| {
+            Error::internal_io(
+                error.to_string(),
+                Some("spawn supervised daemon".to_string()),
+            )
+        })?;
+    let pid = child.id();
+    let output = child.wait_with_output().map_err(|error| {
+        Error::internal_io(
+            error.to_string(),
+            Some("wait for supervised daemon".to_string()),
+        )
+    })?;
+    let state = super::validate_lease_file(&crate::core::paths::daemon_state_file()?)
+        .ok()
+        .and_then(|validation| validation.state);
+    let prior = super::read_termination_evidence()?;
+    let stop_requested = prior
+        .as_ref()
+        .is_some_and(|evidence| evidence.stop_requested && evidence.pid == Some(pid));
+    let (exit_code, signal) = exit_details(&output.status);
+    let evidence = DaemonTerminationEvidence {
+        classification: if stop_requested { DaemonTerminationClassification::CleanStop } else { DaemonTerminationClassification::UnexpectedExit },
+        observed_at: chrono::Utc::now().to_rfc3339(),
+        lease_id: state.as_ref().map(|state| state.lease_id.clone()).or_else(|| prior.and_then(|evidence| evidence.lease_id)),
+        pid: Some(pid),
+        binary_identity: state.as_ref().map(|state| state.build_identity.display.clone()),
+        active_jobs: super::JobStore::active_count_at_path(crate::core::paths::daemon_jobs_file()?)?,
+        resource_evidence: "unavailable: launcher does not collect OS resource snapshots".to_string(),
+        os_evidence: "unavailable: no OS evidence collected; exit status and signal are launcher observations only".to_string(),
+        exit_code, signal,
+        stdout: bounded_redacted(&output.stdout), stderr: bounded_redacted(&output.stderr), stop_requested,
+    };
+    super::write_termination_evidence(&evidence)
+}
+
+fn bounded_redacted(bytes: &[u8]) -> Option<String> {
+    const LIMIT: usize = 4096;
+    if bytes.is_empty() {
+        return None;
+    }
+    let mut text = String::from_utf8_lossy(&bytes[..bytes.len().min(LIMIT)]).to_string();
+    if bytes.len() > LIMIT {
+        text.push_str("\n[truncated]");
+    }
+    Some(crate::core::redaction::redact_string(&text))
+}
+
+#[cfg(unix)]
+fn exit_details(status: &std::process::ExitStatus) -> (Option<i32>, Option<i32>) {
+    use std::os::unix::process::ExitStatusExt;
+    (status.code(), status.signal())
+}
+
+#[cfg(not(unix))]
+fn exit_details(status: &std::process::ExitStatus) -> (Option<i32>, Option<i32>) {
+    (status.code(), None)
+}
+
 fn parse_daemon_process_candidate(
     line: &str,
     jobs_path: &Path,
@@ -1199,7 +1275,14 @@ fn spawn_and_wait_for_lease(addr: &str, startup_token: &str) -> Result<DaemonSta
         )
     })?;
     let child = Command::new(exe)
-        .args(["daemon", "serve", "--addr", addr])
+        .args([
+            "daemon",
+            "supervise",
+            "--addr",
+            addr,
+            "--startup-token",
+            startup_token,
+        ])
         .env(DAEMON_STARTUP_TOKEN_ENV, startup_token)
         .stdin(Stdio::null())
         .stdout(Stdio::null())
@@ -1343,6 +1426,59 @@ fn default_artifact_output_path(artifact_id: &str) -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("artifact.bin"))
 }
 
+#[cfg(test)]
+mod termination_tests {
+    use super::*;
+
+    #[test]
+    fn termination_output_is_bounded_and_redacted() {
+        let output =
+            bounded_redacted(format!("token=super-secret\n{}", "x".repeat(5_000)).as_bytes())
+                .expect("output");
+        assert!(output.contains("[REDACTED]"));
+        assert!(output.contains("[truncated]"));
+        assert!(output.len() < 4_200);
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn nonzero_fixture_exit_preserves_exit_status_without_os_cause_inference() {
+        let status = Command::new("sh")
+            .args(["-c", "exit 23"])
+            .status()
+            .expect("fixture process");
+        assert_eq!(exit_details(&status), (Some(23), None));
+        let evidence = DaemonTerminationEvidence {
+            classification: DaemonTerminationClassification::UnexpectedExit,
+            observed_at: "2026-01-01T00:00:00Z".to_string(),
+            lease_id: Some("lease".to_string()),
+            pid: Some(1),
+            binary_identity: Some("fixture".to_string()),
+            active_jobs: 1,
+            resource_evidence: "unavailable: fixture has no OS resource snapshot".to_string(),
+            os_evidence: "unavailable: fixture has no OS evidence".to_string(),
+            exit_code: Some(23),
+            signal: None,
+            stdout: None,
+            stderr: Some("panic: fixture".to_string()),
+            stop_requested: false,
+        };
+        assert_eq!(
+            evidence.classification,
+            DaemonTerminationClassification::UnexpectedExit
+        );
+        assert!(evidence.os_evidence.starts_with("unavailable:"));
+    }
+
+    #[test]
+    fn requested_stop_is_distinct_from_unexpected_exit() {
+        assert_ne!(
+            DaemonTerminationClassification::CleanStop,
+            DaemonTerminationClassification::UnexpectedExit
+        );
+    }
+}
+
 fn header_value(headers: &reqwest::header::HeaderMap, name: &str) -> Option<String> {
     headers
         .get(name)

src/core/paths/runtime.rs

@@ -24,6 +24,11 @@ pub fn daemon_jobs_file() -> Result<PathBuf> {
     Ok(daemon_state_dir()?.join("jobs.json"))
 }
 
+/// Latest bounded launcher-owned termination evidence for this daemon store.
+pub fn daemon_termination_file() -> Result<PathBuf> {
+    Ok(daemon_state_dir()?.join("termination.json"))
+}
+
 /// Exact state-loss recovery receipt keyed by the operator-supplied lease.
 pub fn daemon_state_loss_recovery_receipt_file(lease_id: &str) -> Result<PathBuf> {
     Ok(daemon_state_dir()?