Merge pull request #10546 from Extra-Chill/fix/9624-runner-failure-evidence

Author: chubes4

crates/homeboy-cli/src/commands/runs/evidence.rs

@@ -162,6 +162,19 @@ mod tests {
                         "scenario_metrics": [{"scenario_id":"cold","metrics":{"p95_ms":42.0}}],
                         "resource_policy": {"hot_command":"bench"},
                         "lab": {
+                            "failure": {
+                                "schema": "homeboy/runner-exec-failure-projection/v1",
+                                "failure_code": "validation.invalid_argument",
+                                "phase": "preflight",
+                                "exit_code": 1,
+                                "runner_id": "lab-default",
+                                "runner_job_id": "job-1",
+                                "stderr_tail": "invalid input",
+                                "stderr_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+                                "runner_job_logs_command": "homeboy runner job logs lab-default job-1",
+                                "remote_command_result_command": "homeboy runner job logs lab-default job-1 --json",
+                                "artifact_refs": []
+                            },
                             "remote_events": [{
                                 "data": {
                                     "data": {
@@ -310,6 +323,17 @@ mod tests {
                 .contains("cleanup-persisted --run-id"));
             assert!(output.failure.failed);
             assert_eq!(output.failure.exit_code, Some(1));
+            let runner_failure = output.failure.runner_failure.expect("runner failure");
+            assert_eq!(
+                runner_failure.failure_code.as_deref(),
+                Some("validation.invalid_argument")
+            );
+            assert_eq!(runner_failure.phase.as_deref(), Some("preflight"));
+            assert_eq!(runner_failure.runner_job_id, "job-1");
+            assert_eq!(
+                runner_failure.runner_job_logs_command,
+                "homeboy runner job logs lab-default job-1"
+            );
             assert_eq!(output.failure.gate_failures, vec!["p95_ms exceeded"]);
             assert_eq!(output.failure.hints, vec!["inspect artifacts"]);
             assert_eq!(

crates/homeboy-core/src/observation/evidence_report.rs

@@ -18,7 +18,7 @@
 use std::fs;
 use std::path::{Path, PathBuf};
 
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
 use super::{run_owner_pid, running_status_note, ArtifactRecord, RunRecord};
@@ -249,6 +249,81 @@ pub struct EvidenceFailureSummary {
     pub hints: Vec<String>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub child_command_failures: Vec<Value>,
+    /// Runner-owned terminal diagnostics projected into controller evidence.
+    /// Missing for local and pre-projection records to preserve their schema.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub runner_failure: Option<RunnerFailureEvidence>,
+}
+
+/// Versioned, controller-owned projection of a runner terminal failure.
+/// Unknown or malformed legacy metadata is omitted from evidence rather than
+/// making `runs evidence` unreadable.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RunnerFailureEvidence {
+    pub schema: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub failure_code: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub message: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub details: Option<Value>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub phase: Option<String>,
+    pub exit_code: i64,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub signal: Option<String>,
+    pub stderr_tail: String,
+    /// Digest input is the original runner stderr bytes, before redaction.
+    pub stderr_sha256: String,
+    pub runner_id: String,
+    pub runner_job_id: String,
+    pub runner_job_logs_command: String,
+    pub remote_command_result_command: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub source_snapshot: Option<Value>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub path_materialization_plan: Option<Value>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub runner_job_projection: Option<Value>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub execution_record: Option<Value>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub orchestration_provenance: Option<Value>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub artifact_refs: Vec<RunnerFailureArtifactRef>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RunnerFailureArtifactRef {
+    pub id: String,
+    pub kind: String,
+    pub path: String,
+    pub sha256: String,
+    pub size_bytes: i64,
+}
+
+impl RunnerFailureEvidence {
+    pub const SCHEMA: &'static str = "homeboy/runner-exec-failure-projection/v1";
+
+    pub fn from_metadata(value: &Value) -> Option<Self> {
+        let evidence: Self = serde_json::from_value(value.clone()).ok()?;
+        (evidence.schema == Self::SCHEMA
+            && !evidence.runner_id.is_empty()
+            && !evidence.runner_job_id.is_empty()
+            && evidence.stderr_sha256.len() == 64
+            && evidence
+                .stderr_sha256
+                .bytes()
+                .all(|byte| byte.is_ascii_hexdigit())
+            && evidence.artifact_refs.iter().all(|artifact| {
+                !artifact.id.is_empty()
+                    && !artifact.path.is_empty()
+                    && artifact.sha256.len() == 64
+                    && artifact.sha256.bytes().all(|byte| byte.is_ascii_hexdigit())
+                    && artifact.size_bytes >= 0
+            }))
+        .then_some(evidence)
+    }
 }
 
 #[derive(Serialize)]
@@ -643,6 +718,9 @@ pub fn evidence_failure_summary(run: &RunRecord) -> EvidenceFailureSummary {
         gate_failures: string_array(metadata.get("gate_failures")),
         hints: string_array(metadata.get("hints")),
         child_command_failures: child_command_failures(metadata),
+        runner_failure: metadata
+            .pointer("/lab/failure")
+            .and_then(RunnerFailureEvidence::from_metadata),
     }
 }
 

crates/homeboy-lab-runner/src/connection.rs

@@ -1798,17 +1798,68 @@ fn parse_runner_jobs(
 
 pub fn reverse_broker_reconcile(runner_id: &str) -> Result<Value> {
     let broker_url = reverse_broker_url(runner_id)?;
+    reverse_broker_reconcile_at(runner_id, &broker_url)
+}
+
+/// Reconcile through a persisted reverse broker endpoint without consulting a
+/// live controller-side session.
+pub(crate) fn reverse_broker_reconcile_at(runner_id: &str, broker_url: &str) -> Result<Value> {
     let client = broker_client("build broker reconcile client")?;
     broker_http::post_json(
         &client,
-        &broker_url,
+        broker_url,
         "/runner/jobs/reconcile",
         serde_json::json!({ "runner_id": runner_id }),
         "reconcile reverse runner broker jobs",
         broker_auth::broker_submit_token_for_runner(runner_id)?.as_deref(),
     )
 }
 
+/// Read a reverse runner job from its persisted broker endpoint. This remains
+/// available after a controller restart because it does not require a live
+/// controller-side runner session.
+pub(crate) fn reverse_broker_job_snapshot_at(
+    broker_url: &str,
+    runner_id: &str,
+    job_id: &str,
+) -> Result<(Job, Vec<homeboy_core::api_jobs::JobEvent>)> {
+    let client = broker_client("build reverse broker job snapshot client")?;
+    let token = broker_auth::broker_submit_token_for_runner(runner_id)?;
+    let job_data = broker_http::get_json(
+        &client,
+        broker_url,
+        &format!("/runner/jobs/{job_id}"),
+        "fetch reverse runner broker job",
+        token.as_deref(),
+    )?;
+    let events_data = broker_http::get_json(
+        &client,
+        broker_url,
+        &format!("/runner/jobs/{job_id}/events"),
+        "fetch reverse runner broker job events",
+        token.as_deref(),
+    )?;
+    let job: Job = serde_json::from_value(job_data["job"].clone()).map_err(|error| {
+        Error::internal_json(
+            error.to_string(),
+            Some("parse reverse broker job".to_string()),
+        )
+    })?;
+    if job.id.to_string() != job_id {
+        return Err(Error::internal_unexpected(format!(
+            "reverse broker returned job `{}` while polling requested job `{job_id}`",
+            job.id
+        )));
+    }
+    let events = serde_json::from_value(events_data["events"].clone()).map_err(|error| {
+        Error::internal_json(
+            error.to_string(),
+            Some("parse reverse broker job events".to_string()),
+        )
+    })?;
+    Ok((job, events))
+}
+
 /// Reconcile terminal runner jobs through the session's authoritative transport.
 /// The returned body is transport-neutral so callers retain one command contract.
 pub fn reconcile_terminal_jobs(runner_id: &str) -> Result<Value> {