feat(evidence): produce the evidence manifest instead of only reading it (#10561)

EvidenceManifest had a reader, a consumer, and no producer: every
`runs evidence` report advertised an `evidence_manifest` member that was
structurally always absent, because nothing in the repository ever built one.

- Derive a manifest from the run record when no producer attached one, with
  conservative status mapping (an unowned status label is `unknown`, a pass
  carrying a critical blocker is `blocked`) and a reviewability-graded
  confidence.
- Add `EvidenceManifest.source`, stamped by the reader from the resolution
  path, so an authored assertion is never confused with a derived reading.
  Additive and optional; the type has no `deny_unknown_fields`.
- Fail closed on structurally empty manifests via `EvidenceManifest::validate`,
  reported through `evidence_manifest_errors` rather than published as an
  empty judgement.
- Register `homeboy/evidence-manifest/v1` in the contract registry and wire
  `homeboy contract validate` so an external producer can discover the schema
  and check a candidate before attaching it.

Refs #10306

Co-authored-by: chubes-bot <266378653+homeboy-ci[bot]@users.noreply.github.com>

Author: chubes4

crates/homeboy-cli/src/command_contract/registry.rs

@@ -186,6 +186,14 @@ pub const CONTRACT_REGISTRY: &[ContractRegistryEntry] = &[
         summary: "Captures the selected agent runtime provider, materialization, secrets, readiness, and capabilities.",
         rust_type: "homeboy::agents::agent_task_provider::ResolvedAgentRuntimeExecutionContract",
     },
+    ContractRegistryEntry {
+        schema_id: crate::core::evidence_manifest::EVIDENCE_MANIFEST_SCHEMA,
+        name: "evidence-manifest",
+        title: "Evidence manifest",
+        owner: "homeboy-core",
+        summary: "Portable interpretation of a body of evidence: state, summary, confidence, blocking conditions, and tracker/pull-request/run/artifact references.",
+        rust_type: "homeboy::core::evidence_manifest::EvidenceManifest",
+    },
     ContractRegistryEntry {
         schema_id: crate::core::artifact_ref::ARTIFACT_REF_SCHEMA,
         name: "reviewer-facing-artifact-ref",

crates/homeboy-cli/src/commands/contract/mod.rs

@@ -21,6 +21,7 @@ use crate::command_contract::{
 use crate::commands::{adapter, CmdResult};
 use crate::core::artifact_ref::{validate_reviewer_facing_artifact_ref, ArtifactReference};
 use crate::core::artifacts::{validate_artifact_postprocess_plan, ArtifactManifest};
+use crate::core::evidence_manifest::{EvidenceManifest, EVIDENCE_MANIFEST_SCHEMA};
 use crate::core::host_mutation_lifecycle::{
     HostMutationLifecycle, HostMutationRevertStrategy, HostMutationStatus,
     HOST_MUTATION_LIFECYCLE_SCHEMA,
@@ -1291,6 +1292,7 @@ static CONTRACT_SCHEMAS: &[ContractSchema] = &[
     contract_schema!(RUNNER_EXECUTION_RECORD_SCHEMA, RunnerExecutionRecord),
     contract_schema!(PATH_MATERIALIZATION_PLAN_SCHEMA, PathMaterializationPlan),
     contract_schema!(RUN_OUTCOME_ENVELOPE_SCHEMA, RunOutcomeEnvelope),
+    contract_schema!(EVIDENCE_MANIFEST_SCHEMA, EvidenceManifest),
 ];
 
 trait ContractValidation: DeserializeOwned {
@@ -1333,6 +1335,7 @@ impl_contract_validation! {
     ResourceCleanupIntentContract => RESOURCE_CLEANUP_INTENT_SCHEMA, |value| value.validate();
     ResourceLifecycleIndex => RESOURCE_LIFECYCLE_INDEX_SCHEMA, |value| value.validate();
     HostMutationLifecycle => HOST_MUTATION_LIFECYCLE_SCHEMA, |value| value.validate();
+    EvidenceManifest => EVIDENCE_MANIFEST_SCHEMA, |value| validate_evidence_manifest(&value);
     Value => FUZZ_WORKLOAD_SCHEMA, |value| {
         FuzzWorkload::from_value(value).map_err(|message| {
             homeboy::core::Error::new(
@@ -1349,6 +1352,24 @@ impl_contract_validation! {
     };
 }
 
+/// Bridge the evidence manifest's own structural validation into the shared
+/// contract-validation error envelope, so a producer can check a manifest with
+/// `homeboy contract validate` before attaching it to a run instead of finding
+/// out from `evidence_manifest_errors` after the fact.
+fn validate_evidence_manifest(manifest: &EvidenceManifest) -> homeboy::core::Result<()> {
+    manifest.validate().map_err(|message| {
+        homeboy::core::Error::new(
+            homeboy::core::ErrorCode::ValidationInvalidArgument,
+            "Contract validation failed",
+            serde_json::json!({
+                "schema": EVIDENCE_MANIFEST_SCHEMA,
+                "valid": false,
+                "error": message,
+            }),
+        )
+    })
+}
+
 fn deserialize_contract<T: DeserializeOwned>(
     raw: &str,
     schema_id: &'static str,
@@ -1465,6 +1486,63 @@ mod tests {
         })
     }
 
+    /// The evidence manifest is an inbound contract: a producer outside this
+    /// repository attaches one to a run. That is only usable if the producer can
+    /// discover the schema and check a candidate before attaching it.
+    #[test]
+    fn evidence_manifest_is_discoverable_and_validatable() {
+        let contract =
+            registered_contract("evidence-manifest").expect("evidence manifest registry entry");
+        assert_eq!(contract.schema_id, EVIDENCE_MANIFEST_SCHEMA);
+
+        let dir = TempDir::new().unwrap();
+        let file = write_json(
+            &dir,
+            "manifest.json",
+            json!({
+                "schema": EVIDENCE_MANIFEST_SCHEMA,
+                "status": { "state": "blocked" },
+                "interpretation": {
+                    "summary": "One scenario regressed.",
+                    "confidence": "medium"
+                },
+                "blocking_conditions": [{
+                    "kind": "coverage_gap",
+                    "summary": "Missing scenario.",
+                    "severity": "warning"
+                }]
+            }),
+        );
+
+        let output = validate_file(EVIDENCE_MANIFEST_SCHEMA, file).expect("valid manifest");
+        assert!(output.valid);
+    }
+
+    #[test]
+    fn evidence_manifest_validation_rejects_an_empty_interpretation() {
+        let dir = TempDir::new().unwrap();
+        let file = write_json(
+            &dir,
+            "manifest.json",
+            json!({
+                "schema": EVIDENCE_MANIFEST_SCHEMA,
+                "status": { "state": "passed" },
+                "interpretation": { "summary": "" }
+            }),
+        );
+
+        let err = validate_file(EVIDENCE_MANIFEST_SCHEMA, file).expect_err("invalid manifest");
+        assert_eq!(err.details["valid"], json!(false));
+        assert!(
+            err.details["error"]
+                .as_str()
+                .expect("error detail")
+                .contains("interpretation.summary"),
+            "{:?}",
+            err.details
+        );
+    }
+
     #[test]
     fn command_registry_export_covers_contract_command() {
         let export = command_registry_export();

crates/homeboy-cli/src/commands/runs/evidence.rs

@@ -356,6 +356,16 @@ mod tests {
             assert_eq!(manifest.schema, "homeboy/evidence-manifest/v1");
             assert_eq!(manifest.tracker_refs[0].id, "Extra-Chill/homeboy#123");
             assert_eq!(manifest.blocking_conditions[0].kind, "review_needed");
+            // An attached manifest is surfaced verbatim, stamped with where it
+            // was found rather than replaced by a derived reading.
+            assert_eq!(
+                manifest.source,
+                Some(homeboy::core::evidence_manifest::EvidenceManifestSource::RunMetadata)
+            );
+            assert_eq!(
+                manifest.interpretation.summary,
+                "Evidence is blocked on reviewer confirmation."
+            );
             assert!(output.evidence_manifest_errors.is_empty());
             let lifecycle_event = output
                 .agent_task_lifecycle_event
@@ -372,6 +382,65 @@ mod tests {
         });
     }
 
+    /// Before this wiring the manifest member was structurally always absent:
+    /// nothing in the repository produced one, so `runs evidence` advertised an
+    /// interpretation layer it never populated. A run nobody attached a manifest
+    /// to must still carry one, marked as Homeboy's own reading.
+    #[test]
+    fn evidence_command_derives_a_manifest_when_no_producer_attached_one() {
+        with_isolated_home(|home| {
+            let _xdg = XdgGuard::unset();
+            let _public_artifact_base = EnvGuard::unset(PUBLIC_ARTIFACT_BASE_URL_ENV);
+            let artifact_root = home.path().join("agent-readable-artifacts");
+            homeboy::core::set_artifact_root_override(Some(artifact_root));
+            let store = ObservationStore::open_initialized().expect("store");
+            let run = store
+                .start_run(sample_run(
+                    "bench",
+                    "homeboy",
+                    "studio",
+                    serde_json::json!({ "gate_failures": ["p95_ms exceeded"] }),
+                ))
+                .expect("run");
+            store
+                .finish_run(&run.id, RunStatus::Fail, None)
+                .expect("finish run");
+
+            let (output, _) = evidence(&run.id).expect("evidence");
+            let RunsOutput::Evidence(output) = output else {
+                panic!("expected evidence output");
+            };
+
+            let manifest = output.evidence_manifest.expect("derived evidence manifest");
+            manifest
+                .validate()
+                .expect("derived manifest is contract-valid");
+            assert_eq!(
+                manifest.source,
+                Some(homeboy::core::evidence_manifest::EvidenceManifestSource::Derived)
+            );
+            assert_eq!(
+                manifest.status.state,
+                homeboy::core::evidence_manifest::EvidenceManifestState::Failed
+            );
+            assert_eq!(manifest.id.as_deref(), Some(run.id.as_str()));
+            assert_eq!(manifest.run_refs[0].id, run.id);
+            assert_eq!(manifest.run_refs[0].kind.as_deref(), Some("bench"));
+            assert_eq!(
+                manifest.run_refs[0].component_id.as_deref(),
+                Some("homeboy")
+            );
+            let gate = manifest
+                .blocking_conditions
+                .iter()
+                .find(|condition| condition.kind == "gate_failure")
+                .expect("gate failure blocker");
+            assert_eq!(gate.summary, "p95_ms exceeded");
+            assert!(output.evidence_manifest_errors.is_empty());
+            homeboy::core::set_artifact_root_override(None);
+        });
+    }
+
     #[test]
     fn evidence_command_surfaces_static_html_preview_entrypoints() {
         with_isolated_home(|home| {