diff --git a/docs/command-inventory.md b/docs/command-inventory.md index 85793b60..768a849e 100644 --- a/docs/command-inventory.md +++ b/docs/command-inventory.md @@ -42,7 +42,7 @@ enabled: run `brigade extras on` once, or set `BRIGADE_EXTRAS=1`. - `brigade notifications` (extras): 4 command path(s) - `brigade openclaw-fragments` (extras): 1 command path(s) - `brigade operator`: 24 command path(s) -- `brigade outcome`: 10 command path(s) +- `brigade outcome`: 11 command path(s) - `brigade pantry` (extras): 5 command path(s) - `brigade profiles`: 2 command path(s) - `brigade projects` (extras): 10 command path(s) @@ -282,6 +282,7 @@ enabled: run `brigade extras on` once, or set `BRIGADE_EXTRAS=1`. - `brigade operator sync-mcp` - `brigade operator sync-tools` - `brigade operator verify-harness` +- `brigade outcome backfill scorecard` - `brigade outcome capture` - `brigade outcome diff` - `brigade outcome doctor` diff --git a/docs/proposals/skill-scorecards.md b/docs/proposals/skill-scorecards.md index 6b1cd1e9..d6e032be 100644 --- a/docs/proposals/skill-scorecards.md +++ b/docs/proposals/skill-scorecards.md @@ -390,7 +390,7 @@ authoritative outcome is routing authority: Read-only projection: ``` -brigade outcome backfill scorecard [--target PATH] [--dry-run] [--json] +brigade outcome backfill scorecard [--target PATH] [--json] ``` 1. Walk `.brigade/work/verify-runs/*/receipt.json`. diff --git a/docs/receipt-schemas.md b/docs/receipt-schemas.md index bd401acd..1d8e9567 100644 --- a/docs/receipt-schemas.md +++ b/docs/receipt-schemas.md @@ -62,6 +62,33 @@ JSON Schema files. | `digests` | object | no | `{algorithm, logs, receipt_sha256, signature?, key_id?}` | | `reused_from` | string | no | Prior run id when reused | | `interruption` | object | no | Cancel metadata | +| `verify_manifest_id` | string | no | Registered manifest id when the run was manifest-selected | +| `required_utility_check_ids` | array of string | no | Manifest-owned utility guardrail ids required for scoring (#503) | +| `subject_binding` | object | no | Verifier-authored scoreable subject metadata (manifest runs only) | +| `failure_class` | string | no | Receipt-level #474-style failure class when status is not completed | +| `failure_kind` | string | no | Receipt-level failure kind paired with `failure_class` | + +**`subject_binding` object** (additive, manifest-selected runs) + +| Field | Type | Notes | +| --- | --- | --- | +| `binding_mode` | string | `patch_backed` or `fixture_eval` | +| `artifact_kind` | string | `skill` or `card` | +| `artifact_id` | string | Verifier-owned subject id | +| `content_fingerprint` | string | Subject content fingerprint at verify time | +| `manifest_binding` | object | `{manifest_id, payload_sha256, source_path?}` for the exact tracked verifier manifest | +| `patch_source` | string | `worktree` or `generated` (patch-backed only) | +| `producer_binding` | object | `{work_session_id, owned_delta_sha256, subject_clean_at_start, start_git}` for patch-backed runs | +| `verifier_identity` | object | `{verifier_id, session_id}` independent verifier session | +| `patch_binding` | object | Patch-backed tuple plus `subject_path` and `subject_hash` | +| `fixture_binding` | object | `{manifest_id, case_id, check_id}` for fixture evaluation runs | + +Ad hoc `--command` / `--argv-json` runs omit `subject_binding` and remain audit-only (non-scoreable). + +Tracked workspace verifier manifests live under `verify/manifests/*.json`. A manifest owns its +subject, ordered checks, required utility ids, optional scoped-write globs, and optional route +opt-in (`route_paths` or exact `route_classes`). Untracked manifests cannot produce scoreable +receipts or routing authority. **Command object** @@ -76,6 +103,11 @@ JSON Schema files. | `duration_seconds` | number | | | `stdout_summary`, `stderr_summary` | string | | | `stdout_log_path`, `stderr_log_path` | string | Paths under run dir | +| `check_role` | string | `effectiveness` or `utility_guardrail` (manifest-selected runs) | +| `check_id` | string | Stable verifier-owned check id (manifest-selected runs) | +| `obligation_id` | string | Optional obligation id from the manifest | +| `failure_class` | string | #474-style class when the command did not succeed | +| `failure_kind` | string | Typed failure kind paired with `failure_class` | --- @@ -153,6 +185,7 @@ original file is missing, corrupt, or not an object. | `roster` | object | no | Resolution metadata | | `lock_workspace` | string | no | | | `route` | object | no | Routing brief | +| `skill_route_policy` | object | no | Frozen pre-plan score inputs, assignments, quota counters, and acceptance reasons | | `worker` | string | no | Direct-worker seat | | `git` | object | no | | | `pre_run_snapshot` | object | no | Run-guard snapshot | @@ -253,6 +286,27 @@ and patch-ref binding may rewrite worker/synthesis artifacts). --- +## `brigade.route-decision.v1` + +**Path:** `.brigade/runs//route-decision.json` + +| Field | Type | Required | Notes | +| --- | --- | --- | --- | +| `schema_version` | string | yes | `brigade.route-decision.v1` | +| `chosen_route` | array of string \| null | yes | Route stages selected for the run | +| `confidence`, `template_version` | string \| null | yes | Route metadata | +| `admissible_seats` | array of string | yes | Non-orchestrator seats | +| `decided_at` | string | no | Pre-plan policy timestamp | +| `policy_version` | string | no | Skill route-policy version | +| `score_inputs` | object | no | Receipt-only score inputs keyed by artifact id | +| `skill_assignments` | array of object | no | Band, authority, manifest, scope, and exploration selection | +| `exploration` | object | no | Route class, 7/30-day counters, quota, and accept/reject reasons | + +When skill routing applies, this receipt preserves the decision made before planning. Finalization +must not recompute it from post-run state. + +--- + ## `brigade.roster_snapshot.v1`: `schema_version: 1` **Path:** `.brigade/runs//roster.json` @@ -303,6 +357,7 @@ and patch-ref binding may rewrite worker/synthesis artifacts). | `worker` | string | Assigned seat name | | `task` | string | Task text for the worker | | `covers` | array of string | Optional covered artifact ids | +| `selected_skill_ids` | array of string | Optional pre-plan exploratory skill binding | --- diff --git a/docs/technical-guide.md b/docs/technical-guide.md index c2256033..efb63093 100644 --- a/docs/technical-guide.md +++ b/docs/technical-guide.md @@ -1197,9 +1197,11 @@ An edited skill therefore earns its score back: once signals for the new text st The fingerprint sees the artifact's own files and the cards a card links, not the runtime harness around them, the same caveat CocoIndex documents for undecorated helpers: a hash cannot see a skill reaching into the wider workspace at run time, only the files it is made of. -`brigade outcome reconcile` and `brigade outcome fork` score the same current cohort when they decide. The promote/rollback ratchet no longer folds over the full lifetime ledger: it scores the current-fingerprint cohort, so an edited skill must re-earn `install_min_helped` against the text that now ships instead of coasting on signals for text that no longer exists. The decision *rules* are untouched (thresholds, cooldown, and the forward-only ratchet are the same functions PR #163 left in place); only the score fed in narrows. Because grandfathering leaves a never-edited artifact with no proven-stale records, its `current` equals lifetime and its decision, receipt, and one-line output are byte-identical to the pre-fingerprint ratchet. When a decision does drop stale evidence, the decision receipt and `reconcile` output carry the audit fields (`content_fingerprint`, `lifetime_*`, `stale_records`, `legacy_records`) and a `scored current text only` tail. +`brigade outcome rank` and `brigade outcome explain` retain those ledger cohorts for historical audit. Skill promotion does not use them. `brigade outcome reconcile` and `brigade outcome fork` project skill decisions only from current-fingerprint verifier receipts that pass the subject, patch or fixture identity, verifier ownership, check-role, digest, and failure-taxonomy gates. Legacy rows in `records.jsonl`, including grandfathered rows, cannot promote a skill. Cards continue to use the legacy ledger path. -Two consequences worth stating. A candidate edited before it reaches the install threshold does not promote on the old text's signals; it holds until the new text earns its own. And an already-promoted skill that is edited keeps its `promoted` status until new evidence lands, rather than demoting on the edit alone: its score is now the near-zero current cohort, so it earns no `bump`, and the first verified regression on the new text rolls it back. Demoting purely on an edit would punish improvements, so the ratchet waits for a real signal. +A candidate skill promotes only when both gates pass. Effectiveness requires at least `install_min_helped` independent passes, no trusted hurt, and a Wilson lower bound of at least 0.15. Utility requires two independent passing evidence units and no trusted failure for every verifier-manifest check marked `utility_guardrail`. Retries and `reused_from` copies do not add evidence units. Missing scorecards and incomplete utility evidence produce explicit hold reasons in reconcile and fork output. + +Promotion writes `route_policy.policy_version: scorecard.v1` into the status projection and decision receipt. The router grants full authority only when that marker, promoted status, a current scorecard, and zero trusted hurts agree. One trusted current-fingerprint hurt removes broad authority immediately, before cooldown and regardless of physical rollback success. Install and rollback remain side effects; a failed install does not write promoted status. ### Context eval metric @@ -1211,6 +1213,46 @@ For example, `brief hit rate 0.50 (2/4 files, 2 missed)` means two of four struc This is a coverage quality signal for skill and runbook ranking, not a claim that the context was useful, sufficient, or correct. Brief parsing is heuristic, and GraphTrail deltas only see structural code changes. Docs-only runs and runs without structural graph changes produce no context eval. +### Receipt scorecard backfill + +`brigade outcome backfill scorecard` is a read-only audit of every verify receipt under `.brigade/work/verify-runs/*/receipt.json`. It never mutates `memory/outcome/records.jsonl`, never appends ledger rows, and never joins receipts to ledger `artifact_id` values. Use `--json` for machine-readable output. + +```bash +brigade outcome backfill scorecard --target . +brigade outcome backfill scorecard --target /path/to/repo --json +``` + +Each discovered `receipt.json` path is counted in `total_receipts`, including malformed files. When a file is unreadable or its JSON is not an object, the audit records one ineligible, unattributed row with stable reason `invalid_receipt_json`. + +Stable JSON fields: + +| Field | Meaning | +| --- | --- | +| `total_receipts` | Count of discovered `receipt.json` paths | +| `eligible` | Receipts that pass scorecard eligibility rules | +| `ineligible` | `total_receipts - eligible`; numerator for `ineligibility_rate` | +| `attributed_ineligible` | Ineligible receipts that carry verifier `subject_binding` | +| `unattributed` | Receipts without attributable `subject_binding` | +| `attributed` | `total_receipts - unattributed` | +| `ineligibility_rate` | `ineligible / total_receipts` (0.0 when empty) | +| `ineligible_by_reason` | Map of stable reason codes to counts; values sum to `ineligible` | +| `leading_ineligibility_reason` | Highest-count reason in `ineligible_by_reason` (ties break lexicographically) | +| `exploration_bands` | Attributed subjects by `unseen`, `candidate`, `provisional`, or `promoted` | +| `latest_receipt_window` | Rolling view of the latest 50 receipt audits | +| `legacy_records_audit_only` | Always `true` for this command | +| `legacy_records_note` | Explains ledger rows are not backfilled into scorecards | + +`eligible + ineligible` always equals `total_receipts`. + +`latest_receipt_window` sorts receipt audits by `started_at`, then `run_id`, then `receipt_path`, all descending (lexicographic). The first 50 audits in that order form the window regardless of eligibility. Nested fields include `limit` (50), `count`, `eligible`, `ineligible`, `ineligibility_rate`, and `leading_ineligibility_reason`. + +Operator surfaces reuse the same audit: + +- `brigade work brief` copies those fields under `outcome_loop`. +- `brigade operator checkup --surface outcome` warns when the loop is half-fed (`outcome_loop_half_fed`) or when more than 50% of the latest 50 receipts are ineligible (`outcome_receipt_ineligibility_high`). Its JSON includes `eligible_receipt_count`, `ineligible_receipt_count`, `attributed_ineligible_receipt_count`, `unattributed_receipt_count`, `ineligibility_rate`, `exploration_bands`, and `latest_receipt_window`. + +The 1,601 legacy outcome ledger rows reported in the scorecard proposal are audit-only. They cannot be converted into receipt scorecards because scorecards require verifier-attributed verify receipts, not caller-supplied ledger `artifact_id` values. + `brigade operator checkup` runs the six first-run doctors by default and reports optional loop station health alongside them. Missing optional stations warn and do not block the default ready verdict. Use repeatable `--surface` values to run only named checks, `--list-surfaces` to inspect the stable names, or `--preset evidence-loop` to gate only work receipt integrity and outcome capture, GraphTrail health and the latest work receipt delta, and MiseLedger work-receipt import state. Scoped JSON reports `selected_ready`, leaves `overall_ready` unevaluated, and includes selected, skipped, and per-surface elapsed data. Use `--handoff` to bridge a completed run back into the memory system. diff --git a/src/brigade/aboyeur.py b/src/brigade/aboyeur.py index 4afcd809..25e050bf 100644 --- a/src/brigade/aboyeur.py +++ b/src/brigade/aboyeur.py @@ -13,7 +13,7 @@ from contextlib import contextmanager from dataclasses import dataclass, replace from datetime import datetime, timezone -from functools import wraps +from functools import partial, wraps from json import JSONDecoder from pathlib import Path from typing import Any, Callable, Iterator @@ -39,6 +39,13 @@ from .run_transport import Assignment, WorkerResult from .roster import Agent, Roster, is_cli_allowed, read_only_capability_error, timeout_for, workers from .route_catalog import RouteBrief, route_brief, uncovered_stages, unknown_covers +from .route_policy import ( + RoutePolicyDecision, + direct_worker_skill_ids, + planner_skill_policy_section, + validate_plan_skill_bindings, + worker_skill_policy_constraint, +) CODE_GRAPH_HEADING = "## Code graph context (GraphTrail, read-only)" CODE_GRAPH_LIMIT = 4000 @@ -385,6 +392,7 @@ def build_plan_prompt( evidence: EvidenceBrief | None = None, route: RouteBrief | None = None, no_file_writes: bool = False, + skill_policy: RoutePolicyDecision | None = None, ) -> str: worker_lines = "\n".join( f"- {agent.name}: cli={agent.cli}; " @@ -407,14 +415,20 @@ def build_plan_prompt( '\n- Tag each assignment with "covers": ["", ...] naming the route ' "stages it satisfies; every required route stage must be covered." ) + skill_section = "" + skill_text = planner_skill_policy_section(skill_policy) + if skill_text: + skill_section = f"\n{skill_text}" prompt = ( "You are the Brigade aboyeur. Split the user's task across the available workers.\n" "Return exactly one JSON object, with no prose outside JSON:\n" - '{"assignments":[{"stage":1,"worker":"","task":"","covers":[""]}]}\n' + '{"assignments":[{"stage":1,"worker":"","task":"",' + '"covers":[""],"selected_skill_ids":[""]}]}\n' f"{note}\n" f"User task:\n{task}\n\n" f"Available workers, excluding you:\n{worker_lines}\n" - f"{route_section}\n" + f"{route_section}" + f"{skill_section}\n" f"Rules:\n- Use at most {roster.max_workers} assignments per stage.\n" "- Stage must be a positive integer starting at stage 1.\n" "- Assignments in the same stage run in parallel; later stages receive earlier-stage worker results.\n" @@ -598,7 +612,13 @@ def _read_only_rules() -> str: ) -def parse_plan(text: str, roster: Roster, *, read_only: bool = False) -> list[Assignment]: +def parse_plan( + text: str, + roster: Roster, + *, + read_only: bool = False, + skill_policy: RoutePolicyDecision | None = None, +) -> list[Assignment]: try: payload = _extract_json(text) except json.JSONDecodeError as exc: @@ -638,7 +658,21 @@ def parse_plan(text: str, roster: Roster, *, read_only: bool = False) -> list[As if not isinstance(raw_covers, list) or any(not isinstance(c, str) or not c.strip() for c in raw_covers): raise ValueError("assignment.covers must be a list of non-empty strings") covers = tuple(dict.fromkeys(c.strip() for c in raw_covers)) - assignment = Assignment(worker=worker, task=subtask.strip(), stage=stage, covers=covers) + raw_selected = item.get("selected_skill_ids", []) + if raw_selected is None: + raw_selected = [] + if not isinstance(raw_selected, list) or any( + not isinstance(skill_id, str) or not skill_id.strip() for skill_id in raw_selected + ): + raise ValueError("assignment.selected_skill_ids must be a list of non-empty strings when set") + selected_skill_ids = tuple(dict.fromkeys(skill_id.strip() for skill_id in raw_selected)) + assignment = Assignment( + worker=worker, + task=subtask.strip(), + stage=stage, + covers=covers, + selected_skill_ids=selected_skill_ids, + ) key = (assignment.stage, assignment.worker, assignment.task) if key not in seen: assignments.append(assignment) @@ -650,12 +684,18 @@ def parse_plan(text: str, roster: Roster, *, read_only: bool = False) -> list[As for index, existing in enumerate(assignments): if (existing.stage, existing.worker, existing.task) == key: merged = tuple(dict.fromkeys(existing.covers + covers)) - assignments[index] = replace(existing, covers=merged) + merged_skills = tuple(dict.fromkeys(existing.selected_skill_ids + selected_skill_ids)) + assignments[index] = replace( + existing, + covers=merged, + selected_skill_ids=merged_skills, + ) break for stage, count in stage_counts.items(): if count > roster.max_workers: raise ValueError(f"plan has {count} assignments in stage {stage}, limit is {roster.max_workers}") + validate_plan_skill_bindings(assignments, skill_policy) return sorted(assignments, key=lambda assignment: assignment.stage) @@ -853,6 +893,7 @@ def plan( drift_impact: DriftImpactBrief | None = None, evidence: EvidenceBrief | None = None, route: RouteBrief | None = None, + skill_policy: RoutePolicyDecision | None = None, codex_transport: str | None = None, process_registry: proc.ProcessRegistry | None = None, ) -> list[Assignment]: @@ -875,6 +916,7 @@ def plan( evidence=evidence, route=route, no_file_writes=no_file_writes, + skill_policy=skill_policy, ), cwd=cwd, read_only=read_only, @@ -887,7 +929,7 @@ def plan( _record_plan_attempt(attempts, stage="initial", result=first) raise RuntimeError(f"orchestrator failed during plan: {first.detail}") try: - assignments = parse_plan(first.text, roster, read_only=read_only) + assignments = parse_plan(first.text, roster, read_only=read_only, skill_policy=skill_policy) _record_plan_attempt( attempts, stage="initial", @@ -913,6 +955,7 @@ def plan( evidence=evidence, route=route, no_file_writes=no_file_writes, + skill_policy=skill_policy, ), cwd=cwd, read_only=read_only, @@ -925,7 +968,7 @@ def plan( _record_plan_attempt(attempts, stage="correction", result=second) raise RuntimeError(f"orchestrator failed during plan correction: {second.detail}") from exc try: - assignments = parse_plan(second.text, roster, read_only=read_only) + assignments = parse_plan(second.text, roster, read_only=read_only, skill_policy=skill_policy) _record_plan_attempt( attempts, stage="correction", @@ -966,6 +1009,7 @@ def plan( evidence=evidence, route=route, no_file_writes=no_file_writes, + skill_policy=skill_policy, ), cwd=cwd, read_only=read_only, @@ -978,7 +1022,7 @@ def plan( _record_plan_attempt(attempts, stage="coverage-correction", result=revised_result) return assignments try: - revised = parse_plan(revised_result.text, roster, read_only=read_only) + revised = parse_plan(revised_result.text, roster, read_only=read_only, skill_policy=skill_policy) except ValueError as exc: _record_plan_attempt(attempts, stage="coverage-correction", result=revised_result, parse_error=str(exc)) return assignments @@ -1024,11 +1068,13 @@ def _worker_prompt( code_graph: CodeGraphBrief | None = None, drift_impact: DriftImpactBrief | None = None, evidence: EvidenceBrief | None = None, + skill_policy: RoutePolicyDecision | None = None, ) -> str: prior_context = "" if prior_results: prior_context = f"\n\nEarlier-stage context:\n{_render_prior_results(prior_results)}" policy = f"\n\n{_read_only_rules()}" if read_only else "" + scope_policy = worker_skill_policy_constraint(skill_policy, assignment) return_instruction = ( "Return a concise, complete final user-visible result." if direct @@ -1040,6 +1086,7 @@ def _worker_prompt( f"Sub-task:\n{assignment.task}\n\n" f"{return_instruction}" f"{prior_context}" + f"{scope_policy}" f"{policy}" ) return _prepend_optional_briefs(prompt, code_graph=code_graph, drift_impact=drift_impact, evidence=evidence) @@ -2010,6 +2057,7 @@ def _run_payload( context_eval_payload: dict[str, object] | None = None, suspected_noop: bool = False, route: RouteBrief | None = None, + skill_route_policy: RoutePolicyDecision | None = None, worker: str | None = None, include_git: bool = True, pre_run_snapshot: dict[str, object] | None = None, @@ -2054,6 +2102,10 @@ def _run_payload( payload["lock_workspace"] = str(lock_workspace) if route is not None: payload["route"] = route.payload() + if skill_route_policy is not None and skill_route_policy.policy_applied: + from .route_policy import route_policy_extensions_from_decision + + payload["skill_route_policy"] = route_policy_extensions_from_decision(skill_route_policy) if worker is not None: payload["worker"] = worker if include_git: @@ -2319,6 +2371,7 @@ def run( route_template: str | None = None, route_overrides: tuple[str, ...] = (), worker: str | None = None, + allow_shadow: bool = False, authorized_writable_worktree: bool = False, fail_fast: bool = True, scheduler: str = "waves", @@ -2347,6 +2400,8 @@ def scheduler_resolved(used: str, fallback_reason: str | None) -> None: scheduler_resolution["fallback_reason"] = fallback_reason def _payload(**kwargs: Any) -> dict[str, object]: + if "skill_route_policy" not in kwargs and skill_policy is not None: + kwargs["skill_route_policy"] = skill_policy return _run_payload( lock_workspace=lock_workspace, pre_run_snapshot=pre_run_snapshot_payload, @@ -2354,6 +2409,8 @@ def _payload(**kwargs: Any) -> dict[str, object]: **kwargs, ) + skill_policy: RoutePolicyDecision | None = None + # Capture pre-run git state before any worker touches the tree so ground # truth can attribute only the worker's changes and a drift check can fail # the run if branch or HEAD moves out from under it. The in-memory snapshot @@ -2472,11 +2529,65 @@ def _drift_failure_rc() -> int | None: ), ) + if cwd is not None and route is not None and route.attached: + from .route_policy import decide_route_skills, route_policy_extensions_from_decision + from .route_receipts import write_route_decision + + skill_policy = decide_route_skills( + cwd, + route_brief=route, + runs_dir=output_dir.parent if output_dir is not None else None, + now=started_at, + allow_shadow=allow_shadow if direct_worker else True, + ) + if output_dir is not None and skill_policy.policy_applied: + write_route_decision( + output_dir, + roster, + runs_dir=output_dir.parent, + policy_extensions=route_policy_extensions_from_decision(skill_policy), + ) + _write_json( + output_dir / "run.json", + _payload( + task=task, + cwd=cwd, + roster=roster, + dry_run=dry_run, + read_only=read_only, + status="started", + started_at=started_at, + output_dir=output_dir, + code_graph=code_graph, + drift_impact=drift_impact, + evidence=evidence, + brief_set=brief_set, + codex_transport=transport_for_payload, + route=route, + code_graph_delta=code_graph_delta, + worker=worker, + ), + ) + control_socket = None control_transport = None plan_attempts: list[dict[str, object]] | None = [] if output_dir is not None else None if worker is not None: - assignments = [Assignment(worker=worker, task=task, stage=1)] + selected_skill_ids, bind_error = direct_worker_skill_ids( + skill_policy, + allow_shadow=allow_shadow, + ) + if bind_error is not None: + print(f"error: {bind_error}", file=sys.stderr) + return 2 + assignments = [ + Assignment( + worker=worker, + task=task, + stage=1, + selected_skill_ids=selected_skill_ids, + ) + ] else: if output_dir is not None: _write_json( @@ -2514,6 +2625,7 @@ def _drift_failure_rc() -> int | None: drift_impact=drift_impact, evidence=evidence, route=route, + skill_policy=skill_policy, codex_transport=transport_for_payload, process_registry=process_registry, ) @@ -2755,6 +2867,7 @@ def dispatch_interrupted() -> None: ) active_seat = active_seats[0] if len(active_seats) == 1 else None + worker_prompt_builder = partial(_worker_prompt, skill_policy=skill_policy) try: try: worker_results = _call_with_process_registry( @@ -2782,6 +2895,7 @@ def dispatch_interrupted() -> None: on_interrupt=dispatch_interrupted, on_scheduler_resolved=scheduler_resolved, process_registry=process_registry, + build_prompt=worker_prompt_builder, ) except runguard.RetainRunLockError: raise diff --git a/src/brigade/cli/outcome.py b/src/brigade/cli/outcome.py index 34df43ce..d96e8e4d 100644 --- a/src/brigade/cli/outcome.py +++ b/src/brigade/cli/outcome.py @@ -90,6 +90,20 @@ def register(sub: argparse._SubParsersAction) -> None: p_fork.add_argument("--revert-min-hurt", type=int, default=None, help="Override revert_min_hurt.") p_fork.add_argument("--bump-min-helped", type=int, default=None, help="Override bump_min_helped.") p_fork.add_argument("--z", type=float, default=None, help="Override the Wilson z score.") + p_fork.add_argument( + "--effective-wilson-min", + type=float, + default=None, + help="Override the scorecard effectiveness Wilson lower-bound threshold.", + ) + p_fork.add_argument( + "--utility-check-min-passing-units", + "--utility-min-passing-units", + type=int, + default=None, + dest="utility_min_passing_units", + help="Override independent passing units required per utility guardrail check.", + ) p_fork.add_argument("--target", "-t", type=Path, default=Path(".")) p_fork.add_argument("--json", action="store_true", help="Emit machine-readable JSON instead of text.") p_fork.set_defaults(func=_dispatch_fork) @@ -112,6 +126,26 @@ def register(sub: argparse._SubParsersAction) -> None: p_record.add_argument("--json", action="store_true", help="Emit machine-readable JSON instead of text.") p_record.set_defaults(func=_dispatch_record) + p_backfill = outcome_sub.add_parser( + "backfill", + help="Read-only verify-receipt scorecard audits (never mutates records.jsonl).", + ) + backfill_sub = p_backfill.add_subparsers(dest="outcome_backfill_command", metavar="") + backfill_sub.required = True + p_backfill_scorecard = backfill_sub.add_parser( + "scorecard", + help="Audit verify receipts for scorecard eligibility without ledger joins.", + ) + p_backfill_scorecard.add_argument("--target", "-t", type=Path, default=Path(".")) + p_backfill_scorecard.add_argument("--json", action="store_true", help="Emit machine-readable JSON instead of text.") + p_backfill_scorecard.set_defaults(func=_dispatch_backfill_scorecard) + + +def _dispatch_backfill_scorecard(args) -> int: + from .. import outcome_cmd + + return outcome_cmd.backfill_scorecard(target=args.target, json_output=args.json) + def _dispatch_score(args) -> int: from .. import outcome_cmd @@ -183,6 +217,12 @@ def _dispatch_fork(args) -> int: bump_min_helped=args.bump_min_helped if args.bump_min_helped is not None else defaults.bump_min_helped, cooldown_seconds=defaults.cooldown_seconds, z=args.z if args.z is not None else defaults.z, + effective_wilson_min=args.effective_wilson_min + if args.effective_wilson_min is not None + else defaults.effective_wilson_min, + utility_min_passing_units=args.utility_min_passing_units + if args.utility_min_passing_units is not None + else defaults.utility_min_passing_units, ) return outcome_cmd.fork(target=args.target, out=args.out, config=config, json_output=args.json) diff --git a/src/brigade/cli/run.py b/src/brigade/cli/run.py index 15509f5d..af21af6d 100644 --- a/src/brigade/cli/run.py +++ b/src/brigade/cli/run.py @@ -527,7 +527,7 @@ def dispatch(args) -> int: if output_dir is not None and (output_dir / "run.json").is_file(): from ..route_receipts import write_route_decision - write_route_decision(output_dir, loaded_roster) + write_route_decision(output_dir, loaded_roster, target=run_cwd) if args.worktree and output_dir is not None: # Until the patch is proven good, the worktree is the only # recoverable copy of the agents' edits; a collection failure diff --git a/src/brigade/cli/work/dispatching.py b/src/brigade/cli/work/dispatching.py index fbd9731d..bf0edb33 100644 --- a/src/brigade/cli/work/dispatching.py +++ b/src/brigade/cli/work/dispatching.py @@ -116,14 +116,38 @@ def dispatch(args) -> int: return work_cmd.sweep_review(target=args.target, sweep_id=args.sweep_id, json_output=args.json) if args.work_command == "verify": if args.verify_command == "plan": - return work_cmd.verify_plan(target=args.target, commands=args.verify_commands, json_output=args.json) + has_manifest = bool(args.verify_manifest_id) + has_command = bool(args.verify_commands) + if has_manifest and has_command: + args._brigade_parser.error("--manifest and --command are mutually exclusive") + return work_cmd.verify_plan( + target=args.target, + commands=args.verify_commands, + manifest_id=args.verify_manifest_id, + json_output=args.json, + ) if args.verify_command == "run": has_command = bool(args.verify_commands) has_argv_json = args.verify_argv_json is not None - if has_command and has_argv_json: - args._brigade_parser.error("--command and --argv-json are mutually exclusive") - if not has_command and not has_argv_json: - args._brigade_parser.error("work verify run requires exactly one of --command or --argv-json") + has_manifest = bool(args.verify_manifest_id) + selected = sum((has_command, has_argv_json, has_manifest)) + if selected > 1: + args._brigade_parser.error("--command, --argv-json, and --manifest are mutually exclusive") + if selected == 0: + args._brigade_parser.error( + "work verify run requires exactly one of --command, --argv-json, or --manifest" + ) + if has_manifest: + return work_cmd.verify_run( + target=args.target, + manifest_id=args.verify_manifest_id, + timeout=args.timeout, + graphtrail_timeout=args.graphtrail_timeout, + json_output=args.json, + capture=args.capture, + capture_kind=args.capture_kind, + reuse=not args.no_reuse, + ) if has_argv_json: try: parsed_argv = json.loads(args.verify_argv_json) @@ -141,6 +165,7 @@ def dispatch(args) -> int: return work_cmd.verify_run( target=args.target, commands=commands, + manifest_id=None, timeout=args.timeout, graphtrail_timeout=args.graphtrail_timeout, json_output=args.json, diff --git a/src/brigade/cli/work/registration.py b/src/brigade/cli/work/registration.py index 0eab039e..cb6e332d 100644 --- a/src/brigade/cli/work/registration.py +++ b/src/brigade/cli/work/registration.py @@ -154,6 +154,12 @@ def register(sub: argparse._SubParsersAction) -> None: default=None, help="Verification command. May be repeated.", ) + p_work_verify_plan.add_argument( + "--manifest", + dest="verify_manifest_id", + default=None, + help="Registered verifier manifest id. Selects scoreable checks; mutually exclusive with --command.", + ) p_work_verify_plan.add_argument("--json", action="store_true", help="Print machine-readable JSON.") p_work_verify_run = verify_sub.add_parser("run", help="Run local verification commands and write a receipt.") p_work_verify_run.add_argument("--target", "-t", type=Path, default=Path("."), help="Repo or workspace to update.") @@ -175,6 +181,15 @@ def register(sub: argparse._SubParsersAction) -> None: "arguments need punctuation like ';' or quotes). Mutually exclusive with --command." ), ) + p_work_verify_run.add_argument( + "--manifest", + dest="verify_manifest_id", + default=None, + help=( + "Registered verifier manifest id. Runs manifest-owned checks with scoreable " + "subject/check metadata. Mutually exclusive with --command and --argv-json." + ), + ) p_work_verify_run.add_argument("--timeout", type=int, default=900, help="Timeout per command in seconds.") p_work_verify_run.add_argument( "--graphtrail-timeout", diff --git a/src/brigade/operator_cmd/lifecycle.py b/src/brigade/operator_cmd/lifecycle.py index 0773981d..1ecfb51b 100644 --- a/src/brigade/operator_cmd/lifecycle.py +++ b/src/brigade/operator_cmd/lifecycle.py @@ -858,7 +858,8 @@ def _surface_issue_count(payload: dict[str, Any]) -> int | None: CHECKUP_DEFAULT_SURFACES = ("doctor", "operator", "handoff", "tools", "skills", "security") CHECKUP_EVIDENCE_SURFACES = ("work", "graph", "ledger") -CHECKUP_SURFACE_NAMES = (*CHECKUP_DEFAULT_SURFACES, *CHECKUP_EVIDENCE_SURFACES) +CHECKUP_OPTIONAL_SURFACES = (*CHECKUP_EVIDENCE_SURFACES, "outcome") +CHECKUP_SURFACE_NAMES = (*CHECKUP_DEFAULT_SURFACES, *CHECKUP_OPTIONAL_SURFACES) CHECKUP_PRESETS = {"evidence-loop": CHECKUP_EVIDENCE_SURFACES} @@ -866,6 +867,7 @@ def checkup_catalog_payload() -> dict[str, Any]: return { "surface_names": list(CHECKUP_SURFACE_NAMES), "default_surfaces": list(CHECKUP_DEFAULT_SURFACES), + "optional_surfaces": list(CHECKUP_OPTIONAL_SURFACES), "presets": {name: list(values) for name, values in CHECKUP_PRESETS.items()}, } @@ -886,7 +888,7 @@ def _resolve_checkup_surfaces(surfaces: list[str] | None, preset: str | None) -> raise ValueError(f"unknown checkup surface: {', '.join(unknown)}") skipped = [name for name in CHECKUP_SURFACE_NAMES if name not in selected] return selected, skipped, True - return list(CHECKUP_DEFAULT_SURFACES), list(CHECKUP_EVIDENCE_SURFACES), False + return list(CHECKUP_DEFAULT_SURFACES), list(CHECKUP_OPTIONAL_SURFACES), False def _print_checkup_surface(payload: dict[str, Any], *, json_output: bool) -> None: @@ -946,9 +948,9 @@ def _checkup_work(*, target: Path, json_output: bool = False) -> int: "next_command": ( "brigade receipts verify --target ." if integrity_failures - else "brigade work verify run --target . --command '' --capture brigade-work" + else "brigade work verify run --target . --manifest " if latest is None - else "brigade outcome capture --run-id latest" + else "brigade work verify run --target . --manifest " if outcome_issue_count else None ), @@ -1053,6 +1055,66 @@ def _checkup_ledger(*, target: Path, json_output: bool = False) -> int: return 0 if ready else 1 +def _checkup_outcome(*, target: Path, json_output: bool = False) -> int: + """Check receipt-only outcome loop health and latest-receipt eligibility (#574).""" + from .. import outcome_cmd, scorecard + + health = outcome_cmd.health(target) + issues: list[dict[str, Any]] = [] + top_issue = health.get("top_issue") if isinstance(health.get("top_issue"), dict) else None + if top_issue and top_issue.get("name") == "outcome_loop_half_fed": + issues.append(top_issue) + + latest_window = health.get("latest_receipt_window") + latest_window = latest_window if isinstance(latest_window, dict) else {} + latest_rate = float(latest_window.get("ineligibility_rate") or 0.0) + if latest_window.get("count", 0) and latest_rate > 0.5: + leading = latest_window.get("leading_ineligibility_reason") or "unknown" + issues.append( + { + "status": "warn", + "name": "outcome_receipt_ineligibility_high", + "detail": ( + f">{int(0.5 * 100)}% of the latest {scorecard.LATEST_RECEIPT_WINDOW} receipts are " + f"ineligible (leading reason: {leading})" + ), + } + ) + + issue_count = len(issues) + ready = issue_count == 0 + payload = { + "status": "ok" if ready else "warn", + "ready": ready, + "summary": ( + "receipt-only outcome loop is healthy" if ready else f"receipt-only outcome loop has {issue_count} issue(s)" + ), + "issue_count": issue_count, + "issues": issues, + "verify_run_count": health.get("verify_run_count"), + "attributed_receipt_count": health.get("attributed_receipt_count"), + "unattributed_receipt_count": health.get("unattributed_receipt_count"), + "eligible_receipt_count": health.get("eligible_receipt_count"), + "ineligible_receipt_count": health.get("ineligible_receipt_count"), + "attributed_ineligible_receipt_count": health.get("attributed_ineligible_receipt_count"), + "ineligibility_rate": health.get("ineligibility_rate"), + "leading_ineligibility_reason": health.get("leading_ineligibility_reason"), + "exploration_bands": health.get("exploration_bands"), + "latest_receipt_window": latest_window, + "legacy_records_audit_only": health.get("legacy_records_audit_only"), + "legacy_records_note": health.get("legacy_records_note"), + "next_command": ( + "brigade work verify run --target . --manifest " + if top_issue and top_issue.get("name") == "outcome_loop_half_fed" + else "brigade outcome backfill scorecard --target . --json" + if issue_count + else None + ), + } + _print_checkup_surface(payload, json_output=json_output) + return 0 if ready else 1 + + def _loop_stations_payload(target: Path) -> dict[str, Any]: """Report GraphTrail / MiseLedger / context-eval loop health (informational). @@ -1173,6 +1235,7 @@ def checkup_payload( ("work", "brigade operator checkup --target . --surface work", _checkup_work, {"target": target}), ("graph", "brigade operator checkup --target . --surface graph", _checkup_graph, {"target": target}), ("ledger", "brigade operator checkup --target . --surface ledger", _checkup_ledger, {"target": target}), + ("outcome", "brigade operator checkup --target . --surface outcome", _checkup_outcome, {"target": target}), ] spec_by_name = {row[0]: row for row in spec} surface_results: list[dict[str, Any]] = [] @@ -1195,7 +1258,7 @@ def checkup_payload( "issue_count": _surface_issue_count(payload), "elapsed_seconds": elapsed, } - if name in CHECKUP_EVIDENCE_SURFACES: + if name in CHECKUP_OPTIONAL_SURFACES: result["details"] = payload surface_results.append(result) selected_ready = blocking == 0 diff --git a/src/brigade/outcome.py b/src/brigade/outcome.py index 51961071..19e8be41 100644 --- a/src/brigade/outcome.py +++ b/src/brigade/outcome.py @@ -94,6 +94,8 @@ class ReconcileConfig: bump_min_helped: int = 3 cooldown_seconds: int = 86_400 z: float = 1.96 + effective_wilson_min: float = 0.15 + utility_min_passing_units: int = 2 @dataclass(frozen=True) @@ -439,11 +441,15 @@ class StatusTransition: The decision receipts under ``memory/outcome/decisions/`` are the transition log; ``status.json`` is the cache they fold into. Keeping this pure lets the rebuild check prove the cache is reproducible from the log. + + ``route_policy`` is optional and additive: promoted skill receipts may carry + the routing-authority marker that ``status.json`` caches alongside status. """ artifact_id: str new_status: str created_at: str # ISO 8601 + route_policy: dict[str, Any] | None = None def fold_status(transitions: list[StatusTransition]) -> dict[str, dict]: @@ -458,7 +464,10 @@ def fold_status(transitions: list[StatusTransition]) -> dict[str, dict]: ordered = sorted(transitions, key=lambda t: (t.created_at, t.artifact_id)) status: dict[str, dict] = {} for t in ordered: - status[t.artifact_id] = {"status": t.new_status, "last_action_ts": t.created_at} + entry: dict[str, Any] = {"status": t.new_status, "last_action_ts": t.created_at} + if t.route_policy is not None: + entry["route_policy"] = t.route_policy + status[t.artifact_id] = entry return status diff --git a/src/brigade/outcome_cmd.py b/src/brigade/outcome_cmd.py index db615a07..8a342d59 100644 --- a/src/brigade/outcome_cmd.py +++ b/src/brigade/outcome_cmd.py @@ -19,7 +19,15 @@ from pathlib import Path from typing import Any -from . import localio, outcome as core, receipt_schema +from . import localio, outcome as core, receipt_schema, scorecard as scorecard_mod + + +@dataclasses.dataclass(frozen=True) +class _ReconcileItem: + decision: core.Decision + prior_status: str + cohorts: core.FingerprintCohorts | None = None + scorecard: scorecard_mod.SubjectScorecard | None = None def _records_path(target: Path) -> Path: @@ -388,6 +396,110 @@ def route_fingerprint(manifest: dict[str, Any]) -> str | None: return localio.canonical_json_digest(vector) +def _scorecards_by_artifact(target: Path) -> dict[str, scorecard_mod.SubjectScorecard]: + return {card.subject.artifact_id: card for card in scorecard_mod.build_scorecards(target)} + + +def _artifact_kinds( + records: list[core.OutcomeRecord], + status_map: dict[str, dict], + scorecards: dict[str, scorecard_mod.SubjectScorecard], +) -> dict[str, str]: + kinds: dict[str, str] = {} + for record in records: + kinds.setdefault(record.artifact_id, record.artifact_kind or "skill") + for artifact_id in status_map: + kinds.setdefault(artifact_id, "skill") + for artifact_id, card in scorecards.items(): + kinds.setdefault(artifact_id, card.subject.artifact_kind) + return kinds + + +def _reconcile_artifact_ids( + kinds: dict[str, str], + status_map: dict[str, dict], + scorecards: dict[str, scorecard_mod.SubjectScorecard], + cohorts_by_artifact: dict[str, core.FingerprintCohorts], +) -> list[str]: + artifact_ids: set[str] = set() + for artifact_id, kind in kinds.items(): + if kind == "card": + if artifact_id in cohorts_by_artifact: + artifact_ids.add(artifact_id) + continue + if artifact_id in scorecards or artifact_id in status_map or artifact_id in cohorts_by_artifact: + artifact_ids.add(artifact_id) + return sorted(artifact_ids) + + +def _decide_reconcile_item( + artifact_id: str, + artifact_kind: str, + *, + cohorts: core.FingerprintCohorts | None, + card: scorecard_mod.SubjectScorecard | None, + prior_status: str, + last_action_ts: Any, + now: Any, + config: core.ReconcileConfig, +) -> core.Decision: + if artifact_kind == "card": + if cohorts is None: + return core.Decision(artifact_id, "hold", prior_status, "no scored evidence") + return core.decide( + cohorts.current, + current_status=prior_status, + last_action_ts=last_action_ts, + now=now, + config=config, + ) + return scorecard_mod.decide_scorecard( + card, + artifact_id=artifact_id, + current_status=prior_status, + last_action_ts=last_action_ts, + now=now, + config=config, + ) + + +def _surface_hold_decision(decision: core.Decision, prior_status: str) -> bool: + """Whether a hold should appear in reconcile/fork output (fail-closed, auditable).""" + if decision.action != "hold": + return False + if prior_status != "candidate": + return False + return decision.reason != "cooldown active" + + +def _scorecard_decision_fields(card: scorecard_mod.SubjectScorecard | None) -> dict[str, Any]: + if card is None: + return {} + return { + "policy_version": scorecard_mod.SCORECARD_POLICY_VERSION, + "scorecard": scorecard_mod.subject_scorecard_to_dict(card), + } + + +def _status_entry_for_transition( + *, + new_status: str, + now: Any, + decision: core.Decision, + install_failed: bool, + artifact_kind: str, +) -> dict[str, Any]: + entry: dict[str, Any] = {"status": new_status, "last_action_ts": now.isoformat()} + if ( + artifact_kind == "skill" + and new_status == "promoted" + and not install_failed + and decision.action in {"install", "bump"} + ): + entry["route_policy"] = scorecard_mod.route_policy_marker_for_promotion() + return entry + + def _fingerprint_cohorts_by_artifact( target: Path, records: list[core.OutcomeRecord] ) -> dict[str, core.FingerprintCohorts]: @@ -589,8 +701,16 @@ def load_transitions(target: Path) -> list[core.StatusTransition]: artifact_id = payload.get("artifact_id") new_status = payload.get("new_status") created_at = payload.get("created_at") + route_policy = payload.get("route_policy") if artifact_id and new_status and created_at: - transitions.append(core.StatusTransition(str(artifact_id), str(new_status), str(created_at))) + transitions.append( + core.StatusTransition( + str(artifact_id), + str(new_status), + str(created_at), + route_policy=route_policy if isinstance(route_policy, dict) else None, + ) + ) return transitions @@ -651,33 +771,65 @@ def fork( config: core.ReconcileConfig | None = None, json_output: bool = False, ) -> int: - """Project what the ratchet would decide over the current signal log under a config. + """Project what the ratchet would decide under a hypothetical config (read-only). - A read-only fork: it replays ``records.jsonl`` through the scorer and the - ratchet from a clean baseline under the given config, writing the resulting - per-artifact projection to ``out``. It never reads or writes the live - status.json, so two forks under different configs can be compared with - ``outcome diff`` to see how a rule change would move promotions. - - Uses the same current-fingerprint cohort score as ``reconcile``, so a fork - projection previews the ratchet the live command would actually run. + Skills use the receipt-only scorecard dual gate (#503); cards still replay + ``records.jsonl`` through the legacy scorer. Never reads or writes live + ``status.json``. """ target = target.expanduser().resolve() config = config or core.ReconcileConfig() - cohorts_by_artifact = _fingerprint_cohorts_by_artifact(target, load_records(target)) - scores = {artifact_id: cohorts.current for artifact_id, cohorts in cohorts_by_artifact.items()} - decisions = core.project_statuses(scores, config=config, now=localio.utc_now()) - artifacts = { - artifact_id: { + records = load_records(target) + cohorts_by_artifact = _fingerprint_cohorts_by_artifact(target, records) + scorecards = _scorecards_by_artifact(target) + kinds = _artifact_kinds(records, {}, scorecards) + now = localio.utc_now() + skill_decisions = scorecard_mod.project_scorecard_statuses(scorecards, config=config, now=now) + card_decisions = core.project_statuses( + {artifact_id: cohorts.current for artifact_id, cohorts in cohorts_by_artifact.items()}, + config=config, + now=now, + ) + artifacts: dict[str, dict[str, Any]] = {} + for artifact_id, decision in sorted(skill_decisions.items()): + card = scorecards[artifact_id] + effectiveness = card.dimensions.get("effectiveness", {}) + artifacts[artifact_id] = { "action": decision.action, "new_status": decision.new_status, "reason": decision.reason, - "score": scores[artifact_id].score, - "helped": scores[artifact_id].helped, - "hurt": scores[artifact_id].hurt, + "policy_version": scorecard_mod.SCORECARD_POLICY_VERSION, + "helped": int(effectiveness.get("helped", 0)), + "hurt": int(effectiveness.get("hurt", 0)), + "wilson": float(effectiveness.get("wilson", 0.0)), + "utility_guardrails": card.utility_guardrails, + "dimensions": card.dimensions, + } + for artifact_id in _reconcile_artifact_ids(kinds, {}, scorecards, cohorts_by_artifact): + if kinds.get(artifact_id, "skill") != "skill" or artifact_id in artifacts: + continue + artifacts[artifact_id] = { + "action": "hold", + "new_status": "candidate", + "reason": "withheld: missing scorecard", + "helped": 0, + "hurt": 0, + "wilson": 0.0, + } + for artifact_id, decision in sorted(card_decisions.items()): + if artifact_id in artifacts: + continue + if kinds.get(artifact_id, "skill") != "card": + continue + score_obj = cohorts_by_artifact[artifact_id].current + artifacts[artifact_id] = { + "action": decision.action, + "new_status": decision.new_status, + "reason": decision.reason, + "score": score_obj.score, + "helped": score_obj.helped, + "hurt": score_obj.hurt, } - for artifact_id, decision in sorted(decisions.items()) - } projection = { "version": 1, "target": str(target), @@ -1012,6 +1164,11 @@ def explain(*, target: Path, artifact_id: str, json_output: bool = False) -> int payload["capability_breakdown"] = capability_breakdown if route_breakdown is not None: payload["route_breakdown"] = route_breakdown + from . import scorecard as scorecard_mod + + scorecard_payload = scorecard_mod.explain_payload(target, artifact_id, artifact_kind=kind) + if scorecard_payload is not None: + payload["scorecard"] = scorecard_payload print(json.dumps(payload, indent=2, sort_keys=True)) return 0 print(f"outcome explain: {artifact_id}") @@ -1057,10 +1214,36 @@ def explain(*, target: Path, artifact_id: str, json_output: bool = False) -> int ) if not trail: print("trail: none") - return 0 - for item in trail: - tag = f" [{item['cohort']}]" if cohorts.pinned else "" - print(f"- {item['ts']} {item['source']} {item['signal_value']:+d} ({item['evidence_ref']}){tag}") + else: + for item in trail: + tag = f" [{item['cohort']}]" if cohorts.pinned else "" + print(f"- {item['ts']} {item['source']} {item['signal_value']:+d} ({item['evidence_ref']}){tag}") + from . import scorecard as scorecard_mod + + scorecard_payload = scorecard_mod.explain_payload(target, artifact_id, artifact_kind=kind) + if scorecard_payload is not None: + print("scorecard (receipt-only):") + dimensions = scorecard_payload["dimensions"] + effectiveness = dimensions["effectiveness"] + print( + f"- effectiveness wilson={effectiveness['wilson']:.3f} " + f"helped={effectiveness['helped']} hurt={effectiveness['hurt']} trials={effectiveness['trials']}" + ) + if scorecard_payload.get("ineligible_summary"): + reasons = ", ".join(f"{key}={value}" for key, value in scorecard_payload["ineligible_summary"].items()) + print(f"- ineligible: {reasons}") + receipt_trail = scorecard_payload.get("receipt_trail") or [] + if not receipt_trail: + print("- receipt trail: none") + else: + for item in receipt_trail: + binding = item.get("subject_binding") or {} + subject_id = binding.get("artifact_id", "?") + print( + f"- {item.get('started_at') or '?'} {subject_id} " + f"eligible={item.get('eligible')} reason={item.get('reason')} " + f"effectiveness={item.get('effectiveness')}" + ) return 0 @@ -1225,47 +1408,60 @@ def reconcile( transition, advances the persisted status, and performs the physical skill install/rollback. No human approval is consulted. - Fingerprint-aware: the promote/rollback decision scores the CURRENT-fingerprint - cohort, not the lifetime ledger, so an edited skill must re-earn its promotion - against the text that now ships instead of coasting on signals for text that no - longer exists. Grandfathering keeps this non-disruptive: a never-edited artifact - has no proven-stale records, so ``current`` equals lifetime and the decision is - byte-identical to the pre-fingerprint ratchet. The decision RULES (thresholds, - cooldown, forward-only ratchet) are untouched; only the score fed in narrows. + Skills promote only through the receipt scorecard dual gate (#503); legacy + ``records.jsonl`` signals never advance skill promotion. Cards still use the + legacy scorer. Routing authority is stamped via ``route_policy.policy_version`` + on promotion; demotion drops broad authority even when physical rollback fails. """ target = target.expanduser().resolve() config = config or core.ReconcileConfig() records = load_records(target) cohorts_by_artifact = _fingerprint_cohorts_by_artifact(target, records) + scorecards = _scorecards_by_artifact(target) graph_counts = _graph_delta_counts_by_artifact(records) brief_stats = _brief_hit_stats_by_artifact(records) - kinds: dict[str, str] = {} - for record in records: - kinds.setdefault(record.artifact_id, record.artifact_kind or "skill") status_map = load_status(target) + kinds = _artifact_kinds(records, status_map, scorecards) now = localio.utc_now() - results: list[tuple[core.Decision, core.FingerprintCohorts, str]] = [] - for artifact_id, cohorts in sorted(cohorts_by_artifact.items()): + results: list[_ReconcileItem] = [] + reported: list[_ReconcileItem] = [] + for artifact_id in _reconcile_artifact_ids(kinds, status_map, scorecards, cohorts_by_artifact): entry = status_map.get(artifact_id) or {} prior_status = entry.get("status", "candidate") last_action_ts = localio.parse_iso_datetime(entry.get("last_action_ts")) - decision = core.decide( - cohorts.current, - current_status=prior_status, + artifact_kind = kinds.get(artifact_id, "skill") + cohorts = cohorts_by_artifact.get(artifact_id) + card = scorecards.get(artifact_id) + decision = _decide_reconcile_item( + artifact_id, + artifact_kind, + cohorts=cohorts, + card=card, + prior_status=prior_status, last_action_ts=last_action_ts, now=now, config=config, ) + item = _ReconcileItem( + decision=decision, + prior_status=prior_status, + cohorts=cohorts, + scorecard=card, + ) if decision.action != "hold": - results.append((decision, cohorts, prior_status)) + results.append(item) + elif _surface_hold_decision(decision, prior_status): + reported.append(item) applied: list[str] = [] executions: dict[str, str] = {} effective_status: dict[str, str] = {} if apply and results: - for decision, cohorts, prior_status in results: - score_obj = cohorts.current + for item in results: + decision = item.decision + cohorts = item.cohorts + prior_status = item.prior_status execution = "noop" if decision.action in ("install", "rollback"): if kinds.get(decision.artifact_id, "skill") == "skill": @@ -1273,19 +1469,26 @@ def reconcile( else: execution = "skipped: card execution is v1.1" executions[decision.artifact_id] = execution - # An install that did not physically install must not advance status to - # 'promoted'. The forward-only ratchet never re-emits install for a - # 'promoted' artifact, so a false promotion would permanently hide the - # failure. Keep it a 'candidate' (stamp last_action_ts for cooldown) so a - # later accept + reconcile retries. Cards are exempt: their promotion is - # status-only (physical card execution is v1.1), so a card never "fails". install_failed = ( decision.action == "install" and kinds.get(decision.artifact_id, "skill") == "skill" and execution != "installed" ) - new_status = prior_status if install_failed else decision.new_status + # Rollback demotes routing authority immediately; physical revert is best-effort. + if decision.action == "rollback": + new_status = decision.new_status + else: + new_status = prior_status if install_failed else decision.new_status effective_status[decision.artifact_id] = new_status + if cohorts is not None: + score_payload: dict[str, Any] = dataclasses.asdict(cohorts.current) + elif item.scorecard is not None: + score_payload = { + "dimensions": item.scorecard.dimensions, + "utility_guardrails": item.scorecard.utility_guardrails, + } + else: + score_payload = {} receipt = { "schema_version": receipt_schema.OUTCOME_DECISION_SCHEMA_VERSION, "artifact_id": decision.artifact_id, @@ -1294,37 +1497,61 @@ def reconcile( "new_status": new_status, "decided_status": decision.new_status, "reason": decision.reason, - "score": dataclasses.asdict(score_obj), + "score": score_payload, "execution": execution, "created_at": now.isoformat(), } - receipt.update(_fingerprint_decision_fields(cohorts)) + if cohorts is not None: + receipt.update(_fingerprint_decision_fields(cohorts)) + receipt.update(_scorecard_decision_fields(item.scorecard)) + if ( + kinds.get(decision.artifact_id, "skill") == "skill" + and new_status == "promoted" + and not install_failed + and decision.action in {"install", "bump"} + ): + receipt["route_policy"] = scorecard_mod.route_policy_marker_for_promotion() localio.write_json(_decision_path(target, now, decision.artifact_id), receipt) - status_map[decision.artifact_id] = {"status": new_status, "last_action_ts": now.isoformat()} + status_map[decision.artifact_id] = _status_entry_for_transition( + new_status=new_status, + now=now, + decision=decision, + install_failed=install_failed, + artifact_kind=kinds.get(decision.artifact_id, "skill"), + ) if not install_failed: applied.append(decision.artifact_id) localio.write_json(_status_path(target), {"version": 1, "artifacts": status_map}) decisions_payload = [] - for decision, cohorts, prior_status in results: - item = { + for item in [*results, *reported]: + decision = item.decision + cohorts = item.cohorts + prior_status = item.prior_status + payload_item: dict[str, Any] = { "artifact_id": decision.artifact_id, "action": decision.action, "prior_status": prior_status, "new_status": effective_status.get(decision.artifact_id, decision.new_status), "decided_status": decision.new_status, "reason": decision.reason, - "score": cohorts.current.score, "execution": executions.get(decision.artifact_id, "dry-run"), } - item.update(_fingerprint_decision_fields(cohorts)) + if cohorts is not None: + payload_item["score"] = cohorts.current.score + payload_item.update(_fingerprint_decision_fields(cohorts)) + elif item.scorecard is not None: + payload_item["score"] = item.scorecard.dimensions.get("effectiveness", {}).get("wilson", 0.0) + payload_item["dimensions"] = item.scorecard.dimensions + payload_item["utility_guardrails"] = item.scorecard.utility_guardrails + payload_item.update(_scorecard_decision_fields(item.scorecard)) counts = graph_counts.get(decision.artifact_id) if counts is not None: - item.update(counts) + payload_item.update(counts) stats = brief_stats.get(decision.artifact_id) if stats is not None: - item.update(stats) - decisions_payload.append(item) + payload_item.update(stats) + decisions_payload.append(payload_item) payload = { "target": str(target), "apply": apply, @@ -1336,15 +1563,17 @@ def reconcile( return 0 mode = "apply" if apply else "dry-run" print(f"outcome reconcile: {target} ({mode})") - if not results: + output_items = [*results, *reported] + if not output_items: print("decisions: none") return 0 - for decision, cohorts, prior_status in results: + for item in output_items: + decision = item.decision + cohorts = item.cohorts + prior_status = item.prior_status shown_status = effective_status.get(decision.artifact_id, decision.new_status) - # On --apply, surface the physical execution so the output is never - # byte-identical to a dry-run that did nothing. tail = f" -> {executions[decision.artifact_id]}" if apply and decision.artifact_id in executions else "" - fingerprint_tail = _fingerprint_decision_suffix(cohorts) + fingerprint_tail = _fingerprint_decision_suffix(cohorts) if cohorts is not None else "" graph_tail = _graph_delta_human_suffix(graph_counts.get(decision.artifact_id)) brief_tail = _brief_hit_human_suffix(brief_stats.get(decision.artifact_id)) print( @@ -1439,6 +1668,9 @@ def rank( } graph_counts = _graph_delta_counts_by_artifact(records) brief_stats = _brief_hit_stats_by_artifact(records) + from . import scorecard as scorecard_mod + + scorecards_by_artifact = {card.subject.artifact_id: card for card in scorecard_mod.build_scorecards(target)} def blended(artifact_id: str) -> float: return core.rank_score(confidence=0.0, outcome=pooled_sort_score[artifact_id], keyword=0.0) @@ -1492,6 +1724,43 @@ def sort_key(item: tuple[str, core.FingerprintCohorts]) -> tuple: stats = brief_stats.get(artifact_id) if stats is not None: entry.update(stats) + scorecard_entry = scorecards_by_artifact.get(artifact_id) + if scorecard_entry is not None: + entry["policy_version"] = scorecard_mod.SCORECARD_POLICY_VERSION + entry["dimensions"] = scorecard_entry.dimensions + entry["utility_guardrails"] = scorecard_entry.utility_guardrails + if scorecard_entry.ineligible_summary: + entry["ineligible_summary"] = scorecard_entry.ineligible_summary + ranking_payload.append(entry) + + ranked_ids = {artifact_id for artifact_id, _ in ordered} + for artifact_id, scorecard_entry in sorted(scorecards_by_artifact.items()): + if artifact_id in ranked_ids: + continue + entry = { + "artifact_id": artifact_id, + "score": 0.0, + "rank_score": 0.0, + "helped": 0, + "hurt": 0, + "content_fingerprint": scorecard_entry.subject.content_fingerprint, + "lifetime_score": 0.0, + "lifetime_helped": 0, + "lifetime_hurt": 0, + "stale_records": 0, + "legacy_records": 0, + "capability_fingerprint": current_capability, + "capability_score": 0.0, + "capability_helped": 0, + "capability_hurt": 0, + "off_capability_records": 0, + "capability_legacy_records": 0, + "policy_version": scorecard_mod.SCORECARD_POLICY_VERSION, + "dimensions": scorecard_entry.dimensions, + "utility_guardrails": scorecard_entry.utility_guardrails, + } + if scorecard_entry.ineligible_summary: + entry["ineligible_summary"] = scorecard_entry.ineligible_summary ranking_payload.append(entry) payload = { "target": str(target), @@ -1580,13 +1849,13 @@ def record( def health(target: Path) -> dict: - """Surface whether the verified-learning loop is actually being fed. + """Surface whether the receipt-only verified-learning loop is actually being fed. - The loop is invisible in ``brigade work brief`` otherwise: an adopter cannot - tell that verify runs are piling up while the outcome ledger stays empty - (loop half-fed) or that neither exists yet (loop dormant). + Scorecard eligibility is projected exclusively from verify receipts. Legacy + ``records.jsonl`` rows remain audit-only and are never joined for scoring. """ target = target.expanduser().resolve() + from . import scorecard as scorecard_mod from .work_cmd import helpers as work_helpers records = load_records(target) @@ -1595,34 +1864,85 @@ def health(target: Path) -> dict: verify_run_count = sum(1 for child in runs_root.iterdir() if child.is_dir()) if runs_root.is_dir() else 0 record_count = len(records) promoted_count = sum(1 for entry in load_status(target).values() if entry.get("status") == "promoted") + receipt_audit = scorecard_mod.receipt_scorecard_audit(target) issues: list[dict] = [] - if verify_run_count > 0 and record_count == 0: + if verify_run_count > 0 and receipt_audit["eligible"] == 0: issues.append( { "status": "warn", "name": "outcome_loop_half_fed", "detail": ( - f"{verify_run_count} verify run(s) but 0 outcome record(s); " - "run `brigade outcome capture ` (or `verify run --capture `) after verifying" + f"{verify_run_count} verify run(s) but 0 eligible receipt(s); " + "run verification through a registered verifier manifest so receipts " + "carry verifier-authored subject_binding and check_role" ), } ) - elif verify_run_count == 0 and record_count == 0: + elif verify_run_count == 0 and receipt_audit["total_receipts"] == 0: issues.append( { "status": "warn", "name": "outcome_loop_dormant", - "detail": "no verify runs or outcome records yet; the verified-learning loop is not running", + "detail": "no verify runs yet; the receipt-only verified-learning loop is not running", } ) return { "records_path": str(_records_path(target)), "verify_run_count": verify_run_count, "record_count": record_count, + "legacy_records_audit_only": True, + "legacy_records_note": ( + "Outcome ledger rows in records.jsonl are audit-only; they cannot be backfilled into receipt scorecards." + ), "scored_artifact_count": len(scores), "promoted_count": promoted_count, + "attributed_receipt_count": receipt_audit["attributed"], + "unattributed_receipt_count": receipt_audit["unattributed"], + "eligible_receipt_count": receipt_audit["eligible"], + "ineligible_receipt_count": receipt_audit["ineligible"], + "attributed_ineligible_receipt_count": receipt_audit["attributed_ineligible"], + "ineligibility_rate": receipt_audit["ineligibility_rate"], + "leading_ineligibility_reason": receipt_audit["leading_ineligibility_reason"], + "exploration_bands": receipt_audit["exploration_bands"], + "latest_receipt_window": receipt_audit["latest_receipt_window"], "issue_count": len(issues), "top_issue": issues[0] if issues else None, "issues": issues, } + + +def backfill_scorecard(*, target: Path, json_output: bool = False) -> int: + """Read-only audit of verify receipts for scorecard eligibility (#574).""" + target = target.expanduser().resolve() + if not target.is_dir(): + print(f"error: --target is not a directory: {target}", file=sys.stderr) + return 2 + from . import scorecard as scorecard_mod + + payload = scorecard_mod.backfill_scorecard_payload(target) + if json_output: + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + print(f"outcome backfill scorecard: {target}") + print( + "receipts: " + f"total={payload['total_receipts']} " + f"eligible={payload['eligible']} " + f"unattributed={payload['unattributed']} " + f"ineligible={payload['ineligible']} " + f"ineligibility_rate={payload['ineligibility_rate']}" + ) + bands = payload.get("exploration_bands") if isinstance(payload.get("exploration_bands"), dict) else {} + if bands: + print( + "exploration_bands: " + f"unseen={bands.get('unseen', 0)} " + f"candidate={bands.get('candidate', 0)} " + f"provisional={bands.get('provisional', 0)} " + f"promoted={bands.get('promoted', 0)}" + ) + if payload.get("leading_ineligibility_reason"): + print(f"leading_ineligibility_reason: {payload['leading_ineligibility_reason']}") + print(payload["legacy_records_note"]) + return 0 diff --git a/src/brigade/route_policy.py b/src/brigade/route_policy.py new file mode 100644 index 00000000..0956ea51 --- /dev/null +++ b/src/brigade/route_policy.py @@ -0,0 +1,1009 @@ +"""Route-level skill exploration policy and band classification (#573).""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, Literal + +from . import outcome_cmd, scorecard, verify_manifest + +if TYPE_CHECKING: + from .route_catalog import RouteBrief + from .run_transport import Assignment + +ROUTE_POLICY_VERSION = "route-policy.v1" + +EXPLORATION_ASSIGNMENT_CAP = 2 +EXPLORATION_ASSIGNMENT_PCT = 0.10 +ONE_EXPLORATORY_SKILL_PER_ROUTE = True +EXPLORATION_DECAY_DAYS = 7 +EXPLORATION_HARD_CEILING_DAYS = 30 +EXPLORATION_HARD_CEILING = 5 + +Band = Literal["unseen", "candidate", "provisional", "promoted"] +RouteAuthority = Literal["shadow", "scoped_write", "full"] +ManifestRole = Literal["production", "fixture_eval"] + + +@dataclass(frozen=True) +class TrustedOptInEntry: + manifest: verify_manifest.VerifyManifest + manifest_path: str + card: scorecard.SubjectScorecard | None = None + + +@dataclass(frozen=True) +class SkillAssignment: + artifact_id: str + band: Band + route_authority: RouteAuthority + exploratory: bool + score_inputs: dict[str, Any] + scope_globs: tuple[str, ...] = () + manifest_path: str | None = None + manifest_id: str | None = None + subject_path: str | None = None + verify_manifest_id: str | None = None + reason: str | None = None + + +@dataclass +class RoutePolicyDecision: + route_class: str | None + assignments: list[SkillAssignment] = field(default_factory=list) + eligible_assignment_count_7d: int = 0 + exploratory_assignment_count_7d: int = 0 + quota: int = 0 + accept_reject: list[dict[str, Any]] = field(default_factory=list) + policy_applied: bool = False + decided_at: datetime | None = None + + +def classify_band( + card: scorecard.SubjectScorecard | None, + *, + persisted_status: str | None = None, + policy_marker: str | None = None, +) -> Band: + """Classify exploration band from receipt-only scorecard and scorecard route policy.""" + hurt = 0 + if card is not None: + hurt = int(card.dimensions.get("effectiveness", {}).get("hurt", 0)) + promoted_authority = ( + persisted_status == "promoted" + and policy_marker == scorecard.SCORECARD_POLICY_VERSION + and hurt == 0 + and card is not None + ) + if promoted_authority: + return "promoted" + if card is None: + return "unseen" + effectiveness = card.dimensions.get("effectiveness", {}) + helped = int(effectiveness.get("helped", 0)) + trials = int(effectiveness.get("trials", 0)) + if trials == 0 or helped == 0: + return "unseen" + utility = card.utility_guardrails + required_ids = utility.get("required_check_ids") or [] + passing = int(utility.get("passing_trials", 0)) + required = int(utility.get("required_trials", scorecard._UTILITY_REQUIRED_TRIALS)) + utility_complete = not required_ids or passing >= required + if helped >= 2 and not utility_complete: + return "provisional" + if helped >= 1: + return "candidate" + return "unseen" + + +def route_authority_for_band(band: Band) -> RouteAuthority: + if band == "promoted": + return "full" + if band == "unseen": + return "shadow" + return "scoped_write" + + +def exploration_quota(eligible_assignment_count: int) -> int: + eligible = max(1, eligible_assignment_count) + return min(EXPLORATION_ASSIGNMENT_CAP, max(1, math.floor(EXPLORATION_ASSIGNMENT_PCT * eligible))) + + +def validate_scope_globs(globs: tuple[str, ...]) -> str | None: + """Return a stable rejection reason when scope_globs are unsafe or unusable.""" + return verify_manifest.validate_scope_globs(globs) + + +def _default_runs_dir(target: Path) -> Path: + return target / ".brigade" / "runs" + + +def discover_route_decision_paths(target: Path, *, runs_dir: Path | None = None) -> list[Path]: + root = runs_dir.expanduser().resolve() if runs_dir is not None else _default_runs_dir(target) + if not root.is_dir(): + return [] + return sorted( + child / "route-decision.json" for child in root.iterdir() if (child / "route-decision.json").is_file() + ) + + +def _parse_timestamp(value: str | None) -> datetime | None: + if not isinstance(value, str) or not value: + return None + normalized = value.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _utc_iso(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _decision_timestamp(decision: dict[str, Any], decision_path: Path) -> datetime | None: + for key in ("decided_at", "assigned_at"): + parsed = _parse_timestamp(decision.get(key)) + if parsed is not None: + return parsed + run_path = decision_path.parent / "run.json" + if run_path.is_file(): + try: + run_receipt = json.loads(run_path.read_text()) + except (OSError, json.JSONDecodeError): + run_receipt = None + if isinstance(run_receipt, dict): + for key in ("started_at", "finished_at"): + parsed = _parse_timestamp(run_receipt.get(key)) + if parsed is not None: + return parsed + return None + + +def _load_route_decision(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _route_class_from_decision(decision: dict[str, Any], decision_path: Path) -> str | None: + exploration = decision.get("exploration") + if isinstance(exploration, dict): + route_class = exploration.get("route_class") + if isinstance(route_class, str) and route_class: + return route_class + run_path = decision_path.parent / "run.json" + if not run_path.is_file(): + return None + try: + run_receipt = json.loads(run_path.read_text()) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(run_receipt, dict): + return None + manifest = outcome_cmd.route_manifest(run_receipt, run_path) + return outcome_cmd.route_fingerprint(manifest) + + +def _assignments_from_decision(decision: dict[str, Any]) -> list[dict[str, Any]]: + raw = decision.get("skill_assignments") + if isinstance(raw, list): + return [item for item in raw if isinstance(item, dict)] + return [] + + +def _within_window(timestamp: datetime | None, *, now: datetime, days: int) -> bool: + if timestamp is None: + return False + return timestamp >= now - timedelta(days=days) + + +def assignment_counts( + decision_paths: list[Path], + *, + route_class: str, + now: datetime, +) -> tuple[int, int, dict[str, int]]: + """Return eligible 7d count, exploratory 7d count, and per-skill 30d exploratory counts.""" + eligible_7d = 0 + exploratory_7d = 0 + exploratory_30d_by_skill: dict[str, int] = {} + for path in decision_paths: + decision = _load_route_decision(path) + if decision is None: + continue + decision_route_class = _route_class_from_decision(decision, path) + if decision_route_class != route_class: + continue + timestamp = _decision_timestamp(decision, path) + if not _within_window(timestamp, now=now, days=EXPLORATION_HARD_CEILING_DAYS): + continue + for assignment in _assignments_from_decision(decision): + artifact_id = assignment.get("artifact_id") + if not isinstance(artifact_id, str) or not artifact_id: + continue + exploratory = bool(assignment.get("exploratory")) + if _within_window(timestamp, now=now, days=EXPLORATION_DECAY_DAYS): + eligible_7d += 1 + if exploratory: + exploratory_7d += 1 + if exploratory: + exploratory_30d_by_skill[artifact_id] = exploratory_30d_by_skill.get(artifact_id, 0) + 1 + return eligible_7d, exploratory_7d, exploratory_30d_by_skill + + +def score_inputs_for_card(card: scorecard.SubjectScorecard, band: Band) -> dict[str, Any]: + effectiveness = card.dimensions.get("effectiveness", {}) + return { + "band": band, + "effectiveness": { + "helped": effectiveness.get("helped", 0), + "hurt": effectiveness.get("hurt", 0), + "wilson": effectiveness.get("wilson", 0.0), + "trials": effectiveness.get("trials", 0), + }, + "utility_guardrails": card.utility_guardrails, + } + + +def route_manifest_from_brief(route: RouteBrief) -> dict[str, Any]: + from . import router + + if not route.attached: + return {"followed": False} + path = next((signal for signal in route.signals if signal in router.PATHS), "unknown") + return { + "followed": True, + "path": path, + "size": route.size, + "signals": sorted(route.signals), + } + + +def _manifest_opts_into_route( + manifest: verify_manifest.VerifyManifest, + *, + route_manifest: dict[str, Any], + route_class: str, +) -> bool: + if manifest.route_paths and verify_manifest.validate_route_paths(manifest.route_paths) is not None: + return False + if manifest.route_classes and verify_manifest.validate_route_classes(manifest.route_classes) is not None: + return False + if not manifest.route_paths and not manifest.route_classes: + return False + if manifest.route_paths and route_manifest.get("path") in manifest.route_paths: + return True + if manifest.route_classes and route_class in manifest.route_classes: + return True + return False + + +def _manifest_role(manifest: verify_manifest.VerifyManifest) -> ManifestRole: + return "fixture_eval" if manifest.binding_mode == "fixture_eval" else "production" + + +def _subject_fingerprint_resolves(target: Path, manifest: verify_manifest.VerifyManifest) -> bool: + return verify_manifest.resolve_subject_fingerprint(target, manifest) is not None + + +def _trusted_card_for_manifest( + target: Path, manifest: verify_manifest.VerifyManifest +) -> scorecard.SubjectScorecard | None: + card = scorecard.scorecard_for_artifact( + target, + manifest.artifact_id, + artifact_kind=manifest.artifact_kind, + ) + if card is None: + return None + fingerprint = verify_manifest.resolve_subject_fingerprint(target, manifest) + if fingerprint is None: + return None + normalized = scorecard._normalize_fingerprint(fingerprint) + if normalized is None or card.subject.content_fingerprint != normalized: + return None + return card + + +def _role_ambiguity_reason(role: ManifestRole) -> str: + if role == "production": + return "ambiguous_production_manifest" + return "ambiguous_fixture_eval_manifest" + + +def _entry_from_selected_manifest( + target: Path, + manifest: verify_manifest.VerifyManifest, + manifest_path: str, +) -> TrustedOptInEntry | None: + if _manifest_role(manifest) == "production": + card = _trusted_card_for_manifest(target, manifest) + if card is None: + return None + return TrustedOptInEntry(manifest=manifest, manifest_path=manifest_path, card=card) + if not _subject_fingerprint_resolves(target, manifest): + return None + card = _trusted_card_for_manifest(target, manifest) + return TrustedOptInEntry(manifest=manifest, manifest_path=manifest_path, card=card) + + +def _select_manifest_for_band( + *, + band: Band, + production: tuple[verify_manifest.VerifyManifest, str] | None, + fixture: tuple[verify_manifest.VerifyManifest, str] | None, +) -> tuple[verify_manifest.VerifyManifest, str] | None: + if band == "unseen": + return fixture + return production or fixture + + +def _artifact_id_from_payload(payload: dict[str, Any]) -> str | None: + subject = payload.get("subject") + if not isinstance(subject, dict): + return None + artifact_id = subject.get("artifact_id") + return str(artifact_id) if isinstance(artifact_id, str) and artifact_id else None + + +def _route_opted_raw_payload( + payload: dict[str, Any], + *, + route_manifest: dict[str, Any], + route_class: str, +) -> bool: + route_paths_raw = payload.get("route_paths", []) + route_classes_raw = payload.get("route_classes", []) + if not route_paths_raw and not route_classes_raw: + return False + route_paths = tuple(str(item) for item in route_paths_raw) if isinstance(route_paths_raw, list) else () + route_classes = tuple(str(item) for item in route_classes_raw) if isinstance(route_classes_raw, list) else () + if route_paths and verify_manifest.validate_route_paths(route_paths) is not None: + return False + if route_classes and verify_manifest.validate_route_classes(route_classes) is not None: + return False + if route_paths and route_manifest.get("path") in route_paths: + return True + if route_classes and route_class in route_classes: + return True + return False + + +def discover_trusted_opt_in_manifests( + target: Path, + *, + route_manifest: dict[str, Any], + route_class: str, +) -> tuple[list[TrustedOptInEntry], list[dict[str, Any]]]: + grouped: dict[str, dict[ManifestRole, list[tuple[verify_manifest.VerifyManifest, str]]]] = {} + rejections: list[dict[str, Any]] = [] + for path in verify_manifest._discover_manifest_files(target, tracked_only=True): + payload = verify_manifest._load_manifest_file(path) + if not isinstance(payload, dict): + continue + try: + manifest = verify_manifest.manifest_from_payload(payload, path=path) + except ValueError: + artifact_id = _artifact_id_from_payload(payload) + if artifact_id and _route_opted_raw_payload( + payload, + route_manifest=route_manifest, + route_class=route_class, + ): + scope_globs_raw = payload.get("scope_globs", []) + if isinstance(scope_globs_raw, list): + scope_reason = verify_manifest.validate_scope_globs(scope_globs_raw) + if scope_reason is not None: + rejections.append( + { + "artifact_id": artifact_id, + "accepted": False, + "reason": scope_reason, + } + ) + continue + if not _manifest_opts_into_route(manifest, route_manifest=route_manifest, route_class=route_class): + continue + manifest_path = verify_manifest.manifest_source_path(target, manifest) + if not manifest_path: + continue + role = _manifest_role(manifest) + grouped.setdefault(manifest.artifact_id, {}).setdefault(role, []).append((manifest, manifest_path)) + + entries: list[TrustedOptInEntry] = [] + for artifact_id in sorted(grouped): + roles = grouped[artifact_id] + ambiguous = False + for role, items in roles.items(): + if len(items) > 1: + rejections.append( + { + "artifact_id": artifact_id, + "accepted": False, + "reason": _role_ambiguity_reason(role), + } + ) + ambiguous = True + if ambiguous: + continue + production = roles.get("production", [None])[0] if roles.get("production") else None + fixture = roles.get("fixture_eval", [None])[0] if roles.get("fixture_eval") else None + card = None + if production is not None: + card = _trusted_card_for_manifest(target, production[0]) + elif fixture is not None: + card = _trusted_card_for_manifest(target, fixture[0]) + band = classify_band( + card, + persisted_status=_persisted_status(target, artifact_id), + policy_marker=_persisted_policy_marker(target, artifact_id), + ) + selected = _select_manifest_for_band(band=band, production=production, fixture=fixture) + if selected is None: + continue + entry = _entry_from_selected_manifest(target, selected[0], selected[1]) + if entry is not None: + entries.append(entry) + return sorted(entries, key=lambda item: (item.manifest.artifact_id, item.manifest_path)), rejections + + +def _fixture_eval_manifest_for_skill( + target: Path, + artifact_id: str, +) -> tuple[verify_manifest.VerifyManifest | None, str | None]: + matches: list[tuple[verify_manifest.VerifyManifest, str]] = [] + for path in verify_manifest._discover_manifest_files(target, tracked_only=True): + payload = verify_manifest._load_manifest_file(path) + if not isinstance(payload, dict): + continue + try: + manifest = verify_manifest.manifest_from_payload(payload, path=path) + except ValueError: + continue + if manifest.binding_mode != "fixture_eval": + continue + if manifest.artifact_id != artifact_id: + continue + manifest_path = verify_manifest.manifest_source_path(target, manifest) + if not manifest_path: + return None, "missing_manifest_path" + matches.append((manifest, manifest_path)) + if not matches: + return None, "missing_fixture_eval_manifest" + if len(matches) > 1: + return None, "ambiguous_fixture_eval_manifest" + return matches[0][0], matches[0][1] + + +def _scope_from_manifest(manifest: verify_manifest.VerifyManifest) -> tuple[tuple[str, ...] | None, str | None]: + if not manifest.scope_globs: + return None, "missing_scope_globs" + reason = validate_scope_globs(manifest.scope_globs) + if reason is not None: + return None, reason + return manifest.scope_globs, None + + +def _persisted_status(target: Path, artifact_id: str) -> str | None: + entry = outcome_cmd.load_status(target).get(artifact_id) + if not isinstance(entry, dict): + return None + status = entry.get("status") + return str(status) if isinstance(status, str) and status else None + + +def _persisted_policy_marker(target: Path, artifact_id: str) -> str | None: + entry = outcome_cmd.load_status(target).get(artifact_id) + if not isinstance(entry, dict): + return None + route_policy = entry.get("route_policy") + if not isinstance(route_policy, dict): + return None + marker = route_policy.get("policy_version") + return str(marker) if isinstance(marker, str) and marker else None + + +@dataclass(frozen=True) +class RouteBudget: + token_budget: int | None = None + work_budget: int | None = None + token_spent: int = 0 + work_spent: int = 0 + + def allows_exploratory(self) -> bool: + if self.token_budget is not None and self.token_spent >= self.token_budget: + return False + if self.work_budget is not None and self.work_spent >= self.work_budget: + return False + return True + + +def _exploration_block_reason( + *, + exploratory_7d: int, + quota: int, + budget: RouteBudget | None, +) -> str | None: + if exploratory_7d >= quota: + return "exploration_quota_exhausted" + if budget is not None and not budget.allows_exploratory(): + return "routing_budget_exhausted" + return None + + +def _reject_exploratory_pool( + decision: RoutePolicyDecision, + pool: Sequence[tuple[TrustedOptInEntry, Band]], + *, + reason: str, +) -> None: + for entry, _band in pool: + decision.accept_reject.append({"artifact_id": entry.manifest.artifact_id, "accepted": False, "reason": reason}) + + +def _select_scoped_exploratory( + target: Path, + decision: RoutePolicyDecision, + entry: TrustedOptInEntry, + band: Band, + *, + exploratory_30d: dict[str, int], +) -> SkillAssignment | None: + artifact_id = entry.manifest.artifact_id + if exploratory_30d.get(artifact_id, 0) >= EXPLORATION_HARD_CEILING: + decision.accept_reject.append( + {"artifact_id": artifact_id, "accepted": False, "reason": "exploration_hard_ceiling"} + ) + return None + globs, scope_reason = _scope_from_manifest(entry.manifest) + if globs is None: + decision.accept_reject.append( + {"artifact_id": artifact_id, "accepted": False, "reason": scope_reason or "missing_scope_globs"} + ) + return None + score_inputs = score_inputs_for_card(entry.card, band) if entry.card is not None else {"band": band} + decision.accept_reject.append({"artifact_id": artifact_id, "accepted": True, "reason": "exploratory_selected"}) + return SkillAssignment( + artifact_id=artifact_id, + band=band, + route_authority="scoped_write", + exploratory=True, + score_inputs=score_inputs, + scope_globs=globs, + manifest_path=entry.manifest_path, + manifest_id=entry.manifest.manifest_id, + subject_path=entry.manifest.subject_path, + ) + + +def _select_shadow_exploratory( + target: Path, + decision: RoutePolicyDecision, + entry: TrustedOptInEntry, + band: Band, + *, + exploratory_30d: dict[str, int], +) -> SkillAssignment | None: + artifact_id = entry.manifest.artifact_id + if exploratory_30d.get(artifact_id, 0) >= EXPLORATION_HARD_CEILING: + decision.accept_reject.append( + {"artifact_id": artifact_id, "accepted": False, "reason": "exploration_hard_ceiling"} + ) + return None + if entry.manifest.binding_mode == "fixture_eval": + fixture_manifest = entry.manifest + fixture_path = entry.manifest_path + else: + resolved_manifest, resolved_path = _fixture_eval_manifest_for_skill(target, artifact_id) + if resolved_manifest is None or resolved_path is None: + decision.accept_reject.append( + { + "artifact_id": artifact_id, + "accepted": False, + "reason": resolved_path or "missing_fixture_eval_manifest", + } + ) + return None + fixture_manifest = resolved_manifest + fixture_path = resolved_path + decision.accept_reject.append({"artifact_id": artifact_id, "accepted": True, "reason": "shadow_with_proven_route"}) + return SkillAssignment( + artifact_id=artifact_id, + band=band, + route_authority="shadow", + exploratory=True, + score_inputs={"band": band}, + manifest_path=fixture_path, + manifest_id=fixture_manifest.manifest_id, + subject_path=fixture_manifest.subject_path, + verify_manifest_id=fixture_manifest.manifest_id, + reason="shadow_fixture_eval", + ) + + +DIRECT_WORKER_REJECTS_SHADOW = "direct_worker_rejects_shadow" + + +def shadow_verify_invocation(manifest_id: str) -> str: + return f"brigade work verify run --target . --manifest {manifest_id}" + + +def direct_worker_skill_ids( + decision: RoutePolicyDecision | None, + *, + allow_shadow: bool = False, +) -> tuple[tuple[str, ...], str | None]: + """Select skill ids for a direct-worker assignment from a pre-run policy decision.""" + if decision is None or not decision.policy_applied: + return (), None + exploratory = [item for item in decision.assignments if item.exploratory] + if len(exploratory) > 1: + return (), "direct_worker_supports_at_most_one_exploratory_skill" + if not exploratory: + return (), None + item = exploratory[0] + if item.route_authority == "shadow": + if not allow_shadow: + return (), DIRECT_WORKER_REJECTS_SHADOW + return (item.artifact_id,), None + if item.route_authority == "scoped_write": + return (item.artifact_id,), None + return (), f"direct_worker_cannot_bind_{item.route_authority}" + + +def decide_route_skills( + target: Path, + *, + route_brief: RouteBrief | None = None, + run_receipt: dict[str, Any] | None = None, + decision_paths: list[Path] | None = None, + now: datetime | None = None, + budget: RouteBudget | None = None, + runs_dir: Path | None = None, + allow_shadow: bool = True, +) -> RoutePolicyDecision: + """Select proven and exploratory skill assignments under exploration caps.""" + target = target.expanduser().resolve() + now = now or datetime.now(timezone.utc) + + if route_brief is not None: + if not route_brief.attached: + return RoutePolicyDecision(route_class=None) + route_manifest = route_manifest_from_brief(route_brief) + elif isinstance(run_receipt, dict): + route = run_receipt.get("route") + if not isinstance(route, dict) or not route.get("attached"): + return RoutePolicyDecision(route_class=None) + route_manifest = outcome_cmd.route_manifest(run_receipt, None) + else: + return RoutePolicyDecision(route_class=None) + + route_class = outcome_cmd.route_fingerprint(route_manifest) + if route_class is None: + return RoutePolicyDecision(route_class=None) + + opt_ins, discovery_rejections = discover_trusted_opt_in_manifests( + target, + route_manifest=route_manifest, + route_class=route_class, + ) + if not opt_ins and not discovery_rejections: + return RoutePolicyDecision(route_class=route_class) + + paths = decision_paths if decision_paths is not None else discover_route_decision_paths(target, runs_dir=runs_dir) + eligible_7d, exploratory_7d, exploratory_30d = assignment_counts(paths, route_class=route_class, now=now) + quota = exploration_quota(eligible_7d) + + classified: list[tuple[TrustedOptInEntry, Band]] = [] + for entry in opt_ins: + artifact_id = entry.manifest.artifact_id + band = classify_band( + entry.card, + persisted_status=_persisted_status(target, artifact_id), + policy_marker=_persisted_policy_marker(target, artifact_id), + ) + classified.append((entry, band)) + + proven = sorted( + [(entry, band) for entry, band in classified if band == "promoted"], + key=lambda item: ( + -item[0].card.dimensions.get("effectiveness", {}).get("wilson", 0.0) # type: ignore[union-attr] + if item[0].card is not None + else 0.0, + item[0].manifest.artifact_id, + ), + ) + exploratory_pool = sorted( + [(entry, band) for entry, band in classified if band in {"candidate", "provisional"}], + key=lambda item: ( + -item[0].card.dimensions.get("effectiveness", {}).get("wilson", 0.0) # type: ignore[union-attr] + if item[0].card is not None + else 0.0, + item[0].manifest.artifact_id, + ), + ) + shadow_pool = sorted( + [(entry, band) for entry, band in classified if band == "unseen"], + key=lambda item: item[0].manifest.artifact_id, + ) + + decision = RoutePolicyDecision( + route_class=route_class, + eligible_assignment_count_7d=eligible_7d, + exploratory_assignment_count_7d=exploratory_7d, + quota=quota, + policy_applied=True, + decided_at=now, + accept_reject=list(discovery_rejections), + ) + + for entry, band in proven: + assert entry.card is not None + decision.assignments.append( + SkillAssignment( + artifact_id=entry.manifest.artifact_id, + band=band, + route_authority="full", + exploratory=False, + score_inputs=score_inputs_for_card(entry.card, band), + manifest_path=entry.manifest_path, + manifest_id=entry.manifest.manifest_id, + subject_path=entry.manifest.subject_path, + ) + ) + decision.accept_reject.append( + {"artifact_id": entry.manifest.artifact_id, "accepted": True, "reason": "promoted_priority"} + ) + + selected_exploratory: SkillAssignment | None = None + block_reason = _exploration_block_reason(exploratory_7d=exploratory_7d, quota=quota, budget=budget) + if block_reason is not None: + _reject_exploratory_pool(decision, exploratory_pool, reason=block_reason) + if proven and allow_shadow: + _reject_exploratory_pool(decision, shadow_pool, reason=block_reason) + else: + for entry, band in exploratory_pool: + selected_exploratory = _select_scoped_exploratory( + target, + decision, + entry, + band, + exploratory_30d=exploratory_30d, + ) + if selected_exploratory is not None: + break + if selected_exploratory is None and proven and allow_shadow: + for entry, band in shadow_pool: + selected_exploratory = _select_shadow_exploratory( + target, + decision, + entry, + band, + exploratory_30d=exploratory_30d, + ) + if selected_exploratory is not None: + break + elif not allow_shadow and shadow_pool: + for entry, _band in shadow_pool: + decision.accept_reject.append( + { + "artifact_id": entry.manifest.artifact_id, + "accepted": False, + "reason": DIRECT_WORKER_REJECTS_SHADOW, + } + ) + if selected_exploratory is None: + pending = {entry.manifest.artifact_id for entry, _band in exploratory_pool} - { + entry["artifact_id"] for entry in decision.accept_reject + } + for artifact_id in sorted(pending): + decision.accept_reject.append( + {"artifact_id": artifact_id, "accepted": False, "reason": "one_exploratory_per_route"} + ) + if proven: + pending_shadow = {entry.manifest.artifact_id for entry, _band in shadow_pool} - { + entry["artifact_id"] for entry in decision.accept_reject + } + for artifact_id in sorted(pending_shadow): + decision.accept_reject.append( + {"artifact_id": artifact_id, "accepted": False, "reason": "one_exploratory_per_route"} + ) + elif shadow_pool: + for entry, _band in shadow_pool: + decision.accept_reject.append( + { + "artifact_id": entry.manifest.artifact_id, + "accepted": False, + "reason": "unseen_cannot_be_sole_provider", + } + ) + + if selected_exploratory is not None: + decision.assignments.append(selected_exploratory) + + return decision + + +def exploratory_skill_ids(decision: RoutePolicyDecision | None) -> frozenset[str]: + if decision is None or not decision.policy_applied: + return frozenset() + return frozenset(item.artifact_id for item in decision.assignments if item.exploratory) + + +def validate_plan_skill_bindings(assignments: list[Assignment], decision: RoutePolicyDecision | None) -> None: + """Ensure planner bindings match the pre-run exploratory skill decision.""" + expected = exploratory_skill_ids(decision) + if not expected: + if any(assignment.selected_skill_ids for assignment in assignments): + raise ValueError("selected_skill_ids must be omitted when no exploratory skill was selected") + return + if len(expected) != 1: + raise ValueError("route policy selected more than one exploratory skill") + skill_id = next(iter(expected)) + bound = [assignment for assignment in assignments if skill_id in assignment.selected_skill_ids] + if len(bound) != 1: + raise ValueError(f"exploratory skill {skill_id!r} must be bound to exactly one assignment") + exploratory = next(item for item in decision.assignments if item.exploratory) # type: ignore[union-attr] + assignment = bound[0] + if exploratory.route_authority == "shadow": + if "verify" not in assignment.covers: + raise ValueError(f"shadow skill {skill_id!r} must bind to a verify-covering assignment") + manifest_id = exploratory.verify_manifest_id or "" + invocation = shadow_verify_invocation(manifest_id) + if invocation not in assignment.task: + raise ValueError( + f"shadow skill {skill_id!r} assignment must include the exact verify invocation {invocation!r}" + ) + elif exploratory.route_authority == "scoped_write": + if assignment.covers == ("verify",) or (len(assignment.covers) == 1 and assignment.covers[0] == "verify"): + raise ValueError(f"scoped-write skill {skill_id!r} must bind to a production assignment") + extra = sorted( + {skill for assignment in assignments for skill in assignment.selected_skill_ids if skill not in expected} + ) + if extra: + raise ValueError(f"unknown selected_skill_ids: {', '.join(extra)}") + + +def _assignment_to_dict(assignment: SkillAssignment) -> dict[str, Any]: + payload: dict[str, Any] = { + "artifact_id": assignment.artifact_id, + "band": assignment.band, + "route_authority": assignment.route_authority, + "exploratory": assignment.exploratory, + "score_inputs": assignment.score_inputs, + } + if assignment.scope_globs: + payload["scope_globs"] = list(assignment.scope_globs) + if assignment.manifest_path: + payload["manifest_path"] = assignment.manifest_path + if assignment.manifest_id: + payload["manifest_id"] = assignment.manifest_id + if assignment.subject_path: + payload["subject_path"] = assignment.subject_path + if assignment.verify_manifest_id: + payload["verify_manifest_id"] = assignment.verify_manifest_id + if assignment.reason: + payload["reason"] = assignment.reason + return payload + + +def route_policy_extensions_from_decision(decision: RoutePolicyDecision) -> dict[str, Any]: + if not decision.policy_applied or decision.decided_at is None: + return {} + return { + "decided_at": _utc_iso(decision.decided_at), + "policy_version": ROUTE_POLICY_VERSION, + "score_inputs": {item.artifact_id: item.score_inputs for item in decision.assignments}, + "skill_assignments": [_assignment_to_dict(item) for item in decision.assignments], + "exploration": { + "route_class": decision.route_class, + "eligible_assignment_count_7d": decision.eligible_assignment_count_7d, + "exploratory_assignment_count_7d": decision.exploratory_assignment_count_7d, + "quota": decision.quota, + "accept_reject": decision.accept_reject, + }, + } + + +def route_policy_payload( + target: Path, + run_receipt: dict[str, Any], + *, + now: datetime | None = None, + budget: RouteBudget | None = None, + runs_dir: Path | None = None, + exclude_decision_path: Path | None = None, + policy_decision: RoutePolicyDecision | None = None, +) -> dict[str, Any]: + """Build additive route-decision fields for scorecard skill exploration.""" + if policy_decision is not None: + return route_policy_extensions_from_decision(policy_decision) + paths = discover_route_decision_paths(target, runs_dir=runs_dir) + if exclude_decision_path is not None: + resolved = exclude_decision_path.resolve() + paths = [path for path in paths if path.resolve() != resolved] + decision = decide_route_skills( + target, + run_receipt=run_receipt, + decision_paths=paths, + now=now, + budget=budget, + runs_dir=runs_dir, + ) + return route_policy_extensions_from_decision(decision) + + +def planner_skill_policy_section(decision: RoutePolicyDecision | None) -> str: + if decision is None or not decision.policy_applied or not decision.assignments: + return "" + lines = ["## Skill route policy (deterministic, pre-run)"] + exploratory_ids = exploratory_skill_ids(decision) + if exploratory_ids: + lines.append( + "- Bind each accepted exploratory skill to exactly one assignment via " + '`"selected_skill_ids": [""]`. ' + "Each exploratory skill id must appear on exactly one assignment." + ) + for assignment in decision.assignments: + if assignment.route_authority == "full": + lines.append( + f"- promoted skill `{assignment.artifact_id}` (manifest `{assignment.manifest_path}`): " + "full route authority" + ) + elif assignment.route_authority == "scoped_write": + globs = ", ".join(assignment.scope_globs) + lines.append( + f"- exploratory skill `{assignment.artifact_id}` ({assignment.band}, manifest " + f"`{assignment.manifest_path}`): scoped write only within globs [{globs}]; " + "bind it to one production assignment with selected_skill_ids" + ) + elif assignment.route_authority == "shadow": + invocation = shadow_verify_invocation(assignment.verify_manifest_id or "") + lines.append( + f"- shadow skill `{assignment.artifact_id}` (unseen, manifest `{assignment.manifest_path}`): " + f"read-only fixture evaluation via verify manifest `{assignment.verify_manifest_id}`; " + "bind it to one verify-covering assignment whose task includes the exact invocation " + f"`{invocation}`" + ) + return "\n".join(lines) + "\n" + + +def worker_skill_policy_constraint( + decision: RoutePolicyDecision | None, + assignment: Assignment | None = None, +) -> str: + if decision is None or not decision.policy_applied or assignment is None: + return "" + selected = set(assignment.selected_skill_ids) + if not selected: + return "" + lines: list[str] = [] + for item in decision.assignments: + if item.artifact_id not in selected: + continue + if item.route_authority == "scoped_write" and item.scope_globs: + globs = ", ".join(item.scope_globs) + lines.append( + "Scoped-write constraint: edits are limited to verifier-manifest globs " + f"[{globs}] from `{item.manifest_path}`." + ) + elif item.route_authority == "shadow": + invocation = shadow_verify_invocation(item.verify_manifest_id or "") + lines.append( + "Shadow skill constraint: read-only fixture evaluation only via " + f"`{invocation}`; do not perform production edits." + ) + if not lines: + return "" + return "\n" + "\n".join(lines) diff --git a/src/brigade/route_receipts.py b/src/brigade/route_receipts.py index a8cbb214..0ccde677 100644 --- a/src/brigade/route_receipts.py +++ b/src/brigade/route_receipts.py @@ -12,13 +12,42 @@ ROUTE_DECISION_SCHEMA_VERSION = "brigade.route-decision.v1" +_POLICY_EXTENSION_KEYS = ( + "decided_at", + "policy_version", + "score_inputs", + "skill_assignments", + "exploration", +) -class RouteDecisionArtifact(TypedDict): + +class RouteDecisionArtifact(TypedDict, total=False): schema_version: str chosen_route: list[str] | None confidence: str | None template_version: str | None admissible_seats: list[str] + decided_at: str + policy_version: str + score_inputs: dict[str, Any] + skill_assignments: list[dict[str, Any]] + exploration: dict[str, Any] + + +def _apply_policy_extensions( + payload: RouteDecisionArtifact, + extensions: dict[str, Any], +) -> None: + if "decided_at" in extensions: + payload["decided_at"] = extensions["decided_at"] + if "policy_version" in extensions: + payload["policy_version"] = extensions["policy_version"] + if "score_inputs" in extensions: + payload["score_inputs"] = extensions["score_inputs"] + if "skill_assignments" in extensions: + payload["skill_assignments"] = extensions["skill_assignments"] + if "exploration" in extensions: + payload["exploration"] = extensions["exploration"] def admissible_seats(roster: Roster) -> list[str]: @@ -29,6 +58,11 @@ def admissible_seats(roster: Roster) -> list[str]: def route_decision_payload( run_receipt: dict[str, Any], roster: Roster, + *, + target: Path | None = None, + runs_dir: Path | None = None, + exclude_decision_path: Path | None = None, + policy_extensions: dict[str, Any] | None = None, ) -> RouteDecisionArtifact: route = run_receipt.get("route") if isinstance(route, dict) and route.get("attached"): @@ -47,32 +81,97 @@ def route_decision_payload( chosen_route = None confidence = None template_version = None - return { + payload: RouteDecisionArtifact = { "schema_version": ROUTE_DECISION_SCHEMA_VERSION, "chosen_route": chosen_route, "confidence": confidence, "template_version": template_version, "admissible_seats": admissible_seats(roster), } + if policy_extensions: + _apply_policy_extensions(payload, policy_extensions) + elif target is not None: + from .route_policy import route_policy_payload + + budget = _route_budget_from_run_receipt(run_receipt) + extensions = route_policy_payload( + target, + run_receipt, + budget=budget, + runs_dir=runs_dir, + exclude_decision_path=exclude_decision_path, + ) + _apply_policy_extensions(payload, extensions) + return payload + + +def _route_budget_from_run_receipt(run_receipt: dict[str, Any]): + route = run_receipt.get("route") + if not isinstance(route, dict): + return None + raw = route.get("routing_budget") + if not isinstance(raw, dict): + return None + token_budget = raw.get("token_budget") + work_budget = raw.get("work_budget") + token_spent = raw.get("token_spent", 0) + work_spent = raw.get("work_spent", 0) + from .route_policy import RouteBudget + + return RouteBudget( + token_budget=int(token_budget) if isinstance(token_budget, int) else None, + work_budget=int(work_budget) if isinstance(work_budget, int) else None, + token_spent=int(token_spent) if isinstance(token_spent, int) else 0, + work_spent=int(work_spent) if isinstance(work_spent, int) else 0, + ) + + +def _existing_policy_extensions(decision_path: Path) -> dict[str, Any] | None: + existing = localio.read_json_dict(decision_path) + if not isinstance(existing, dict) or "policy_version" not in existing: + return None + extensions: dict[str, Any] = {} + for key in _POLICY_EXTENSION_KEYS: + if key in existing: + extensions[key] = existing[key] + return extensions or None def write_route_decision( output_dir: Path, roster: Roster, + *, + target: Path | None = None, + runs_dir: Path | None = None, + policy_extensions: dict[str, Any] | None = None, ) -> Path: run_receipt = localio.read_json_dict(output_dir / "run.json") if run_receipt is None: raise ValueError(f"missing or invalid run receipt: {output_dir / 'run.json'}") - payload = route_decision_payload(run_receipt, roster) + decision_path = output_dir / "route-decision.json" + preserved = policy_extensions or _existing_policy_extensions(decision_path) + payload = route_decision_payload( + run_receipt, + roster, + target=target, + runs_dir=runs_dir, + exclude_decision_path=decision_path, + policy_extensions=preserved, + ) ordered: dict[str, object] = {} - for key, value in ( - ("schema_version", payload["schema_version"]), - ("chosen_route", payload["chosen_route"]), - ("confidence", payload["confidence"]), - ("template_version", payload["template_version"]), - ("admissible_seats", payload["admissible_seats"]), + for key in ( + "schema_version", + "chosen_route", + "confidence", + "template_version", + "admissible_seats", + "decided_at", + "policy_version", + "score_inputs", + "skill_assignments", + "exploration", ): - ordered[key] = value - path = output_dir / "route-decision.json" - localio.write_text_atomic(path, json.dumps(ordered, indent=2) + "\n") - return path + if key in payload: + ordered[key] = payload[key] + localio.write_text_atomic(decision_path, json.dumps(ordered, indent=2) + "\n") + return decision_path diff --git a/src/brigade/run_receipts.py b/src/brigade/run_receipts.py index 8013bc66..c5b3fc97 100644 --- a/src/brigade/run_receipts.py +++ b/src/brigade/run_receipts.py @@ -21,6 +21,8 @@ def assignment_payload(assignments: list[Assignment]) -> list[dict[str, object]] } if assignment.covers: entry["covers"] = list(assignment.covers) + if assignment.selected_skill_ids: + entry["selected_skill_ids"] = list(assignment.selected_skill_ids) payload.append(entry) return payload diff --git a/src/brigade/run_transport.py b/src/brigade/run_transport.py index 92477743..788fd08e 100644 --- a/src/brigade/run_transport.py +++ b/src/brigade/run_transport.py @@ -28,6 +28,7 @@ class Assignment: task: str stage: int = 1 covers: tuple[str, ...] = () + selected_skill_ids: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/src/brigade/scorecard.py b/src/brigade/scorecard.py new file mode 100644 index 00000000..dca40ff0 --- /dev/null +++ b/src/brigade/scorecard.py @@ -0,0 +1,863 @@ +"""Pure receipt-only skill scorecard projection from verify receipts (#572).""" + +from __future__ import annotations + +import datetime as dt +import json +import statistics +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from . import outcome as outcome_core, outcome_cmd, verify_trial + +SCORECARD_POLICY_VERSION = "scorecard.v1" +EFFECTIVE_WILSON_MIN = 0.15 +_UTILITY_REQUIRED_TRIALS = 2 +LATEST_RECEIPT_WINDOW = 50 +"""Stable latest-receipt window for outcome checkup ineligibility warnings (#574). + +Receipts are ordered by ``started_at`` descending, then ``run_id``, then +``receipt_path`` (all lexicographic, descending). The first 50 audits in that +order form the window regardless of eligibility. +""" + + +@dataclass(frozen=True) +class SubjectRef: + artifact_kind: str + artifact_id: str + content_fingerprint: str + + +@dataclass +class ReceiptAudit: + receipt_path: str + run_id: str | None + started_at: str | None + eligible: bool + reason: str + attributed: bool + subject_binding: dict[str, Any] | None + effectiveness: int | None + verifier_cost_s: float | None + reused: bool + evidence_unit_key: tuple[Any, ...] | None + + +@dataclass +class SubjectScorecard: + subject: SubjectRef + dimensions: dict[str, Any] + utility_guardrails: dict[str, Any] + ineligible_summary: dict[str, int] = field(default_factory=dict) + receipt_trail: list[ReceiptAudit] = field(default_factory=list) + + +def _verify_runs_root(target: Path) -> Path: + return target / ".brigade" / "work" / "verify-runs" + + +def discover_verify_receipt_paths(target: Path) -> list[Path]: + root = _verify_runs_root(target) + if not root.is_dir(): + return [] + return sorted(root.glob("*/receipt.json")) + + +def _load_receipt(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _binding_dict(receipt: dict[str, Any]) -> dict[str, Any] | None: + binding = receipt.get("subject_binding") + return binding if isinstance(binding, dict) else None + + +def _normalized_command(command: dict[str, Any]) -> str | None: + argv = command.get("argv") + if isinstance(argv, list) and argv: + return " ".join(str(item) for item in argv) + value = command.get("command") + if isinstance(value, str) and value: + return value + return None + + +def _effectiveness_plan_tuple(receipt: dict[str, Any]) -> tuple[tuple[str, str], ...] | None: + pairs: list[tuple[str, str]] = [] + for command in _effectiveness_commands(receipt): + check_id = command.get("check_id") + if not isinstance(check_id, str) or not check_id: + return None + normalized = _normalized_command(command) + if normalized is None: + return None + pairs.append((check_id, normalized)) + if not pairs: + return None + return tuple(pairs) + + +def _patch_unit_key(binding: dict[str, Any]) -> tuple[Any, ...] | None: + patch_binding = binding.get("patch_binding") + if not isinstance(patch_binding, dict): + return None + parts: list[Any] = [] + for key in ("baseline_commit", "tree_fingerprint", "changes_patch_sha256", "subject_path"): + value = patch_binding.get(key) + if not isinstance(value, str) or not value: + return None + parts.append(value) + return tuple(parts) + + +def _fixture_unit_key(binding: dict[str, Any]) -> tuple[Any, ...] | None: + fixture_binding = binding.get("fixture_binding") + if not isinstance(fixture_binding, dict): + return None + parts: list[Any] = [] + for key in ("manifest_id", "case_id", "check_id"): + value = fixture_binding.get(key) + if not isinstance(value, str) or not value: + return None + parts.append(value) + return tuple(parts) + + +def _binding_unit_key(binding: dict[str, Any]) -> tuple[Any, ...] | None: + mode = binding.get("binding_mode") + if mode == "patch_backed": + return _patch_unit_key(binding) + if mode == "fixture_eval": + return _fixture_unit_key(binding) + return None + + +def utility_evidence_unit_key(binding: dict[str, Any], check_id: str) -> tuple[Any, ...] | None: + fingerprint = binding.get("content_fingerprint") + if not isinstance(fingerprint, str) or not fingerprint: + return None + unit = _binding_unit_key(binding) + if unit is None: + return None + return (fingerprint, unit, check_id) + + +def utility_trial_unit_key(binding: dict[str, Any]) -> tuple[Any, ...] | None: + fingerprint = binding.get("content_fingerprint") + if not isinstance(fingerprint, str) or not fingerprint: + return None + unit = _binding_unit_key(binding) + if unit is None: + return None + return (fingerprint, unit) + + +def evidence_unit_key(receipt: dict[str, Any], binding: dict[str, Any]) -> tuple[Any, ...] | None: + fingerprint = binding.get("content_fingerprint") + mode = binding.get("binding_mode") + if not isinstance(fingerprint, str) or not fingerprint: + return None + if not isinstance(mode, str) or not mode: + return None + unit = _binding_unit_key(binding) + if unit is None: + return None + plan = _effectiveness_plan_tuple(receipt) + if plan is None: + return None + return (fingerprint, mode, unit, plan) + + +def _effectiveness_commands(receipt: dict[str, Any]) -> list[dict[str, Any]]: + commands = receipt.get("commands") + if not isinstance(commands, list): + return [] + return [ + command for command in commands if isinstance(command, dict) and command.get("check_role") == "effectiveness" + ] + + +def effectiveness_outcome(receipt: dict[str, Any]) -> int | None: + """Return +1, -1, or None when no effectiveness commands are present.""" + effectiveness = _effectiveness_commands(receipt) + if not effectiveness: + return None + for command in effectiveness: + if command.get("status") != "completed" or command.get("exit_code") != 0: + return -1 + return 1 + + +def _utility_commands(receipt: dict[str, Any]) -> list[dict[str, Any]]: + commands = receipt.get("commands") + if not isinstance(commands, list): + return [] + return [ + command + for command in commands + if isinstance(command, dict) and command.get("check_role") == "utility_guardrail" + ] + + +def utility_check_outcomes(receipt: dict[str, Any]) -> dict[str, bool]: + outcomes: dict[str, bool] = {} + for command in _utility_commands(receipt): + check_id = command.get("check_id") + if not isinstance(check_id, str) or not check_id: + continue + outcomes[check_id] = command.get("status") == "completed" and command.get("exit_code") == 0 + return outcomes + + +def _receipt_duration_seconds(receipt: dict[str, Any]) -> float | None: + duration = receipt.get("duration_seconds") + if isinstance(duration, (int, float)) and duration >= 0: + return float(duration) + return None + + +def _command_durations_seconds(receipt: dict[str, Any]) -> list[float]: + commands = receipt.get("commands") + if not isinstance(commands, list): + return [] + durations: list[float] = [] + for command in commands: + if not isinstance(command, dict): + continue + duration = command.get("duration_seconds") + if isinstance(duration, (int, float)) and duration >= 0: + durations.append(float(duration)) + return durations + + +def _percentile(values: list[float], percentile: float) -> float: + if not values: + return 0.0 + if len(values) == 1: + return values[0] + ordered = sorted(values) + rank = (len(ordered) - 1) * (percentile / 100.0) + low = int(rank) + high = min(low + 1, len(ordered) - 1) + weight = rank - low + return ordered[low] * (1.0 - weight) + ordered[high] * weight + + +def _current_fingerprint(target: Path, artifact_id: str, artifact_kind: str) -> str | None: + return outcome_cmd.artifact_fingerprint(target, artifact_id, artifact_kind) + + +def _normalize_fingerprint(value: str | None) -> str | None: + if not isinstance(value, str) or not value: + return None + return value.removeprefix("sha256:") + + +def _fingerprints_match(left: str | None, right: str | None) -> bool: + normalized_left = _normalize_fingerprint(left) + normalized_right = _normalize_fingerprint(right) + if normalized_left is None or normalized_right is None: + return False + return normalized_left == normalized_right + + +def _subject_ref(binding: dict[str, Any]) -> SubjectRef | None: + artifact_kind = binding.get("artifact_kind") + artifact_id = binding.get("artifact_id") + fingerprint = binding.get("content_fingerprint") + if not isinstance(artifact_kind, str) or not artifact_kind: + return None + if not isinstance(artifact_id, str) or not artifact_id: + return None + if not isinstance(fingerprint, str) or not fingerprint: + return None + return SubjectRef(artifact_kind=artifact_kind, artifact_id=artifact_id, content_fingerprint=fingerprint) + + +def _invalid_receipt_audit(receipt_path: Path) -> ReceiptAudit: + """Audit a discovered receipt path whose JSON is missing or not an object.""" + return ReceiptAudit( + receipt_path=str(receipt_path), + run_id=None, + started_at=None, + eligible=False, + reason="invalid_receipt_json", + attributed=False, + subject_binding=None, + effectiveness=None, + verifier_cost_s=None, + reused=False, + evidence_unit_key=None, + ) + + +def _audit_receipt( + *, + receipt_path: Path, + receipt: dict[str, Any], + target: Path, +) -> ReceiptAudit: + binding = _binding_dict(receipt) + projection = verify_trial.project_trial( + receipt, + target=target, + patch_path=receipt_path.parent / "changes.patch", + ) + reused = isinstance(receipt.get("reused_from"), str) and bool(receipt.get("reused_from")) + unit_key = evidence_unit_key(receipt, binding) if binding is not None else None + return ReceiptAudit( + receipt_path=str(receipt_path), + run_id=str(receipt.get("run_id")) if receipt.get("run_id") is not None else None, + started_at=str(receipt.get("started_at")) if receipt.get("started_at") is not None else None, + eligible=projection.eligible, + reason=projection.reason, + attributed=projection.attributed, + subject_binding=binding, + effectiveness=effectiveness_outcome(receipt), + verifier_cost_s=_receipt_duration_seconds(receipt), + reused=reused, + evidence_unit_key=unit_key, + ) + + +def _dedupe_audits(audits: list[ReceiptAudit]) -> list[ReceiptAudit]: + """Keep the first non-reused receipt per evidence unit.""" + seen: set[tuple[Any, ...]] = set() + deduped: list[ReceiptAudit] = [] + for audit in sorted(audits, key=lambda item: (item.started_at or "", item.receipt_path)): + if audit.reused: + continue + if audit.evidence_unit_key is None: + deduped.append(audit) + continue + if audit.evidence_unit_key in seen: + continue + seen.add(audit.evidence_unit_key) + deduped.append(audit) + return deduped + + +def _retry_stability_metrics(audits: list[ReceiptAudit]) -> dict[str, Any]: + cohorts: dict[tuple[Any, ...], list[ReceiptAudit]] = defaultdict(list) + for audit in audits: + if not audit.eligible or audit.reused or audit.evidence_unit_key is None: + continue + cohorts[audit.evidence_unit_key].append(audit) + + consistent = 0 + sequences = 0 + first_pass = 0 + fail_to_pass = 0 + flips = 0 + for cohort_audits in cohorts.values(): + ordered = sorted(cohort_audits, key=lambda item: (item.started_at or "", item.receipt_path)) + outcomes = [audit.effectiveness for audit in ordered if audit.effectiveness in {1, -1}] + if not outcomes: + continue + sequences += 1 + if outcomes[0] == 1: + first_pass += 1 + previous = outcomes[0] + for outcome in outcomes[1:]: + if previous == -1 and outcome == 1: + fail_to_pass += 1 + if outcome != previous: + flips += 1 + previous = outcome + if len(set(outcomes)) == 1: + consistent += 1 + rate = consistent / sequences if sequences else 0.0 + return { + "consistent": consistent, + "sequences": sequences, + "first_pass_yield": first_pass / sequences if sequences else 0.0, + "fail_to_pass": fail_to_pass, + "flips": flips, + "rate": rate, + } + + +def _utility_guardrail_summary( + *, + audits: list[ReceiptAudit], + receipts_by_path: dict[str, dict[str, Any]], + required_check_ids: list[str], +) -> dict[str, Any]: + if not required_check_ids: + return { + "required_check_ids": [], + "passing_trials": 0, + "required_trials": 0, + "per_check": {}, + } + passing_units = 0 + seen_trial_units: set[tuple[Any, ...]] = set() + seen_check_units: dict[str, set[tuple[Any, ...]]] = {check_id: set() for check_id in required_check_ids} + per_check = {check_id: {"passing_units": 0, "failing_units": 0} for check_id in required_check_ids} + for audit in audits: + if not audit.eligible or audit.reused: + continue + binding = audit.subject_binding + if not isinstance(binding, dict): + continue + trial_unit = utility_trial_unit_key(binding) + receipt = receipts_by_path.get(audit.receipt_path) + if receipt is None: + continue + outcomes = utility_check_outcomes(receipt) + for check_id in required_check_ids: + unit_key = utility_evidence_unit_key(binding, check_id) + if unit_key is None or unit_key in seen_check_units[check_id]: + continue + seen_check_units[check_id].add(unit_key) + if outcomes.get(check_id) is True: + per_check[check_id]["passing_units"] += 1 + else: + per_check[check_id]["failing_units"] += 1 + if trial_unit is None or trial_unit in seen_trial_units: + continue + seen_trial_units.add(trial_unit) + if all(outcomes.get(check_id) is True for check_id in required_check_ids): + passing_units += 1 + return { + "required_check_ids": required_check_ids, + "passing_trials": passing_units, + "required_trials": _UTILITY_REQUIRED_TRIALS, + "per_check": per_check, + } + + +def _build_subject_scorecard( + *, + subject: SubjectRef, + audits: list[ReceiptAudit], + receipts_by_path: dict[str, dict[str, Any]], +) -> SubjectScorecard: + ineligible_summary = Counter(audit.reason for audit in audits if audit.attributed and not audit.eligible) + eligible_audits = [audit for audit in audits if audit.eligible and not audit.reused] + deduped = _dedupe_audits(eligible_audits) + stability_audits = sorted(eligible_audits, key=lambda item: (item.started_at or "", item.receipt_path)) + + helped = sum(1 for audit in deduped if audit.effectiveness == 1) + hurt = sum(1 for audit in deduped if audit.effectiveness == -1) + trials = helped + hurt + wilson = outcome_core.wilson_lower_bound(helped, trials) + + receipt_durations = [audit.verifier_cost_s for audit in deduped if audit.verifier_cost_s is not None] + command_durations: list[float] = [] + for audit in deduped: + receipt = receipts_by_path.get(audit.receipt_path) + if receipt is None: + continue + command_durations.extend(_command_durations_seconds(receipt)) + verifier_cost = { + "median_s": statistics.median(receipt_durations) if receipt_durations else 0.0, + "p95_s": _percentile(receipt_durations, 95.0), + "trials": len(receipt_durations), + "receipt_median_s": statistics.median(receipt_durations) if receipt_durations else 0.0, + "receipt_p95_s": _percentile(receipt_durations, 95.0), + "receipt_samples": len(receipt_durations), + "command_median_s": statistics.median(command_durations) if command_durations else 0.0, + "command_p95_s": _percentile(command_durations, 95.0), + "command_samples": len(command_durations), + } + + integrity_eligible = len(eligible_audits) + integrity_audit = len([audit for audit in audits if audit.attributed]) + evidence_integrity = { + "eligible": integrity_eligible, + "audit": integrity_audit, + "ratio": integrity_eligible / integrity_audit if integrity_audit else 0.0, + } + + required_ids: list[str] = [] + for audit in audits: + receipt = receipts_by_path.get(audit.receipt_path) + if receipt is None: + continue + required = receipt.get("required_utility_check_ids") + if isinstance(required, list): + for check_id in required: + if isinstance(check_id, str) and check_id and check_id not in required_ids: + required_ids.append(check_id) + + return SubjectScorecard( + subject=subject, + dimensions={ + "effectiveness": { + "helped": helped, + "hurt": hurt, + "wilson": wilson, + "trials": trials, + }, + "verifier_cost": verifier_cost, + "retry_stability": _retry_stability_metrics(stability_audits), + "evidence_integrity": evidence_integrity, + }, + utility_guardrails=_utility_guardrail_summary( + audits=deduped, + receipts_by_path=receipts_by_path, + required_check_ids=required_ids, + ), + ineligible_summary=dict(sorted(ineligible_summary.items())), + receipt_trail=sorted(audits, key=lambda item: (item.started_at or "", item.receipt_path)), + ) + + +def build_scorecards(target: Path) -> list[SubjectScorecard]: + target = target.expanduser().resolve() + grouped: dict[tuple[str, str, str], list[ReceiptAudit]] = defaultdict(list) + receipts_by_path: dict[str, dict[str, Any]] = {} + + for receipt_path in discover_verify_receipt_paths(target): + receipt = _load_receipt(receipt_path) + if receipt is None: + continue + receipts_by_path[str(receipt_path)] = receipt + binding = _binding_dict(receipt) + if binding is None: + continue + subject = _subject_ref(binding) + if subject is None: + continue + current_fp = _current_fingerprint(target, subject.artifact_id, subject.artifact_kind) + cohort_fp = current_fp or subject.content_fingerprint + if not _fingerprints_match(subject.content_fingerprint, cohort_fp): + continue + audit = _audit_receipt(receipt_path=receipt_path, receipt=receipt, target=target) + normalized_fp = _normalize_fingerprint(cohort_fp) or cohort_fp + grouped[(subject.artifact_kind, subject.artifact_id, normalized_fp)].append(audit) + + cards: list[SubjectScorecard] = [] + for (artifact_kind, artifact_id, fingerprint), audits in sorted(grouped.items()): + subject = SubjectRef( + artifact_kind=artifact_kind, + artifact_id=artifact_id, + content_fingerprint=fingerprint, + ) + cards.append( + _build_subject_scorecard( + subject=subject, + audits=audits, + receipts_by_path=receipts_by_path, + ) + ) + return cards + + +def scorecard_for_artifact( + target: Path, + artifact_id: str, + *, + artifact_kind: str | None = None, +) -> SubjectScorecard | None: + for card in build_scorecards(target): + if card.subject.artifact_id != artifact_id: + continue + if artifact_kind is not None and card.subject.artifact_kind != artifact_kind: + continue + return card + return None + + +def _receipt_audit_to_dict(audit: ReceiptAudit) -> dict[str, Any]: + return { + "receipt_path": audit.receipt_path, + "run_id": audit.run_id, + "started_at": audit.started_at, + "eligible": audit.eligible, + "reason": audit.reason, + "attributed": audit.attributed, + "subject_binding": audit.subject_binding, + "effectiveness": audit.effectiveness, + "verifier_cost_s": audit.verifier_cost_s, + "reused": audit.reused, + "evidence_unit_key": list(audit.evidence_unit_key) if audit.evidence_unit_key is not None else None, + } + + +def subject_scorecard_to_dict(card: SubjectScorecard) -> dict[str, Any]: + return { + "schema_version": 1, + "policy_version": SCORECARD_POLICY_VERSION, + "subject": { + "artifact_kind": card.subject.artifact_kind, + "artifact_id": card.subject.artifact_id, + "content_fingerprint": card.subject.content_fingerprint, + }, + "dimensions": card.dimensions, + "utility_guardrails": card.utility_guardrails, + "ineligible_summary": card.ineligible_summary, + "receipt_trail": [_receipt_audit_to_dict(audit) for audit in card.receipt_trail], + } + + +def explain_payload(target: Path, artifact_id: str, *, artifact_kind: str | None = None) -> dict[str, Any] | None: + card = scorecard_for_artifact(target, artifact_id, artifact_kind=artifact_kind) + if card is None: + return None + return subject_scorecard_to_dict(card) + + +def _receipt_audit_sort_key(audit: ReceiptAudit) -> tuple[str, str, str]: + return (audit.started_at or "", audit.run_id or "", audit.receipt_path) + + +def audit_all_verify_receipts(target: Path) -> list[ReceiptAudit]: + """Project every verify receipt for scorecard eligibility without ledger joins.""" + target = target.expanduser().resolve() + audits: list[ReceiptAudit] = [] + for receipt_path in discover_verify_receipt_paths(target): + receipt = _load_receipt(receipt_path) + if receipt is None: + audits.append(_invalid_receipt_audit(receipt_path)) + continue + audits.append(_audit_receipt(receipt_path=receipt_path, receipt=receipt, target=target)) + return audits + + +def latest_receipt_audits( + audits: list[ReceiptAudit], + *, + limit: int = LATEST_RECEIPT_WINDOW, +) -> list[ReceiptAudit]: + ordered = sorted(audits, key=_receipt_audit_sort_key, reverse=True) + return ordered[:limit] + + +def exploration_band_counts(target: Path) -> dict[str, int]: + """Count attributed scorecard subjects by exploration band.""" + target = target.expanduser().resolve() + status_map = outcome_cmd.load_status(target) + counts = {"unseen": 0, "candidate": 0, "provisional": 0, "promoted": 0} + for card in build_scorecards(target): + entry = status_map.get(card.subject.artifact_id, {}) + status = entry.get("status") if isinstance(entry, dict) else None + route_policy = entry.get("route_policy") if isinstance(entry, dict) else None + policy_marker = route_policy.get("policy_version") if isinstance(route_policy, dict) else None + band = classify_band(card, persisted_status=status, policy_marker=policy_marker) + counts[band] = counts.get(band, 0) + 1 + return counts + + +def receipt_scorecard_audit(target: Path) -> dict[str, Any]: + """Aggregate receipt-only scorecard eligibility for operator surfaces (#574).""" + target = target.expanduser().resolve() + audits = audit_all_verify_receipts(target) + total = len(audits) + eligible = sum(1 for audit in audits if audit.eligible) + unattributed = sum(1 for audit in audits if not audit.attributed) + ineligible = total - eligible + attributed_ineligible = sum(1 for audit in audits if audit.attributed and not audit.eligible) + attributed = total - unattributed + ineligible_by_reason = Counter(audit.reason for audit in audits if not audit.eligible) + ineligibility_rate = round(ineligible / total, 4) if total else 0.0 + leading_reason = None + if ineligible_by_reason: + leading_reason = max(ineligible_by_reason.items(), key=lambda item: (item[1], item[0]))[0] + + latest = latest_receipt_audits(audits) + latest_ineligible = sum(1 for audit in latest if not audit.eligible) + latest_ineligibility_rate = round(latest_ineligible / len(latest), 4) if latest else 0.0 + latest_reasons = Counter(audit.reason for audit in latest if not audit.eligible) + latest_leading_reason = None + if latest_reasons: + latest_leading_reason = max(latest_reasons.items(), key=lambda item: (item[1], item[0]))[0] + + return { + "total_receipts": total, + "eligible": eligible, + "unattributed": unattributed, + "ineligible": ineligible, + "attributed_ineligible": attributed_ineligible, + "attributed": attributed, + "ineligibility_rate": ineligibility_rate, + "ineligible_by_reason": dict(sorted(ineligible_by_reason.items())), + "leading_ineligibility_reason": leading_reason, + "exploration_bands": exploration_band_counts(target), + "latest_receipt_window": { + "limit": LATEST_RECEIPT_WINDOW, + "count": len(latest), + "eligible": sum(1 for audit in latest if audit.eligible), + "ineligible": latest_ineligible, + "ineligibility_rate": latest_ineligibility_rate, + "leading_ineligibility_reason": latest_leading_reason, + "sort": "started_at,run_id,receipt_path descending", + }, + } + + +def backfill_scorecard_payload(target: Path) -> dict[str, Any]: + """Read-only verify-receipt scorecard backfill audit (#574). + + Legacy ``records.jsonl`` outcome rows are audit-only and are never joined + or backfilled into scorecards. + """ + target = target.expanduser().resolve() + audit = receipt_scorecard_audit(target) + return { + "target": str(target), + "policy_version": SCORECARD_POLICY_VERSION, + "legacy_records_audit_only": True, + "legacy_records_note": ( + "Outcome ledger rows in records.jsonl are audit-only; they cannot be backfilled into receipt scorecards." + ), + **audit, + } + + +def classify_band( + card: SubjectScorecard | None, + *, + persisted_status: str | None = None, + policy_marker: str | None = None, +) -> str: + """Exploration band classifier used by route policy (#573).""" + from .route_policy import classify_band as _classify_band + + return _classify_band(card, persisted_status=persisted_status, policy_marker=policy_marker) + + +def route_policy_marker_for_promotion() -> dict[str, str]: + """Persisted on promoted status so route_policy grants full authority (#503).""" + return { + "policy_version": SCORECARD_POLICY_VERSION, + "route_authority": "full", + } + + +def utility_gate_reason( + card: SubjectScorecard, + *, + min_passing_units: int = _UTILITY_REQUIRED_TRIALS, +) -> str | None: + """Return a withhold reason when utility guardrails fail, else None.""" + utility = card.utility_guardrails + required_ids = utility.get("required_check_ids") or [] + if not required_ids: + return None + per_check = utility.get("per_check") or {} + for check_id in required_ids: + check_stats = per_check.get(check_id) if isinstance(per_check, dict) else None + if not isinstance(check_stats, dict): + return f"withheld: utility_guardrail {check_id}" + failing = int(check_stats.get("failing_units", 0)) + if failing > 0: + return f"withheld: utility_guardrail {check_id}" + passing = int(check_stats.get("passing_units", 0)) + if passing < min_passing_units: + return f"withheld: utility_guardrail {check_id}" + return None + + +def effectiveness_gate_reason( + card: SubjectScorecard, + *, + config: outcome_core.ReconcileConfig, +) -> str | None: + """Return a withhold reason when effectiveness fails promotion, else None.""" + effectiveness = card.dimensions.get("effectiveness", {}) + helped = int(effectiveness.get("helped", 0)) + hurt = int(effectiveness.get("hurt", 0)) + trials = helped + hurt + wilson = outcome_core.wilson_lower_bound(helped, trials, config.z) + if hurt > 0: + return "withheld: verified regression present" + if helped < config.install_min_helped: + return "insufficient verified evidence" + wilson_min = config.effective_wilson_min + if wilson < wilson_min: + return f"withheld: effectiveness wilson below {wilson_min:g}" + return None + + +def dual_gate_passes( + card: SubjectScorecard, + *, + config: outcome_core.ReconcileConfig, +) -> tuple[bool, str]: + """Evaluate effectiveness AND utility promotion criteria (#503).""" + effectiveness_reason = effectiveness_gate_reason(card, config=config) + if effectiveness_reason is not None: + return False, effectiveness_reason + utility_reason = utility_gate_reason(card, min_passing_units=config.utility_min_passing_units) + if utility_reason is not None: + return False, utility_reason + return True, "verified helped, no regressions" + + +def decide_scorecard( + card: SubjectScorecard | None, + *, + artifact_id: str, + current_status: str, + last_action_ts: dt.datetime | None, + now: dt.datetime, + config: outcome_core.ReconcileConfig, +) -> outcome_core.Decision: + """Decide promote/hold/rollback from a receipt-only scorecard (#503).""" + if card is None: + if current_status == "promoted": + return outcome_core.Decision( + artifact_id, + "rollback", + "demoted", + "withheld: missing scorecard", + ) + return outcome_core.Decision(artifact_id, "hold", current_status, "withheld: missing scorecard") + + effectiveness = card.dimensions.get("effectiveness", {}) + helped = int(effectiveness.get("helped", 0)) + hurt = int(effectiveness.get("hurt", 0)) + + # Any trusted current-cohort hurt on a promoted skill demotes immediately, + # before cooldown and regardless of revert_min_hurt. + if current_status == "promoted" and hurt > 0: + return outcome_core.Decision(artifact_id, "rollback", "demoted", "verified regression measured") + + if last_action_ts is not None and (now - last_action_ts).total_seconds() < config.cooldown_seconds: + return outcome_core.Decision(artifact_id, "hold", current_status, "cooldown active") + + if current_status == "candidate": + if hurt > 0: + return outcome_core.Decision(artifact_id, "hold", "candidate", "withheld: verified regression present") + passes, reason = dual_gate_passes(card, config=config) + if passes: + return outcome_core.Decision(artifact_id, "install", "promoted", reason) + return outcome_core.Decision(artifact_id, "hold", "candidate", reason) + + if current_status == "promoted": + if helped >= config.bump_min_helped: + passes, reason = dual_gate_passes(card, config=config) + if passes: + return outcome_core.Decision(artifact_id, "bump", "promoted", "sustained verified helped") + return outcome_core.Decision(artifact_id, "hold", "promoted", "no change") + + return outcome_core.Decision(artifact_id, "hold", current_status, "terminal status") + + +def project_scorecard_statuses( + scorecards: dict[str, SubjectScorecard], + *, + config: outcome_core.ReconcileConfig, + now: dt.datetime, +) -> dict[str, outcome_core.Decision]: + """Fork primitive: project scorecard ratchet from a clean candidate baseline.""" + return { + artifact_id: decide_scorecard( + card, + artifact_id=artifact_id, + current_status="candidate", + last_action_ts=None, + now=now, + config=config, + ) + for artifact_id, card in scorecards.items() + } diff --git a/src/brigade/verify_manifest.py b/src/brigade/verify_manifest.py new file mode 100644 index 00000000..bb418b7f --- /dev/null +++ b/src/brigade/verify_manifest.py @@ -0,0 +1,420 @@ +"""Tracked verifier and fixture manifest contract for scoreable verify runs.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from . import localio, outcome_cmd, runguard +from .router import PATHS + +VERIFY_MANIFEST_SCHEMA = "brigade.verify_manifest.v1" +VERIFY_MANIFEST_SCHEMA_VERSION = 1 + +CHECK_ROLES = frozenset({"effectiveness", "utility_guardrail"}) +BINDING_MODES = frozenset({"patch_backed", "fixture_eval"}) +PATCH_SOURCES = frozenset({"worktree", "generated"}) +_WHOLE_REPO_SCOPE_GLOBS = frozenset({"**", "**/*", "*", "/*"}) +_ROUTE_CLASS_FINGERPRINT = re.compile(r"^[a-f0-9]{64}$") + +_BUILTIN_MANIFESTS_DIR = Path(__file__).resolve().parent / "templates" / "verify" / "manifests" +_WORKSPACE_MANIFESTS_REL = Path("verify") / "manifests" + + +@dataclass(frozen=True) +class VerifyCheckSpec: + check_id: str + check_role: str + command: str | list[str] + obligation_id: str | None = None + + +@dataclass(frozen=True) +class VerifyManifest: + manifest_id: str + binding_mode: str + artifact_kind: str + artifact_id: str + verifier_id: str + checks: tuple[VerifyCheckSpec, ...] + required_utility_check_ids: tuple[str, ...] = () + scope_globs: tuple[str, ...] = () + route_paths: tuple[str, ...] = () + route_classes: tuple[str, ...] = () + subject_path: str | None = None + content_fingerprint: str | None = None + fixture_manifest_id: str | None = None + fixture_case_id: str | None = None + fixture_check_id: str | None = None + patch_source: str = "worktree" + path: Path | None = None + + @property + def planned_commands(self) -> list[str | list[str]]: + return [check.command for check in self.checks] + + +def validate_route_paths(route_paths: tuple[str, ...] | list[str]) -> str | None: + """Return a stable rejection reason when route_paths are outside router.PATHS.""" + for item in route_paths: + normalized = str(item).strip() + if not normalized: + return "invalid_route_path" + if normalized not in PATHS: + return "invalid_route_path" + return None + + +def validate_route_classes(route_classes: tuple[str, ...] | list[str]) -> str | None: + """Return a stable rejection reason when route_classes are not fingerprint-shaped.""" + for item in route_classes: + normalized = str(item).strip() + if not normalized or _ROUTE_CLASS_FINGERPRINT.fullmatch(normalized) is None: + return "invalid_route_class" + return None + + +def validate_scope_globs(globs: tuple[str, ...] | list[str]) -> str | None: + """Return a stable rejection reason when scope_globs are unsafe or unusable.""" + if not globs: + return "missing_scope_globs" + for item in globs: + normalized = str(item).strip() + if not normalized: + return "invalid_scope_glob" + if normalized.startswith("/") or normalized.startswith("\\"): + return "invalid_scope_glob" + parts = normalized.replace("\\", "/").split("/") + if any(part == ".." for part in parts): + return "invalid_scope_glob" + if normalized in _WHOLE_REPO_SCOPE_GLOBS: + return "invalid_scope_glob" + return None + + +def _workspace_manifests_root(target: Path) -> Path: + return target / _WORKSPACE_MANIFESTS_REL + + +def _is_builtin_manifest(path: Path) -> bool: + try: + path.resolve().relative_to(_BUILTIN_MANIFESTS_DIR.resolve()) + return True + except ValueError: + return False + + +def _is_git_tracked(target: Path, path: Path) -> bool: + try: + relative = path.resolve().relative_to(target.resolve()) + except ValueError: + return False + result = runguard._git(target, "ls-files", "--error-unmatch", "--", relative.as_posix()) + return result.code == 0 + + +def _load_manifest_file(path: Path) -> dict[str, Any] | None: + payload = localio.read_json_dict(path) + return payload + + +def _parse_check(raw: object, *, index: int) -> tuple[VerifyCheckSpec | None, str | None]: + if not isinstance(raw, dict): + return None, f"checks[{index}] must be an object" + check_id = raw.get("check_id") + check_role = raw.get("check_role") + command = raw.get("command") + if not isinstance(check_id, str) or not check_id.strip(): + return None, f"checks[{index}].check_id is required" + if not isinstance(check_role, str) or check_role not in CHECK_ROLES: + return None, f"checks[{index}].check_role must be one of {sorted(CHECK_ROLES)}" + if isinstance(command, str): + if not command.strip(): + return None, f"checks[{index}].command is required" + elif isinstance(command, list): + if not command or not all(isinstance(item, str) for item in command): + return None, f"checks[{index}].command must be a non-empty string array" + else: + return None, f"checks[{index}].command is required" + obligation_id = raw.get("obligation_id") + if obligation_id is not None and (not isinstance(obligation_id, str) or not obligation_id.strip()): + return None, f"checks[{index}].obligation_id must be a non-empty string when set" + return ( + VerifyCheckSpec( + check_id=check_id, + check_role=check_role, + command=command, + obligation_id=obligation_id if isinstance(obligation_id, str) else None, + ), + None, + ) + + +def validate_manifest_payload(payload: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if payload.get("schema") != VERIFY_MANIFEST_SCHEMA: + errors.append(f"schema must be {VERIFY_MANIFEST_SCHEMA!r}") + if payload.get("schema_version") != VERIFY_MANIFEST_SCHEMA_VERSION: + errors.append(f"schema_version must be {VERIFY_MANIFEST_SCHEMA_VERSION}") + manifest_id = payload.get("manifest_id") + if not isinstance(manifest_id, str) or not manifest_id.strip(): + errors.append("manifest_id is required") + binding_mode = payload.get("binding_mode") + if not isinstance(binding_mode, str) or binding_mode not in BINDING_MODES: + errors.append(f"binding_mode must be one of {sorted(BINDING_MODES)}") + verifier_id = payload.get("verifier_id") + if not isinstance(verifier_id, str) or not verifier_id.strip(): + errors.append("verifier_id is required") + subject = payload.get("subject") + if not isinstance(subject, dict): + errors.append("subject object is required") + subject = {} + artifact_kind = subject.get("artifact_kind") + artifact_id = subject.get("artifact_id") + if not isinstance(artifact_kind, str) or artifact_kind not in {"skill", "card"}: + errors.append("subject.artifact_kind must be 'skill' or 'card'") + if not isinstance(artifact_id, str) or not artifact_id.strip(): + errors.append("subject.artifact_id is required") + subject_path = subject.get("subject_path") + if binding_mode == "patch_backed": + if not isinstance(subject_path, str) or not subject_path.strip(): + errors.append("subject.subject_path is required for patch_backed manifests") + content_fingerprint = subject.get("content_fingerprint") + if content_fingerprint is not None and ( + not isinstance(content_fingerprint, str) or not content_fingerprint.strip() + ): + errors.append("subject.content_fingerprint must be a non-empty string when set") + patch_source = payload.get("patch_source", "worktree") + if patch_source not in PATCH_SOURCES: + errors.append(f"patch_source must be one of {sorted(PATCH_SOURCES)}") + checks = payload.get("checks") + parsed_checks: list[VerifyCheckSpec] = [] + if not isinstance(checks, list) or not checks: + errors.append("checks must be a non-empty array") + else: + for index, item in enumerate(checks): + parsed, error = _parse_check(item, index=index) + if error: + errors.append(error) + elif parsed is not None: + parsed_checks.append(parsed) + required_utility = payload.get("required_utility_check_ids", []) + if required_utility is None: + errors.append("required_utility_check_ids must be an array") + required_utility = [] + elif not isinstance(required_utility, list): + errors.append("required_utility_check_ids must be an array") + required_utility = [] + else: + utility_check_ids = {check.check_id for check in parsed_checks if check.check_role == "utility_guardrail"} + for index, check_id in enumerate(required_utility): + if not isinstance(check_id, str) or not check_id.strip(): + errors.append(f"required_utility_check_ids[{index}] must be a non-empty string") + continue + if check_id not in utility_check_ids: + errors.append(f"required_utility_check_ids[{index}] must name a utility_guardrail check in checks") + scope_globs = payload.get("scope_globs") + if scope_globs is not None: + if not isinstance(scope_globs, list): + errors.append("scope_globs must be an array when set") + else: + for index, item in enumerate(scope_globs): + if not isinstance(item, str) or not item.strip(): + errors.append(f"scope_globs[{index}] must be a non-empty string") + elif validate_scope_globs([item]) == "invalid_scope_glob": + errors.append(f"scope_globs[{index}] must be a relative, non-traversing glob") + route_paths = payload.get("route_paths") + if route_paths is not None: + if not isinstance(route_paths, list): + errors.append("route_paths must be an array when set") + else: + for index, item in enumerate(route_paths): + if not isinstance(item, str) or not item.strip(): + errors.append(f"route_paths[{index}] must be a non-empty string") + elif item.strip() not in PATHS: + errors.append(f"route_paths[{index}] must be one of: {', '.join(PATHS)}") + route_classes = payload.get("route_classes") + if route_classes is not None: + if not isinstance(route_classes, list): + errors.append("route_classes must be an array when set") + else: + for index, item in enumerate(route_classes): + if not isinstance(item, str) or not item.strip(): + errors.append(f"route_classes[{index}] must be a non-empty string") + elif _ROUTE_CLASS_FINGERPRINT.fullmatch(item.strip()) is None: + errors.append(f"route_classes[{index}] must be a 64-character lowercase hex fingerprint") + if binding_mode == "fixture_eval": + fixture = payload.get("fixture") + if not isinstance(fixture, dict): + errors.append("fixture object is required for fixture_eval manifests") + else: + for key in ("manifest_id", "case_id", "check_id"): + value = fixture.get(key) + if not isinstance(value, str) or not value.strip(): + errors.append(f"fixture.{key} is required for fixture_eval manifests") + return errors + + +def manifest_from_payload(payload: dict[str, Any], *, path: Path | None = None) -> VerifyManifest: + errors = validate_manifest_payload(payload) + if errors: + raise ValueError("; ".join(errors)) + subject = payload["subject"] + checks: list[VerifyCheckSpec] = [] + for index, item in enumerate(payload["checks"]): + parsed, error = _parse_check(item, index=index) + if error or parsed is None: + raise ValueError(error or f"checks[{index}] is invalid") + checks.append(parsed) + fixture = payload.get("fixture") if payload.get("binding_mode") == "fixture_eval" else None + required_utility_raw = payload.get("required_utility_check_ids", []) + required_utility_check_ids = ( + tuple(str(item) for item in required_utility_raw) if isinstance(required_utility_raw, list) else () + ) + scope_globs_raw = payload.get("scope_globs", []) + scope_globs = tuple(str(item) for item in scope_globs_raw) if isinstance(scope_globs_raw, list) else () + route_paths_raw = payload.get("route_paths", []) + route_paths = tuple(str(item) for item in route_paths_raw) if isinstance(route_paths_raw, list) else () + route_classes_raw = payload.get("route_classes", []) + route_classes = tuple(str(item) for item in route_classes_raw) if isinstance(route_classes_raw, list) else () + return VerifyManifest( + manifest_id=str(payload["manifest_id"]), + binding_mode=str(payload["binding_mode"]), + artifact_kind=str(subject["artifact_kind"]), + artifact_id=str(subject["artifact_id"]), + verifier_id=str(payload["verifier_id"]), + checks=tuple(checks), + required_utility_check_ids=required_utility_check_ids, + scope_globs=scope_globs, + route_paths=route_paths, + route_classes=route_classes, + subject_path=subject.get("subject_path") if isinstance(subject.get("subject_path"), str) else None, + content_fingerprint=subject.get("content_fingerprint") + if isinstance(subject.get("content_fingerprint"), str) + else None, + fixture_manifest_id=fixture.get("manifest_id") if isinstance(fixture, dict) else None, + fixture_case_id=fixture.get("case_id") if isinstance(fixture, dict) else None, + fixture_check_id=fixture.get("check_id") if isinstance(fixture, dict) else None, + patch_source=str(payload.get("patch_source") or "worktree"), + path=path, + ) + + +def _discover_builtin_manifest_files() -> list[Path]: + if _BUILTIN_MANIFESTS_DIR.is_dir(): + return sorted(_BUILTIN_MANIFESTS_DIR.glob("*.json")) + return [] + + +def _discover_workspace_manifest_files(target: Path, *, tracked_only: bool = False) -> list[Path]: + workspace_root = _workspace_manifests_root(target) + if not workspace_root.is_dir(): + return [] + paths = sorted(workspace_root.glob("*.json")) + if tracked_only: + paths = [path for path in paths if _is_git_tracked(target, path)] + return paths + + +def _discover_manifest_files(target: Path, *, tracked_only: bool = False) -> list[Path]: + return _discover_builtin_manifest_files() + _discover_workspace_manifest_files( + target, + tracked_only=tracked_only, + ) + + +def registered_manifest_ids(target: Path) -> list[str]: + ids: list[str] = [] + for path in _discover_manifest_files(target, tracked_only=True): + payload = _load_manifest_file(path) + if not isinstance(payload, dict): + continue + manifest_id = payload.get("manifest_id") + if isinstance(manifest_id, str) and manifest_id: + ids.append(manifest_id) + return sorted(set(ids)) + + +def resolve_manifest(target: Path, manifest_id: str) -> tuple[VerifyManifest | None, str | None]: + target = target.expanduser().resolve() + matches: list[VerifyManifest] = [] + untracked_workspace_match = False + for path in _discover_manifest_files(target): + payload = _load_manifest_file(path) + if not isinstance(payload, dict): + continue + if str(payload.get("manifest_id") or "") != manifest_id: + continue + if not _is_builtin_manifest(path) and not _is_git_tracked(target, path): + untracked_workspace_match = True + continue + try: + matches.append(manifest_from_payload(payload, path=path)) + except ValueError as exc: + return None, str(exc) + if not matches: + if untracked_workspace_match: + return None, f"verify manifest not tracked: {manifest_id}" + return None, f"verify manifest not found: {manifest_id}" + if len(matches) > 1: + return None, f"verify manifest id is ambiguous: {manifest_id}" + return matches[0], None + + +def resolve_subject_fingerprint(target: Path, manifest: VerifyManifest) -> str | None: + if manifest.content_fingerprint: + return manifest.content_fingerprint + fingerprint = outcome_cmd.artifact_fingerprint(target, manifest.artifact_id, manifest.artifact_kind) + if fingerprint is not None: + return fingerprint + if manifest.subject_path: + from . import verify_trial + + return verify_trial.subject_hash_for_path(target, manifest.subject_path) + return None + + +def manifest_source_path(target: Path, manifest: VerifyManifest) -> str | None: + if manifest.path is None: + return None + if _is_builtin_manifest(manifest.path): + try: + return manifest.path.resolve().relative_to(_BUILTIN_MANIFESTS_DIR.resolve()).as_posix() + except ValueError: + return manifest.path.as_posix() + try: + return manifest.path.resolve().relative_to(target.resolve()).as_posix() + except ValueError: + return manifest.path.as_posix() + + +def manifest_payload_sha256(manifest: VerifyManifest) -> str | None: + if manifest.path is None or not manifest.path.is_file(): + return None + return f"sha256:{localio.file_sha256(manifest.path)}" + + +def build_manifest_binding(target: Path, manifest: VerifyManifest) -> dict[str, Any]: + payload_sha256 = manifest_payload_sha256(manifest) + if payload_sha256 is None: + raise ValueError(f"verify manifest payload is unreadable: {manifest.manifest_id}") + binding: dict[str, Any] = { + "manifest_id": manifest.manifest_id, + "payload_sha256": payload_sha256, + } + source_path = manifest_source_path(target, manifest) + if source_path: + binding["source_path"] = source_path + return binding + + +def write_workspace_manifest(target: Path, payload: dict[str, Any]) -> Path: + """Write a workspace-local manifest after validation (tests and operators).""" + manifest = manifest_from_payload(payload) + root = _workspace_manifests_root(target) + root.mkdir(parents=True, exist_ok=True) + path = root / f"{manifest.manifest_id}.json" + localio.write_json(path, payload) + return path diff --git a/src/brigade/verify_trial.py b/src/brigade/verify_trial.py new file mode 100644 index 00000000..7b16d5a4 --- /dev/null +++ b/src/brigade/verify_trial.py @@ -0,0 +1,662 @@ +"""Fail-closed eligibility projection for scoreable verify receipts.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shlex +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from . import localio, verify_manifest + +VERIFY_INFRASTRUCTURE_FAILURE_CLASSES = frozenset( + { + "infrastructure", + } +) + +VERIFY_SKILL_FAILURE_CLASSES = frozenset( + { + "verification", + } +) + +KNOWN_VERIFY_FAILURE_CLASSES = VERIFY_INFRASTRUCTURE_FAILURE_CLASSES | VERIFY_SKILL_FAILURE_CLASSES + + +@dataclass(frozen=True) +class TrialProjection: + eligible: bool + reason: str + attributed: bool + infrastructure_excluded: bool = False + + +def classify_verify_failure(status: object, exit_code: object) -> tuple[str, str]: + """Map a verify command outcome to the #474-style failure taxonomy.""" + normalized_status = str(status or "") + if normalized_status == "timed_out": + return "infrastructure", "timeout" + if normalized_status == "interrupted": + return "infrastructure", "interrupted" + if normalized_status == "rejected": + return "infrastructure", "rejected" + if normalized_status == "failed": + if exit_code == 127: + return "infrastructure", "command_not_found" + return "verification", "nonzero_exit" + if normalized_status == "completed" and exit_code not in (0, None): + return "verification", "nonzero_exit" + return "infrastructure", "unknown" + + +def stamp_verify_command_failure_taxonomy(command: dict[str, Any]) -> None: + status = command.get("status") + exit_code = command.get("exit_code") + if status == "completed" and exit_code == 0: + command.pop("failure_class", None) + command.pop("failure_kind", None) + return + failure_class, failure_kind = classify_verify_failure(status, exit_code) + command["failure_class"] = failure_class + command["failure_kind"] = failure_kind + + +def stamp_verify_receipt_failure_taxonomy(receipt: dict[str, Any]) -> None: + status = receipt.get("status") + if status in {"completed", "running"}: + receipt.pop("failure_class", None) + receipt.pop("failure_kind", None) + return + if status == "canceled": + receipt["failure_class"] = "infrastructure" + receipt["failure_kind"] = "canceled" + return + commands = receipt.get("commands") + if isinstance(commands, list): + for command in commands: + if isinstance(command, dict): + stamp_verify_command_failure_taxonomy(command) + if status == "failed": + receipt["failure_class"] = "verification" + receipt["failure_kind"] = "receipt_failed" + return + if status == "rejected": + receipt["failure_class"] = "infrastructure" + receipt["failure_kind"] = "rejected" + + +def _sha256_prefixed(data: bytes) -> str: + return f"sha256:{hashlib.sha256(data).hexdigest()}" + + +def _patch_delta_bytes_for_path(patch_bytes: bytes, subject_path: str) -> bytes | None: + if not patch_bytes.strip(): + return None + text = patch_bytes.decode("utf-8", errors="replace") + if not text.strip(): + return None + chunks: list[str] = [] + active = False + for line in text.splitlines(keepends=True): + if line.startswith("diff --git "): + active = False + parts = line.split() + if len(parts) >= 4: + for token in (parts[2], parts[3]): + candidate = token[2:] if token.startswith(("a/", "b/")) else token + if candidate == subject_path or candidate.endswith(f"/{subject_path}"): + active = True + break + if active: + chunks.append(line) + continue + if active: + chunks.append(line) + if not chunks: + return None + return "".join(chunks).encode("utf-8") + + +def owned_delta_sha256(patch_bytes: bytes, subject_path: str) -> str | None: + delta = _patch_delta_bytes_for_path(patch_bytes, subject_path) + if delta is None: + return None + return hashlib.sha256(delta).hexdigest() + + +def _normalize_repo_path(path: str) -> str: + return path.replace("\\", "/").lstrip("./") + + +def subject_path_dirty_in_git_status(subject_path: str, dirty_files: object) -> bool: + if not isinstance(dirty_files, list): + return False + normalized = _normalize_repo_path(subject_path) + for line in dirty_files: + if not isinstance(line, str) or len(line) < 3: + continue + path_part = line[3:].strip() + if " -> " in path_part: + path_part = path_part.split(" -> ", 1)[1] + candidate = _normalize_repo_path(path_part) + if candidate == normalized or candidate.endswith(f"/{normalized}"): + return True + return False + + +def start_git_dirty_files_fingerprint(dirty_files: object) -> str | None: + if not isinstance(dirty_files, list): + return None + lines = sorted(str(line) for line in dirty_files) + return hashlib.sha256("\n".join(lines).encode("utf-8")).hexdigest() + + +def load_work_session_payload(target: Path, session_id: str) -> dict[str, Any] | None: + session_json = target / ".brigade" / "work" / session_id / "session.json" + try: + payload = json.loads(session_json.read_text()) + except (OSError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def build_producer_binding( + *, + session_id: str | None, + session_payload: dict[str, Any] | None, + subject_path: str, + owned_delta: str | None, +) -> dict[str, Any]: + binding: dict[str, Any] = { + "work_session_id": session_id, + "owned_delta_sha256": owned_delta, + } + if session_payload is None or not session_id: + binding["subject_clean_at_start"] = False + return binding + start = session_payload.get("start") + if not isinstance(start, dict): + binding["subject_clean_at_start"] = False + return binding + git = start.get("git") + if not isinstance(git, dict): + binding["subject_clean_at_start"] = False + return binding + dirty_files = git.get("dirty_files") + fingerprint = start_git_dirty_files_fingerprint(dirty_files) + if fingerprint is not None: + binding["start_git"] = {"dirty_files_sha256": fingerprint} + binding["subject_clean_at_start"] = not subject_path_dirty_in_git_status(subject_path, dirty_files) + return binding + + +def _validate_producer_binding( + producer: dict[str, Any], + *, + subject_path: str, + patch_bytes: bytes, + target: Path | None, +) -> str | None: + session_id = producer.get("work_session_id") + if not isinstance(session_id, str) or not session_id: + return "producer_session_missing" + + owned_delta = producer.get("owned_delta_sha256") + if not isinstance(owned_delta, str) or not owned_delta: + return "no_owned_work" + expected_owned = owned_delta_sha256(patch_bytes, subject_path) + if expected_owned is None or expected_owned != owned_delta: + return "no_owned_work" + + subject_clean_at_start = producer.get("subject_clean_at_start") + if subject_clean_at_start is False: + return "subject_pre_existing_dirty" + + if target is None: + if subject_clean_at_start is not True: + return "producer_session_missing" + start_git = producer.get("start_git") + if not isinstance(start_git, dict) or not isinstance(start_git.get("dirty_files_sha256"), str): + return "producer_session_missing" + return None + + session_payload = load_work_session_payload(target, session_id) + if session_payload is None: + return "producer_session_missing" + if session_payload.get("id") != session_id: + return "producer_session_missing" + + start = session_payload.get("start") + if not isinstance(start, dict): + return "producer_session_missing" + git = start.get("git") + if not isinstance(git, dict): + return "producer_session_missing" + + dirty_files = git.get("dirty_files") + if subject_path_dirty_in_git_status(subject_path, dirty_files): + if producer.get("subject_clean_at_start") is True: + return "concurrent_session_ownership" + return "subject_pre_existing_dirty" + + start_git = producer.get("start_git") + computed_fp = start_git_dirty_files_fingerprint(dirty_files) + if not isinstance(start_git, dict): + return "producer_session_missing" + stored_fp = start_git.get("dirty_files_sha256") + if not isinstance(stored_fp, str) or not stored_fp or computed_fp is None or stored_fp != computed_fp: + return "producer_session_mismatch" + + if subject_clean_at_start is not True: + return "producer_session_missing" + return None + + +def _missing_required_utility_checks(receipt: dict[str, Any]) -> bool: + required = receipt.get("required_utility_check_ids") + if required is None: + return False + if not isinstance(required, list): + return True + if not required: + return False + commands = receipt.get("commands") + if not isinstance(commands, list): + return True + utility_ids = { + command.get("check_id") + for command in commands + if isinstance(command, dict) and command.get("check_role") == "utility_guardrail" + } + return any(not isinstance(check_id, str) or check_id not in utility_ids for check_id in required) + + +def subject_hash_for_path(target: Path, subject_path: str) -> str | None: + path = target / subject_path + try: + return _sha256_prefixed(path.read_bytes()) + except OSError: + return None + + +def verifier_session_id() -> str | None: + claude_session = os.environ.get("BRIGADE_CLAUDE_SESSION") + if claude_session and re.fullmatch(r"[0-9a-f]{16}", claude_session): + return claude_session + override = os.environ.get("BRIGADE_VERIFY_SESSION_ID", "").strip() + return override or None + + +def _verifier_session_id() -> str | None: + return verifier_session_id() + + +def _binding_dict(receipt: dict[str, Any]) -> dict[str, Any] | None: + binding = receipt.get("subject_binding") + return binding if isinstance(binding, dict) else None + + +def _commands_missing_check_role(receipt: dict[str, Any]) -> bool: + commands = receipt.get("commands") + if not isinstance(commands, list) or not commands: + return True + for command in commands: + if not isinstance(command, dict): + return True + role = command.get("check_role") + if role not in {"effectiveness", "utility_guardrail"}: + return True + check_id = command.get("check_id") + if not isinstance(check_id, str) or not check_id: + return True + return False + + +def _receipt_has_unclassified_failure(receipt: dict[str, Any]) -> tuple[bool, bool, bool]: + """Return (has_failure, infrastructure, missing_or_unknown_taxonomy).""" + has_failure = receipt.get("status") not in {None, "completed", "running"} + infrastructure = False + missing_or_unknown = False + if receipt.get("status") not in {None, "completed", "running"}: + failure_class = receipt.get("failure_class") + if not isinstance(failure_class, str) or failure_class not in KNOWN_VERIFY_FAILURE_CLASSES: + missing_or_unknown = True + elif failure_class in VERIFY_INFRASTRUCTURE_FAILURE_CLASSES: + infrastructure = True + commands = receipt.get("commands") + if isinstance(commands, list): + for command in commands: + if not isinstance(command, dict): + continue + status = command.get("status") + exit_code = command.get("exit_code") + if status == "completed" and exit_code == 0: + continue + has_failure = True + failure_class = command.get("failure_class") + if not isinstance(failure_class, str) or failure_class not in KNOWN_VERIFY_FAILURE_CLASSES: + missing_or_unknown = True + elif failure_class in VERIFY_INFRASTRUCTURE_FAILURE_CLASSES: + infrastructure = True + return has_failure, infrastructure, missing_or_unknown + + +def _receipt_digest_reason(receipt: dict[str, Any]) -> str | None: + digests = receipt.get("digests") + if not isinstance(digests, dict): + return "receipt_digest_missing" + expected = digests.get("receipt_sha256") + if not isinstance(expected, str) or not expected: + return "receipt_digest_missing" + actual = localio.canonical_json_digest(receipt, exclude_keys={"digests"}) + if expected != actual: + return "receipt_digest_mismatch" + return None + + +def _manifest_binding_dict(binding: dict[str, Any]) -> dict[str, Any] | None: + manifest_binding = binding.get("manifest_binding") + return manifest_binding if isinstance(manifest_binding, dict) else None + + +def _manifest_binding_structure_complete(manifest_binding: dict[str, Any]) -> bool: + manifest_id = manifest_binding.get("manifest_id") + payload_sha256 = manifest_binding.get("payload_sha256") + return ( + isinstance(manifest_id, str) and bool(manifest_id) and isinstance(payload_sha256, str) and bool(payload_sha256) + ) + + +def _verifier_identity_complete(binding: dict[str, Any]) -> bool: + verifier = binding.get("verifier_identity") + if not isinstance(verifier, dict): + return False + verifier_id = verifier.get("verifier_id") + session_id = verifier.get("session_id") + return isinstance(verifier_id, str) and bool(verifier_id) and isinstance(session_id, str) and bool(session_id) + + +def _normalize_command_identity(command: dict[str, Any]) -> str | None: + argv = command.get("argv") + if isinstance(argv, list) and argv: + return shlex.join(str(item) for item in argv) + value = command.get("command") + if isinstance(value, str) and value: + try: + return shlex.join(shlex.split(value)) + except ValueError: + return value + return None + + +def _normalize_manifest_command(command: str | list[str]) -> str: + if isinstance(command, list): + return shlex.join(command) + try: + return shlex.join(shlex.split(command)) + except ValueError: + return command + + +def _manifest_command_plan(manifest: verify_manifest.VerifyManifest) -> list[tuple[str, str, str | None, str]]: + plan: list[tuple[str, str, str | None, str]] = [] + for check in manifest.checks: + plan.append( + ( + check.check_id, + check.check_role, + check.obligation_id, + _normalize_manifest_command(check.command), + ) + ) + return plan + + +def _receipt_command_plan(receipt: dict[str, Any]) -> list[tuple[str, str, str | None, str]] | None: + commands = receipt.get("commands") + if not isinstance(commands, list) or not commands: + return None + plan: list[tuple[str, str, str | None, str]] = [] + for command in commands: + if not isinstance(command, dict): + return None + check_id = command.get("check_id") + check_role = command.get("check_role") + if not isinstance(check_id, str) or not check_id: + return None + if check_role not in {"effectiveness", "utility_guardrail"}: + return None + normalized = _normalize_command_identity(command) + if normalized is None: + return None + obligation_id = command.get("obligation_id") + if obligation_id is not None and (not isinstance(obligation_id, str) or not obligation_id): + return None + plan.append((check_id, check_role, obligation_id if isinstance(obligation_id, str) else None, normalized)) + return plan + + +def _required_utility_ids_match(receipt: dict[str, Any], manifest: verify_manifest.VerifyManifest) -> bool: + receipt_required = receipt.get("required_utility_check_ids") + if not manifest.required_utility_check_ids: + return receipt_required is None or receipt_required == [] + if not isinstance(receipt_required, list): + return False + return tuple(str(item) for item in receipt_required) == manifest.required_utility_check_ids + + +def _manifest_binding_matches_resolved( + target: Path, + manifest_binding: dict[str, Any], + manifest: verify_manifest.VerifyManifest, +) -> bool: + if manifest_binding.get("manifest_id") != manifest.manifest_id: + return False + expected_payload = verify_manifest.manifest_payload_sha256(manifest) + if expected_payload is None or manifest_binding.get("payload_sha256") != expected_payload: + return False + source_path = manifest_binding.get("source_path") + if source_path is not None: + expected_source = verify_manifest.manifest_source_path(target, manifest) + if not isinstance(source_path, str) or source_path != expected_source: + return False + return True + + +def _binding_matches_manifest(binding: dict[str, Any], manifest: verify_manifest.VerifyManifest) -> bool: + if binding.get("artifact_kind") != manifest.artifact_kind: + return False + if binding.get("artifact_id") != manifest.artifact_id: + return False + verifier = binding.get("verifier_identity") + if not isinstance(verifier, dict) or verifier.get("verifier_id") != manifest.verifier_id: + return False + if manifest.binding_mode == "patch_backed": + if binding.get("patch_source") != manifest.patch_source: + return False + if manifest.binding_mode == "fixture_eval": + fixture_binding = binding.get("fixture_binding") + if not isinstance(fixture_binding, dict): + return False + if fixture_binding.get("manifest_id") != manifest.fixture_manifest_id: + return False + if fixture_binding.get("case_id") != manifest.fixture_case_id: + return False + if fixture_binding.get("check_id") != manifest.fixture_check_id: + return False + return True + + +def _validate_manifest_contract( + receipt: dict[str, Any], + binding: dict[str, Any], + *, + target: Path | None, +) -> str | None: + manifest_binding = _manifest_binding_dict(binding) + if manifest_binding is None or not _manifest_binding_structure_complete(manifest_binding): + return "verifier_manifest_missing" + + binding_mode = binding.get("binding_mode") + if binding_mode in {"patch_backed", "fixture_eval"} and not _verifier_identity_complete(binding): + return "verifier_identity_missing" + + if target is None: + return None + + verify_manifest_id = receipt.get("verify_manifest_id") + if not isinstance(verify_manifest_id, str) or not verify_manifest_id: + return "verifier_manifest_missing" + if verify_manifest_id != manifest_binding.get("manifest_id"): + return "verifier_manifest_mismatch" + + manifest, error = verify_manifest.resolve_manifest(target, verify_manifest_id) + if manifest is None: + return "verifier_manifest_missing" if error and "not found" in error else "verifier_manifest_mismatch" + + if not _manifest_binding_matches_resolved(target, manifest_binding, manifest): + return "verifier_manifest_mismatch" + if not _binding_matches_manifest(binding, manifest): + return "verifier_manifest_mismatch" + + receipt_plan = _receipt_command_plan(receipt) + manifest_plan = _manifest_command_plan(manifest) + if receipt_plan is None or receipt_plan != manifest_plan: + return "verifier_manifest_mismatch" + if not _required_utility_ids_match(receipt, manifest): + return "verifier_manifest_mismatch" + return None + + +def _patch_identity_complete(receipt: dict[str, Any]) -> bool: + for key in ("baseline_commit", "tree_fingerprint", "changes_patch_sha256"): + value = receipt.get(key) + if not isinstance(value, str) or not value: + return False + return True + + +def project_trial( + receipt: dict[str, Any], *, target: Path | None = None, patch_path: Path | None = None +) -> TrialProjection: + binding = _binding_dict(receipt) + if binding is None: + return TrialProjection(eligible=False, reason="unattributed", attributed=False) + if _commands_missing_check_role(receipt): + return TrialProjection(eligible=False, reason="missing_check_role", attributed=True) + + binding_mode = binding.get("binding_mode") + if binding_mode == "patch_backed": + patch_binding = binding.get("patch_binding") + if not isinstance(patch_binding, dict): + return TrialProjection(eligible=False, reason="patch_binding_incomplete", attributed=True) + subject_path = patch_binding.get("subject_path") + subject_hash = patch_binding.get("subject_hash") + if not isinstance(subject_path, str) or not subject_path: + return TrialProjection(eligible=False, reason="patch_binding_incomplete", attributed=True) + if not isinstance(subject_hash, str) or not subject_hash: + return TrialProjection(eligible=False, reason="patch_binding_incomplete", attributed=True) + if not _patch_identity_complete(receipt): + return TrialProjection(eligible=False, reason="incomplete_patch_identity", attributed=True) + resolved_target = target + if resolved_target is None: + target_value = receipt.get("target") + if isinstance(target_value, str) and target_value: + resolved_target = Path(target_value) + patch_file = patch_path + if patch_file is None: + receipt_path = receipt.get("path") + if isinstance(receipt_path, str) and receipt_path: + patch_file = Path(receipt_path) / "changes.patch" + patch_bytes = b"" + if patch_file is not None and patch_file.is_file(): + try: + patch_bytes = patch_file.read_bytes() + except OSError: + patch_bytes = b"" + if not patch_bytes.strip(): + return TrialProjection(eligible=False, reason="empty_patch", attributed=True) + patch_hash = receipt.get("changes_patch_sha256") + if not isinstance(patch_hash, str) or hashlib.sha256(patch_bytes).hexdigest() != patch_hash: + return TrialProjection(eligible=False, reason="patch_digest_mismatch", attributed=True) + producer = binding.get("producer_binding") + if not isinstance(producer, dict): + return TrialProjection(eligible=False, reason="producer_session_missing", attributed=True) + producer_reason = _validate_producer_binding( + producer, + subject_path=subject_path, + patch_bytes=patch_bytes, + target=resolved_target, + ) + if producer_reason is not None: + return TrialProjection(eligible=False, reason=producer_reason, attributed=True) + if binding.get("patch_source") == "generated": + verifier = binding.get("verifier_identity") + producer_session = producer.get("work_session_id") + if not isinstance(verifier, dict): + return TrialProjection(eligible=False, reason="verifier_not_independent", attributed=True) + verifier_session = verifier.get("session_id") + if ( + not isinstance(verifier_session, str) + or not verifier_session + or not isinstance(producer_session, str) + or not producer_session + or verifier_session == producer_session + ): + return TrialProjection(eligible=False, reason="verifier_not_independent", attributed=True) + if resolved_target is not None: + live_hash = subject_hash_for_path(resolved_target, subject_path) + if live_hash is None or live_hash != subject_hash: + return TrialProjection(eligible=False, reason="subject_hash_mismatch", attributed=True) + patch_tuple = patch_binding + for key in ("baseline_commit", "tree_fingerprint", "changes_patch_sha256"): + if patch_tuple.get(key) != receipt.get(key): + return TrialProjection(eligible=False, reason="patch_binding_incomplete", attributed=True) + elif binding_mode == "fixture_eval": + fixture_binding = binding.get("fixture_binding") + if not isinstance(fixture_binding, dict): + return TrialProjection(eligible=False, reason="fixture_binding_incomplete", attributed=True) + for key in ("manifest_id", "case_id", "check_id"): + value = fixture_binding.get(key) + if not isinstance(value, str) or not value: + return TrialProjection(eligible=False, reason="fixture_binding_incomplete", attributed=True) + fingerprint = binding.get("content_fingerprint") + if not isinstance(fingerprint, str) or not fingerprint: + return TrialProjection(eligible=False, reason="fixture_binding_incomplete", attributed=True) + else: + return TrialProjection(eligible=False, reason="missing_subject_binding", attributed=True) + + manifest_reason = _validate_manifest_contract(receipt, binding, target=target) + if manifest_reason is not None: + return TrialProjection(eligible=False, reason=manifest_reason, attributed=True) + + for key in ("artifact_kind", "artifact_id", "content_fingerprint"): + value = binding.get(key) + if not isinstance(value, str) or not value: + return TrialProjection(eligible=False, reason="missing_subject_binding", attributed=True) + + if _missing_required_utility_checks(receipt): + return TrialProjection(eligible=False, reason="missing_required_utility_check", attributed=True) + + has_failure, infrastructure, missing_or_unknown = _receipt_has_unclassified_failure(receipt) + if has_failure: + if missing_or_unknown: + return TrialProjection(eligible=False, reason="failure_taxonomy_missing", attributed=True) + if infrastructure: + return TrialProjection( + eligible=False, + reason="failure_taxonomy_infrastructure", + attributed=True, + infrastructure_excluded=True, + ) + + digest_reason = _receipt_digest_reason(receipt) + if digest_reason is not None: + return TrialProjection(eligible=False, reason=digest_reason, attributed=True) + + return TrialProjection(eligible=True, reason="eligible", attributed=True) diff --git a/src/brigade/work_cmd/session/briefing.py b/src/brigade/work_cmd/session/briefing.py index e912326b..7707b59a 100644 --- a/src/brigade/work_cmd/session/briefing.py +++ b/src/brigade/work_cmd/session/briefing.py @@ -492,8 +492,19 @@ def _brief_payload(target: Path, *, limit: int = 3, include_code_graph: bool = F "records_path": outcome_health["records_path"], "verify_run_count": outcome_health["verify_run_count"], "record_count": outcome_health["record_count"], + "legacy_records_audit_only": outcome_health["legacy_records_audit_only"], + "legacy_records_note": outcome_health["legacy_records_note"], "scored_artifact_count": outcome_health["scored_artifact_count"], "promoted_count": outcome_health["promoted_count"], + "attributed_receipt_count": outcome_health["attributed_receipt_count"], + "unattributed_receipt_count": outcome_health["unattributed_receipt_count"], + "eligible_receipt_count": outcome_health["eligible_receipt_count"], + "ineligible_receipt_count": outcome_health["ineligible_receipt_count"], + "attributed_ineligible_receipt_count": outcome_health["attributed_ineligible_receipt_count"], + "ineligibility_rate": outcome_health["ineligibility_rate"], + "leading_ineligibility_reason": outcome_health["leading_ineligibility_reason"], + "exploration_bands": outcome_health["exploration_bands"], + "latest_receipt_window": outcome_health["latest_receipt_window"], "issue_count": outcome_health["issue_count"], "top_issue": outcome_health["top_issue"], }, @@ -583,10 +594,22 @@ def brief(*, target: Path, limit: int = 3, json_output: bool = False) -> int: print( "outcome_loop: " f"verify_runs={outcome_loop.get('verify_run_count')} " - f"records={outcome_loop.get('record_count')} " - f"scored={outcome_loop.get('scored_artifact_count')} " + f"attributed_receipts={outcome_loop.get('attributed_receipt_count')} " + f"eligible={outcome_loop.get('eligible_receipt_count')} " + f"ineligible={outcome_loop.get('ineligible_receipt_count')} " + f"unattributed={outcome_loop.get('unattributed_receipt_count')} " + f"ineligibility_rate={outcome_loop.get('ineligibility_rate')} " f"promoted={outcome_loop.get('promoted_count')}" ) + bands = outcome_loop.get("exploration_bands") if isinstance(outcome_loop.get("exploration_bands"), dict) else {} + if bands: + print( + "outcome_exploration_bands: " + f"unseen={bands.get('unseen', 0)} " + f"candidate={bands.get('candidate', 0)} " + f"provisional={bands.get('provisional', 0)} " + f"promoted={bands.get('promoted', 0)}" + ) top_outcome = outcome_loop.get("top_issue") if isinstance(outcome_loop.get("top_issue"), dict) else None if top_outcome: print(f"outcome_loop_issue: {top_outcome.get('name')} {helpers._short(str(top_outcome.get('detail', '')))}") diff --git a/src/brigade/work_cmd/verification.py b/src/brigade/work_cmd/verification.py index c36bf119..07e061b6 100644 --- a/src/brigade/work_cmd/verification.py +++ b/src/brigade/work_cmd/verification.py @@ -15,6 +15,7 @@ from typing import Any from uuid import uuid4 from .. import config, graphtrail_delta, localio, proc, receipt_schema, receipt_signing, runguard +from .. import verify_manifest, verify_trial from . import constants, helpers, ledger as ledger_mod from . import reviews as reviews_mod from . import scanners as scanners_mod @@ -203,6 +204,107 @@ def _run_verify_child_process( return "timed_out", None, stdout_text, stderr_text +def _active_work_session_id(target: Path) -> str | None: + current = helpers._current_path(target) + if not current.is_file(): + return None + session_id = current.read_text().strip() + return session_id or None + + +def _stamp_manifest_command_metadata(receipt: dict[str, Any], manifest: verify_manifest.VerifyManifest) -> None: + commands = receipt.get("commands") + if not isinstance(commands, list): + return + for index, command in enumerate(commands): + if not isinstance(command, dict) or index >= len(manifest.checks): + continue + spec = manifest.checks[index] + command["check_id"] = spec.check_id + command["check_role"] = spec.check_role + if spec.obligation_id: + command["obligation_id"] = spec.obligation_id + verify_trial.stamp_verify_command_failure_taxonomy(command) + + +def _build_subject_binding( + target: Path, + manifest: verify_manifest.VerifyManifest, + receipt: dict[str, Any], + run_dir: Path, +) -> dict[str, Any] | None: + fingerprint = verify_manifest.resolve_subject_fingerprint(target, manifest) + if fingerprint is None: + return None + verifier_session = verify_trial.verifier_session_id() or f"verify-{receipt.get('run_id')}" + binding: dict[str, Any] = { + "binding_mode": manifest.binding_mode, + "artifact_kind": manifest.artifact_kind, + "artifact_id": manifest.artifact_id, + "content_fingerprint": fingerprint, + "manifest_binding": verify_manifest.build_manifest_binding(target, manifest), + "verifier_identity": { + "verifier_id": manifest.verifier_id, + "session_id": verifier_session, + }, + } + if manifest.binding_mode == "fixture_eval": + binding["fixture_binding"] = { + "manifest_id": manifest.fixture_manifest_id, + "case_id": manifest.fixture_case_id, + "check_id": manifest.fixture_check_id, + } + return binding + if manifest.subject_path is None: + return None + patch_path = run_dir / "changes.patch" + try: + patch_bytes = patch_path.read_bytes() + except OSError: + patch_bytes = b"" + owned_delta = verify_trial.owned_delta_sha256(patch_bytes, manifest.subject_path) + subject_hash = verify_trial.subject_hash_for_path(target, manifest.subject_path) + if subject_hash is None: + return None + binding["patch_source"] = manifest.patch_source + session_dir = helpers._active_session_dir(target) + session_payload = helpers._read_session(session_dir) if session_dir is not None else None + if isinstance(session_payload, dict) and session_payload.get("id"): + session_id = str(session_payload["id"]) + else: + session_id = _active_work_session_id(target) + binding["producer_binding"] = verify_trial.build_producer_binding( + session_id=session_id, + session_payload=session_payload, + subject_path=manifest.subject_path, + owned_delta=owned_delta, + ) + binding["patch_binding"] = { + "baseline_commit": receipt.get("baseline_commit"), + "tree_fingerprint": receipt.get("tree_fingerprint"), + "changes_patch_sha256": receipt.get("changes_patch_sha256"), + "subject_path": manifest.subject_path, + "subject_hash": subject_hash, + } + return binding + + +def _apply_manifest_scoring_fields( + target: Path, + receipt: dict[str, Any], + manifest: verify_manifest.VerifyManifest, + run_dir: Path, +) -> None: + receipt["verify_manifest_id"] = manifest.manifest_id + if manifest.required_utility_check_ids: + receipt["required_utility_check_ids"] = list(manifest.required_utility_check_ids) + _stamp_manifest_command_metadata(receipt, manifest) + subject_binding = _build_subject_binding(target, manifest, receipt, run_dir) + if subject_binding is not None: + receipt["subject_binding"] = subject_binding + verify_trial.stamp_verify_receipt_failure_taxonomy(receipt) + + def _finalize_verify_receipt( target: Path, run_dir: Path, @@ -230,6 +332,12 @@ def _finalize_verify_receipt( receipt["status"] = "failed" else: receipt["status"] = "rejected" + commands = receipt.get("commands") + if isinstance(commands, list): + for command in commands: + if isinstance(command, dict): + verify_trial.stamp_verify_command_failure_taxonomy(command) + verify_trial.stamp_verify_receipt_failure_taxonomy(receipt) try: git = _receipt_git_snapshot(target) if git is not None: @@ -544,18 +652,37 @@ def _verification_evidence_payload(target: Path, session: tuple[Path, dict[str, } -def _verify_plan_payload(target: Path, commands: list[str] | None = None) -> dict[str, Any]: +def _verify_plan_payload( + target: Path, + commands: list[str] | None = None, + *, + manifest_id: str | None = None, +) -> dict[str, Any]: target = target.expanduser().resolve() - planned_commands = commands if commands is not None else _default_verify_commands(target) + manifest: verify_manifest.VerifyManifest | None = None + if manifest_id: + manifest, error = verify_manifest.resolve_manifest(target, manifest_id) + if manifest is None: + return { + "target": str(target), + "verify_runs_root": str(helpers._verify_runs_root(target)), + "commands": [], + "blockers": [error or f"verify manifest not found: {manifest_id}"], + "evidence": _verification_evidence_payload(target), + "suggested_command": f'brigade work verify run --manifest "{manifest_id}"', + } + planned_commands = manifest.planned_commands + else: + planned_commands = commands if commands is not None else _default_verify_commands(target) evidence = _verification_evidence_payload(target) blockers: list[str] = [] if not planned_commands: - blockers.append("no verification commands found; pass --command") + blockers.append("no verification commands found; pass --command or --manifest") for command in planned_commands: _, _, error = _verify_parse_command(command, target) if error: blockers.append(f"{command}: {error}") - return { + payload = { "target": str(target), "verify_runs_root": str(helpers._verify_runs_root(target)), "commands": planned_commands, @@ -565,6 +692,13 @@ def _verify_plan_payload(target: Path, commands: list[str] | None = None) -> dic if planned_commands else 'brigade work verify run --command "..."', } + if manifest is not None: + payload["manifest_id"] = manifest.manifest_id + payload["scoreable"] = True + payload["suggested_command"] = f'brigade work verify run --manifest "{manifest.manifest_id}"' + elif commands is not None: + payload["scoreable"] = False + return payload def _write_verify_markdown(run_dir: Path, receipt: dict[str, Any]) -> None: @@ -722,6 +856,7 @@ def _run_verify_commands( timeout: int, *, graphtrail_timeout: float, + manifest: verify_manifest.VerifyManifest | None = None, ) -> tuple[dict[str, Any], int]: started = helpers._now() run_id = f"{started.strftime('%Y%m%d-%H%M%S')}-work-verify-{uuid4().hex[:6]}" @@ -860,6 +995,8 @@ def _run_verify_commands( ) finally: if receipt is not None and not finalized: + if manifest is not None: + _apply_manifest_scoring_fields(target, receipt, manifest, run_dir) receipt, rc = _safe_finalize_verify_receipt( target, run_dir, @@ -1070,13 +1207,14 @@ def verify_plan( *, target: Path, commands: list[str] | None = None, + manifest_id: str | None = None, json_output: bool = False, ) -> int: target = target.expanduser().resolve() if not target.is_dir(): print(f"error: --target is not a directory: {target}", file=sys.stderr) return 2 - payload = _verify_plan_payload(target, commands) + payload = _verify_plan_payload(target, commands, manifest_id=manifest_id) if json_output: print(json.dumps(payload, indent=2, sort_keys=True)) return 0 if not payload["blockers"] else 1 @@ -1120,6 +1258,7 @@ def verify_run( *, target: Path, commands: list[str | list[str]] | None = None, + manifest_id: str | None = None, timeout: int = 900, graphtrail_timeout: float | None = None, json_output: bool = False, @@ -1134,15 +1273,23 @@ def verify_run( if timeout < 1: print("error: --timeout must be a positive integer", file=sys.stderr) return 2 + manifest: verify_manifest.VerifyManifest | None = None + if manifest_id: + manifest, error = verify_manifest.resolve_manifest(target, manifest_id) + if manifest is None: + print(f"error: {error}", file=sys.stderr) + return 1 if error and "not found" in error else 2 + planned = manifest.planned_commands + else: + planned = commands if commands is not None else _default_verify_commands(target) + if not planned: + print("error: no verification commands found; pass --command or --manifest", file=sys.stderr) + return 2 try: effective_graphtrail_timeout = config.resolve_graphtrail_delta_timeout(target, graphtrail_timeout) except ValueError as exc: print(f"error: {exc}", file=sys.stderr) return 2 - planned = commands if commands is not None else _default_verify_commands(target) - if not planned: - print("error: no verification commands found; pass --command", file=sys.stderr) - return 2 planned_display = _planned_commands_display(planned) planned_identity = _planned_commands_identity(planned) try: @@ -1155,7 +1302,7 @@ def verify_run( return blocked_rc try: receipt = None - if reuse: + if reuse and manifest is None: fingerprint = _tree_fingerprint(target) latest = _latest_verify_receipt(target) if ( @@ -1168,7 +1315,11 @@ def verify_run( receipt, rc = _write_reused_receipt(target, latest, planned_display, timeout) if receipt is None: receipt, rc = _run_verify_commands( - target, planned, timeout, graphtrail_timeout=effective_graphtrail_timeout + target, + planned, + timeout, + graphtrail_timeout=effective_graphtrail_timeout, + manifest=manifest, ) except KeyboardInterrupt: print("error: verification canceled by user", file=sys.stderr) diff --git a/tests/test_aboyeur.py b/tests/test_aboyeur.py index ba92676a..816b4902 100644 --- a/tests/test_aboyeur.py +++ b/tests/test_aboyeur.py @@ -1022,6 +1022,26 @@ def test_assignment_payload_serializes_stage(): assert payload == [{"stage": 2, "worker": "coder", "task": "implement it"}] +def test_assignment_payload_serializes_selected_skill_ids(): + payload = aboyeur._assignment_payload( + [ + aboyeur.Assignment( + worker="coder", + task="implement it", + selected_skill_ids=("brigade-work",), + ) + ] + ) + assert payload == [ + { + "stage": 1, + "worker": "coder", + "task": "implement it", + "selected_skill_ids": ["brigade-work"], + } + ] + + def test_run_dry_run_stops_after_plan(monkeypatch, capsys): calls = [] diff --git a/tests/test_operator_checkup.py b/tests/test_operator_checkup.py index 8e313b0e..a479a5e2 100644 --- a/tests/test_operator_checkup.py +++ b/tests/test_operator_checkup.py @@ -58,7 +58,7 @@ def test_operator_checkup_is_ready_when_all_surfaces_pass(monkeypatch, capsys): assert payload["ready"] is True assert payload["blocking_surface_count"] == 0 assert payload["next_command"] is None - assert payload["skipped_surfaces"] == ["work", "graph", "ledger"] + assert payload["skipped_surfaces"] == ["work", "graph", "ledger", "outcome"] def test_operator_checkup_loop_reports_graph_ledger_and_brief_hit_rate(monkeypatch, tmp_path, capsys): @@ -162,6 +162,7 @@ def unexpected(**kwargs): "work", "graph", "ledger", + "outcome", ] assert payload["surfaces"][0]["name"] == "doctor" assert isinstance(payload["surfaces"][0]["elapsed_seconds"], float) @@ -183,6 +184,7 @@ def test_operator_checkup_lists_surfaces_and_evidence_loop_preset(tmp_path, caps "work", "graph", "ledger", + "outcome", } diff --git a/tests/test_outcome_activegraph.py b/tests/test_outcome_activegraph.py index 86b7719f..bc2a61a4 100644 --- a/tests/test_outcome_activegraph.py +++ b/tests/test_outcome_activegraph.py @@ -55,10 +55,27 @@ def test_rebuild_status_detects_drift(tmp_path, capsys): assert any(d["artifact_id"] == "card-x" and d["issue"] == "mismatch" for d in payload["drift"]) +def test_rebuild_status_reproduces_promoted_route_policy(tmp_path, capsys, monkeypatch): + from tests.test_scorecard_reconcile import _stub_execute, seed_registry_skill_scorecard_promotion + from tests.test_outcome_cmd import _write_registry_skill + + _stub_execute(monkeypatch) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") + assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 + capsys.readouterr() + + assert outcome_cmd.rebuild_status(target=tmp_path, check=True, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["reproducible"] is True + entry = outcome_cmd.load_status(tmp_path)["skill-x"] + assert entry["route_policy"]["policy_version"] == "scorecard.v1" + + def test_fork_and_diff_reflect_config_sensitivity(tmp_path, capsys): - # skill-x has enough signal to install under any threshold; skill-y only + # card-x has enough signal to install under any threshold; card-y only # installs once the threshold drops to 1. - _seed(tmp_path, _helped("skill-x", "skill", 2) + _helped("skill-y", "skill", 1)) + _seed(tmp_path, _helped("card-x", "card", 2) + _helped("card-y", "card", 1)) fork_a = tmp_path / "A.json" fork_b = tmp_path / "B.json" @@ -76,8 +93,8 @@ def test_fork_and_diff_reflect_config_sensitivity(tmp_path, capsys): payload = json.loads(capsys.readouterr().out) assert payload["identical"] is False changed = {c["artifact_id"]: c for c in payload["changed"]} - assert "skill-y" in changed # hold@candidate under A, install@promoted under B - assert "skill-x" not in changed # installs under both + assert "card-y" in changed # hold@candidate under A, install@promoted under B + assert "card-x" not in changed # installs under both # A fork is read-only: the live ledger has no status.json written by fork. assert not (tmp_path / "memory" / "outcome" / "status.json").exists() diff --git a/tests/test_outcome_backfill_scorecard.py b/tests/test_outcome_backfill_scorecard.py new file mode 100644 index 00000000..68376629 --- /dev/null +++ b/tests/test_outcome_backfill_scorecard.py @@ -0,0 +1,325 @@ +"""Receipt-only outcome backfill scorecard audit and operator surfaces (#574).""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from brigade import cli, outcome_cmd, scorecard, verify_trial, work_cmd +from brigade.work_cmd.session import briefing + +from tests.test_operator_checkup import _patch_all_doctors +from tests.test_scorecard import ( + _effectiveness_command, + _fixture_binding, + _write_verify_receipt, +) +from tests.test_verify_trial import _write_skill_subject +from tests.work_cmd_test_helpers import _init_git_repo + +pytest_plugins = ["tests.test_scorecard"] + + +def _records_path(target: Path) -> Path: + return target / "memory" / "outcome" / "records.jsonl" + + +def _records_mtime(target: Path) -> float | None: + path = _records_path(target) + return path.stat().st_mtime if path.is_file() else None + + +def test_backfill_scorecard_eligible_fixture_receipt(scoreable_target, capsys): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt( + scoreable_target, + "fixture-ok", + binding=binding, + commands=[_effectiveness_command()], + ) + + assert outcome_cmd.backfill_scorecard(target=scoreable_target, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["total_receipts"] == 1 + assert payload["eligible"] == 1 + assert payload["unattributed"] == 0 + assert payload["ineligible"] == 0 + assert payload["ineligibility_rate"] == 0.0 + assert payload["legacy_records_audit_only"] is True + assert "records.jsonl" in payload["legacy_records_note"] + + +def test_backfill_scorecard_eligible_patch_backed_receipt(scoreable_target, monkeypatch): + from tests.test_verify_trial import _run_manifest_verify, _write_patch_manifest + + _write_patch_manifest(scoreable_target) + assert work_cmd.start(target=scoreable_target, title="verify-trial") == 0 + skill = _write_skill_subject(scoreable_target, content="# brigade-work\npatched\n") + import subprocess + + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch, start_session=False) + + audit = scorecard.receipt_scorecard_audit(scoreable_target) + assert audit["total_receipts"] == 1 + assert audit["eligible"] == 1 + + +def test_backfill_scorecard_missing_binding_is_unattributed(scoreable_target, capsys): + _write_verify_receipt( + scoreable_target, + "no-binding", + binding=None, + commands=[_effectiveness_command()], + ) + + assert outcome_cmd.backfill_scorecard(target=scoreable_target, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["unattributed"] == 1 + assert payload["ineligible"] == 1 + assert payload["eligible"] + payload["ineligible"] == payload["total_receipts"] + assert sum(payload["ineligible_by_reason"].values()) == payload["ineligible"] + assert payload["ineligible_by_reason"]["unattributed"] == 1 + + +def test_backfill_scorecard_invalid_receipt_json_is_ineligible_unattributed(scoreable_target): + runs_root = scoreable_target / ".brigade" / "work" / "verify-runs" + (runs_root / "bad-json").mkdir(parents=True) + (runs_root / "bad-json" / "receipt.json").write_text("{not json\n", encoding="utf-8") + (runs_root / "not-object").mkdir(parents=True) + (runs_root / "not-object" / "receipt.json").write_text("[1, 2, 3]\n", encoding="utf-8") + + audit = scorecard.receipt_scorecard_audit(scoreable_target) + + assert audit["total_receipts"] == 2 + assert audit["eligible"] == 0 + assert audit["ineligible"] == 2 + assert audit["unattributed"] == 2 + assert audit["attributed_ineligible"] == 0 + assert audit["eligible"] + audit["ineligible"] == audit["total_receipts"] + assert sum(audit["ineligible_by_reason"].values()) == audit["ineligible"] + assert audit["ineligible_by_reason"]["invalid_receipt_json"] == 2 + + +def test_backfill_scorecard_patch_mismatch_is_ineligible(scoreable_target, monkeypatch): + from tests.test_verify_trial import _run_manifest_verify, _write_patch_manifest + + _write_patch_manifest(scoreable_target) + assert work_cmd.start(target=scoreable_target, title="verify-trial") == 0 + skill = _write_skill_subject(scoreable_target, content="# brigade-work\npatched\n") + import subprocess + + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + receipt = _run_manifest_verify( + scoreable_target, + "test-skill-patch", + monkeypatch=monkeypatch, + start_session=False, + ) + patch_path = Path(receipt["path"]) / "changes.patch" + patch_path.write_bytes(b"diff --git a/x b/x\n") + + audit = scorecard.receipt_scorecard_audit(scoreable_target) + assert audit["ineligible"] == 1 + assert audit["ineligible_by_reason"]["patch_digest_mismatch"] == 1 + + +def test_backfill_scorecard_infrastructure_failure_is_ineligible(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + command = _effectiveness_command() + command["status"] = "timed_out" + command["exit_code"] = None + command["failure_class"] = "infrastructure" + command["failure_kind"] = "timeout" + _write_verify_receipt( + scoreable_target, + "infra-fail", + binding=binding, + status="failed", + commands=[command], + extra={ + "failure_class": "infrastructure", + "failure_kind": "timeout", + }, + ) + + audit = scorecard.receipt_scorecard_audit(scoreable_target) + assert audit["ineligible"] == 1 + assert audit["ineligible_by_reason"]["failure_taxonomy_infrastructure"] == 1 + + +def test_backfill_scorecard_does_not_mutate_records_jsonl(scoreable_target, capsys): + records = _records_path(scoreable_target) + records.parent.mkdir(parents=True, exist_ok=True) + records.write_text( + json.dumps( + { + "artifact_id": "brigade-work", + "artifact_kind": "skill", + "source": "verify", + "signal_value": 1, + } + ) + + "\n" + ) + before_mtime = _records_mtime(scoreable_target) + before_text = records.read_text() + + _write_verify_receipt(scoreable_target, "audit-only", binding=None) + assert outcome_cmd.backfill_scorecard(target=scoreable_target, json_output=False) == 0 + capsys.readouterr() + + assert records.read_text() == before_text + assert _records_mtime(scoreable_target) == before_mtime + + +def test_health_and_work_brief_expose_receipt_scorecard_fields(scoreable_target, capsys): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt( + scoreable_target, + "brief-fixture", + binding=binding, + commands=[_effectiveness_command()], + ) + + health = outcome_cmd.health(scoreable_target) + assert health["attributed_receipt_count"] >= 1 + assert health["eligible_receipt_count"] >= 1 + assert sum(health["exploration_bands"].values()) >= 1 + assert health["legacy_records_audit_only"] is True + + payload = briefing._brief_payload(scoreable_target, limit=3) + loop = payload["outcome_loop"] + assert loop["eligible_receipt_count"] == health["eligible_receipt_count"] + assert loop["exploration_bands"] == health["exploration_bands"] + assert loop["leading_ineligibility_reason"] == health["leading_ineligibility_reason"] + + assert work_cmd.brief(target=scoreable_target, json_output=False) == 0 + out = capsys.readouterr().out + assert "attributed_receipts=" in out + assert "eligible=" in out + assert "ineligible=" in out + assert "ineligibility_rate=" in out + assert "outcome_exploration_bands:" in out + + +def test_operator_checkup_outcome_surface_warns_on_half_fed_and_high_ineligibility(scoreable_target, capsys): + from brigade.operator_cmd import lifecycle + + for index in range(51): + _write_verify_receipt( + scoreable_target, + f"missing-binding-{index:02d}", + binding=None, + started_at=f"2026-07-26T{index % 24:02d}:00:00+00:00", + ) + + rc = lifecycle.checkup(target=scoreable_target, surfaces=["outcome"], json_output=True) + payload = json.loads(capsys.readouterr().out) + + assert rc == 1 + assert payload["selected_ready"] is False + issue_names = {issue["name"] for issue in payload["surfaces"][0]["details"]["issues"]} + assert "outcome_loop_half_fed" in issue_names + assert "outcome_receipt_ineligibility_high" in issue_names + latest = payload["surfaces"][0]["details"]["latest_receipt_window"] + assert latest["limit"] == scorecard.LATEST_RECEIPT_WINDOW + assert latest["count"] == scorecard.LATEST_RECEIPT_WINDOW + assert latest["ineligibility_rate"] > 0.5 + + +def test_operator_checkup_default_skips_outcome_surface(monkeypatch, capsys): + from brigade.operator_cmd import lifecycle + + def unexpected(**kwargs): + raise AssertionError("outcome surface should not run in default checkup") + + _patch_all_doctors(monkeypatch, skills_rc=0) + monkeypatch.setattr(lifecycle, "_checkup_outcome", unexpected) + + rc = lifecycle.checkup(target=Path("."), json_output=True) + payload = json.loads(capsys.readouterr().out) + + assert rc == 0 + assert "outcome" in payload["skipped_surfaces"] + assert payload["selected_surfaces"] == list(lifecycle.CHECKUP_DEFAULT_SURFACES) + + +def test_operator_checkup_lists_outcome_surface(tmp_path, capsys): + assert cli.main(["operator", "checkup", "--target", str(tmp_path), "--list-surfaces", "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + + assert "outcome" in payload["surface_names"] + assert "outcome" not in payload["default_surfaces"] + + +def test_backfill_scorecard_human_output(scoreable_target, capsys): + _write_verify_receipt(scoreable_target, "human", binding=None) + assert outcome_cmd.backfill_scorecard(target=scoreable_target, json_output=False) == 0 + out = capsys.readouterr().out + + assert "outcome backfill scorecard:" in out + assert "total=1" in out + assert "records.jsonl" in out + + +def test_backfill_scorecard_cli_dispatch(scoreable_target, capsys): + _write_verify_receipt(scoreable_target, "cli", binding=None) + assert ( + cli.main( + [ + "outcome", + "backfill", + "scorecard", + "--target", + str(scoreable_target), + "--json", + ] + ) + == 0 + ) + payload = json.loads(capsys.readouterr().out) + assert set(payload) >= { + "total_receipts", + "eligible", + "unattributed", + "ineligible", + "attributed_ineligible", + "ineligibility_rate", + "ineligible_by_reason", + "exploration_bands", + } + + +def test_health_half_fed_uses_receipt_only_advice(tmp_path): + _init_git_repo(tmp_path) + assert work_cmd.verify_run(target=tmp_path, commands=[f"{sys.executable} -c \"print('ok')\""]) == 0 + health = outcome_cmd.health(tmp_path) + + assert health["verify_run_count"] >= 1 + assert health["eligible_receipt_count"] == 0 + assert health["top_issue"]["name"] == "outcome_loop_half_fed" + assert "subject_binding" in health["top_issue"]["detail"] + assert "outcome capture" not in health["top_issue"]["detail"].lower() + + +def test_latest_receipt_window_sort_is_stable(scoreable_target): + for run_id, started_at in ( + ("b-run", "2026-07-26T12:00:00+00:00"), + ("a-run", "2026-07-26T12:00:00+00:00"), + ("c-run", "2026-07-25T12:00:00+00:00"), + ): + _write_verify_receipt(scoreable_target, run_id, binding=None, started_at=started_at) + + audits = scorecard.audit_all_verify_receipts(scoreable_target) + latest = scorecard.latest_receipt_audits(audits, limit=2) + assert [audit.run_id for audit in latest] == ["b-run", "a-run"] diff --git a/tests/test_outcome_cmd.py b/tests/test_outcome_cmd.py index 02f7235b..5e966ad2 100644 --- a/tests/test_outcome_cmd.py +++ b/tests/test_outcome_cmd.py @@ -1,8 +1,9 @@ import datetime as dt import json -from brigade import cli, localio, outcome, outcome_cmd, receipts_cmd, work_cmd +from brigade import cli, localio, outcome, outcome_cmd, receipts_cmd, scorecard, work_cmd +from tests.test_scorecard_reconcile import seed_registry_skill_scorecard_hurt, seed_registry_skill_scorecard_promotion from tests.work_cmd_test_helpers import _init_git_repo @@ -505,7 +506,8 @@ def _decisions_dir(target): def test_reconcile_dry_run_reports_install_without_writing(tmp_path, capsys): - _seed(tmp_path, _helped("skill-x", 2)) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=False, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) assert payload["apply"] is False @@ -527,7 +529,8 @@ def _fake(target, artifact_id, action): def test_reconcile_apply_installs_and_persists_status_and_receipt(tmp_path, capsys, monkeypatch): _stub_execute(monkeypatch) - _seed(tmp_path, _helped("skill-x", 2)) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) assert payload["applied"] == ["skill-x"] @@ -540,7 +543,8 @@ def test_reconcile_apply_does_not_promote_when_install_fails(tmp_path, capsys, m # A skill that crosses the threshold but cannot physically install must NOT be # marked promoted (the forward-only ratchet would hide the failure forever). _stub_execute(monkeypatch, install="install-skipped: not in registry") - _seed(tmp_path, _helped("skill-x", 2)) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) # not counted as applied, and the surfaced status stays candidate @@ -555,7 +559,8 @@ def test_reconcile_apply_does_not_promote_when_install_fails(tmp_path, capsys, m def test_reconcile_holds_inside_cooldown_after_apply(tmp_path, capsys, monkeypatch): _stub_execute(monkeypatch) - _seed(tmp_path, _helped("skill-x", 2)) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 capsys.readouterr() assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 @@ -596,13 +601,11 @@ def test_rebuild_status_check_accepts_mixed_legacy_and_delta_ledger(tmp_path, ca def test_reconcile_rolls_back_promoted_artifact_on_regression(tmp_path, capsys, monkeypatch): _stub_execute(monkeypatch) cfg = outcome.ReconcileConfig(cooldown_seconds=0) - _seed(tmp_path, _helped("skill-x", 2)) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, config=cfg, json_output=True) == 0 capsys.readouterr() - _seed( - tmp_path, - [outcome.OutcomeRecord("skill-x", "skill", "t9", "verify", -1, "regress", "2026-06-20T09:00:00+00:00")], - ) + seed_registry_skill_scorecard_hurt(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, config=cfg, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) actions = {d["artifact_id"]: d for d in payload["decisions"]} @@ -612,7 +615,8 @@ def test_reconcile_rolls_back_promoted_artifact_on_regression(tmp_path, capsys, def test_cli_outcome_reconcile_dispatch(tmp_path, capsys): - _seed(tmp_path, _helped("skill-x", 2)) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert cli.main(["outcome", "reconcile", "--target", str(tmp_path), "--json"]) == 0 assert "skill-x" in capsys.readouterr().out @@ -790,6 +794,9 @@ def test_rank_and_reconcile_count_verify_and_run_receipt_graph_deltas_identicall ) capsys.readouterr() + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") + assert outcome_cmd.rank(target=tmp_path, json_output=True) == 0 ranking = {item["artifact_id"]: item for item in json.loads(capsys.readouterr().out)["ranking"]} assert ranking["skill-x"]["helped"] == 2 @@ -837,6 +844,8 @@ def test_reconcile_dry_run_json_surfaces_graph_delta_counters(tmp_path, capsys): ), ], ) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=False, json_output=True) == 0 @@ -949,6 +958,8 @@ def test_reconcile_json_includes_brief_hit_rate_stats(tmp_path, capsys): ), ], ) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=False, json_output=True) == 0 decision = {item["artifact_id"]: item for item in json.loads(capsys.readouterr().out)["decisions"]}["skill-x"] assert decision["action"] == "install" @@ -1343,27 +1354,33 @@ def test_reconcile_does_not_promote_a_candidate_on_proven_stale_evidence(tmp_pat assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) - assert payload["decisions"] == [] # held, not installed + hold = payload["decisions"][0] + assert hold["action"] == "hold" + assert hold["reason"] == "withheld: missing scorecard" assert payload["applied"] == [] assert not _status_file(tmp_path).exists() def test_reconcile_still_promotes_a_never_edited_skill_grandfathered(tmp_path, capsys, monkeypatch): - # Grandfathering safety: a registry skill with only pre-fingerprint (legacy) - # signals promotes exactly as the pre-fingerprint ratchet did. No proven-stale - # records, so the decision and its receipt stay byte-identical. + # Legacy ledger rows alone no longer promote skills; receipt scorecards do. _stub_execute(monkeypatch) _write_registry_skill(tmp_path, "skill-x") _seed(tmp_path, _helped("skill-x", 2)) # legacy, no fingerprint + assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["applied"] == [] + hold = {d["artifact_id"]: d for d in payload["decisions"]}["skill-x"] + assert hold["action"] == "hold" + assert hold["reason"] == "withheld: missing scorecard" + + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) assert payload["applied"] == ["skill-x"] decision = {d["artifact_id"]: d for d in payload["decisions"]}["skill-x"] assert decision["action"] == "install" - # No stale evidence dropped, so no fingerprint audit fields leak into the receipt. - assert "stale_records" not in decision - assert "content_fingerprint" not in decision + assert decision["policy_version"] == scorecard.SCORECARD_POLICY_VERSION def test_reconcile_lets_an_edited_skill_re_earn_promotion_on_fresh_signals(tmp_path, capsys, monkeypatch): @@ -1372,18 +1389,14 @@ def test_reconcile_lets_an_edited_skill_re_earn_promotion_on_fresh_signals(tmp_p old_fp = _sha256_of(skill_md) _seed(tmp_path, _fp_helped("skill-x", 2, old_fp, start_hour=0)) skill_md.write_text("# rewritten text\n") - new_fp = _sha256_of(skill_md) - _seed(tmp_path, _fp_helped("skill-x", 2, new_fp, start_hour=5)) + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) assert payload["applied"] == ["skill-x"] decision = {d["artifact_id"]: d for d in payload["decisions"]}["skill-x"] assert decision["action"] == "install" - # The decision was scored on the current text; the audit fields record what it dropped. - assert decision["content_fingerprint"] == new_fp - assert decision["stale_records"] == 2 - assert decision["lifetime_helped"] == 4 + assert decision["policy_version"] == scorecard.SCORECARD_POLICY_VERSION def test_reconcile_human_output_notes_a_fingerprint_narrowed_decision(tmp_path, capsys, monkeypatch): @@ -1392,30 +1405,26 @@ def test_reconcile_human_output_notes_a_fingerprint_narrowed_decision(tmp_path, old_fp = _sha256_of(skill_md) _seed(tmp_path, _fp_helped("skill-x", 2, old_fp, start_hour=0)) skill_md.write_text("# rewritten text\n") - new_fp = _sha256_of(skill_md) - _seed(tmp_path, _fp_helped("skill-x", 2, new_fp, start_hour=5)) + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=False) == 0 line = next(line for line in capsys.readouterr().out.splitlines() if "skill-x" in line) - assert "scored current text only" in line - assert f"rev {new_fp[:12]}" in line - assert "stale=2" in line + assert "verified helped, no regressions" in line + assert "[install]" in line def test_reconcile_output_byte_identical_for_unedited_skill(tmp_path, capsys, monkeypatch): - # The teeth must not disturb the common case: a skill with no proven-stale - # records produces exactly the pre-fingerprint one-line output. _stub_execute(monkeypatch) _write_registry_skill(tmp_path, "skill-x") - _seed(tmp_path, _helped("skill-x", 2)) + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=False, json_output=False) == 0 line = next(line for line in capsys.readouterr().out.splitlines() if "skill-x" in line) assert line == "- skill-x candidate -> promoted [install] verified helped, no regressions" def test_fork_projection_uses_the_current_fingerprint_cohort(tmp_path, capsys): - # Lifetime would cross install_min_helped (2 helped), but all of it is proven - # stale, so the fork must project a hold, not a promotion. + # Lifetime legacy rows do not promote; without a current-fingerprint scorecard + # the fork projection surfaces an explicit fail-closed hold. skill_md = _write_registry_skill(tmp_path, "skill-x", "# old text\n") old_fp = _sha256_of(skill_md) _seed(tmp_path, _fp_helped("skill-x", 2, old_fp)) @@ -1426,8 +1435,9 @@ def test_fork_projection_uses_the_current_fingerprint_cohort(tmp_path, capsys): capsys.readouterr() projection = json.loads(out.read_text()) entry = projection["artifacts"]["skill-x"] - assert entry["new_status"] != "promoted" - assert entry["helped"] == 0 # current cohort, not the 2 stale lifetime signals + assert entry["action"] == "hold" + assert entry["new_status"] == "candidate" + assert entry["reason"] == "withheld: missing scorecard" def _registry_skill_dir(target, skill_id): diff --git a/tests/test_outcome_execute.py b/tests/test_outcome_execute.py index e7628c7d..a82ab41d 100644 --- a/tests/test_outcome_execute.py +++ b/tests/test_outcome_execute.py @@ -4,6 +4,7 @@ from brigade import outcome, outcome_cmd, skills_cmd +from tests.test_scorecard_reconcile import seed_registry_skill_scorecard_hurt, seed_registry_skill_scorecard_promotion from tests.test_skills_cmd import _write_skill @@ -23,7 +24,7 @@ def _helped(artifact_id, n): def test_reconcile_apply_physically_installs_a_verified_skill(tmp_path, capsys): name = _import_skill(tmp_path) - outcome_cmd.append_records(tmp_path, _helped(name, 2)) + seed_registry_skill_scorecard_promotion(tmp_path, name) assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) decision = {d["artifact_id"]: d for d in payload["decisions"]}[name] @@ -35,15 +36,13 @@ def test_reconcile_apply_physically_installs_a_verified_skill(tmp_path, capsys): def test_reconcile_rollback_of_a_first_install_uninstalls(tmp_path, capsys): cfg = outcome.ReconcileConfig(cooldown_seconds=0) name = _import_skill(tmp_path) - outcome_cmd.append_records(tmp_path, _helped(name, 2)) + seed_registry_skill_scorecard_promotion(tmp_path, name) assert outcome_cmd.reconcile(target=tmp_path, apply=True, config=cfg, json_output=True) == 0 capsys.readouterr() dest = skills_cmd._install_dir(tmp_path.resolve(), "claude", name) assert dest.exists() - outcome_cmd.append_records( - tmp_path, [outcome.OutcomeRecord(name, "skill", "t9", "verify", -1, "regress", "2026-06-20T09:00:00+00:00")] - ) + seed_registry_skill_scorecard_hurt(tmp_path, name) assert outcome_cmd.reconcile(target=tmp_path, apply=True, config=cfg, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) decision = {d["artifact_id"]: d for d in payload["decisions"]}[name] @@ -54,7 +53,7 @@ def test_reconcile_rollback_of_a_first_install_uninstalls(tmp_path, capsys): def test_reconcile_dry_run_does_not_install(tmp_path, capsys): name = _import_skill(tmp_path) - outcome_cmd.append_records(tmp_path, _helped(name, 2)) + seed_registry_skill_scorecard_promotion(tmp_path, name) assert outcome_cmd.reconcile(target=tmp_path, apply=False, json_output=True) == 0 payload = json.loads(capsys.readouterr().out) decision = {d["artifact_id"]: d for d in payload["decisions"]}[name] diff --git a/tests/test_receipt_schema_audit.py b/tests/test_receipt_schema_audit.py index cfa77a19..ae43add8 100644 --- a/tests/test_receipt_schema_audit.py +++ b/tests/test_receipt_schema_audit.py @@ -10,7 +10,7 @@ import pytest -from brigade import aboyeur, outcome, outcome_cmd, receipt_schema, runguard +from brigade import aboyeur, outcome_cmd, receipt_schema, runguard from brigade.work_cmd import helpers, verification as verify_mod @@ -378,42 +378,12 @@ def test_work_closeout_emits_schema_version(tmp_path, monkeypatch): def test_outcome_decision_emits_schema_version(tmp_path, monkeypatch): - def _fake_execute(_target, _artifact_id, _action): - return "installed" - - monkeypatch.setattr(outcome_cmd, "_execute_skill_decision", _fake_execute) - records_path = tmp_path / "memory" / "outcome" / "records.jsonl" - records_path.parent.mkdir(parents=True) - row = outcome_cmd._record_payload( - outcome.OutcomeRecord( - "skill-x", - "skill", - "", - "verify", - 1, - "", - "2026-01-01T00:00:00+00:00", - ) - ) - records_path.write_text(json.dumps(row, sort_keys=True) + "\n") - records_path.write_text( - records_path.read_text() - + json.dumps( - outcome_cmd._record_payload( - outcome.OutcomeRecord( - "skill-x", - "skill", - "", - "verify", - 1, - "", - "2026-01-01T01:00:00+00:00", - ) - ), - sort_keys=True, - ) - + "\n" - ) + from tests.test_outcome_cmd import _write_registry_skill + from tests.test_scorecard_reconcile import _stub_execute, seed_registry_skill_scorecard_promotion + + _stub_execute(monkeypatch) + _write_registry_skill(tmp_path, "skill-x") + seed_registry_skill_scorecard_promotion(tmp_path, "skill-x") assert outcome_cmd.reconcile(target=tmp_path, apply=True, json_output=True) == 0 decision_path = next((tmp_path / "memory" / "outcome" / "decisions").glob("*.json")) decision = json.loads(decision_path.read_text()) diff --git a/tests/test_route_policy.py b/tests/test_route_policy.py new file mode 100644 index 00000000..ef33ac18 --- /dev/null +++ b/tests/test_route_policy.py @@ -0,0 +1,1206 @@ +"""Route-level exploration policy and band classifier (#573).""" + +from __future__ import annotations + +import copy +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from brigade import aboyeur, outcome_cmd, route_policy, scorecard, verify_manifest, verify_trial +from brigade.roster import Agent, Roster +from brigade.route_receipts import route_decision_payload, write_route_decision + +from tests.test_scorecard import ( + _SCORECARD_MANIFEST_ID, + _effectiveness_command, + _ensure_scorecard_manifest, + _fixture_binding, + _track_workspace_manifest, + _write_verify_receipt, +) +from tests.test_verify_trial import _write_skill_subject + +pytest_plugins = ["tests.test_scorecard"] + + +def _plan_mode_roster(orchestrator_cli: str) -> Roster: + return Roster( + orchestrator="chef", + agents={ + "chef": Agent("chef", orchestrator_cli, "plan and synthesize"), + "coder": Agent("coder", "codex", "write code"), + }, + max_workers=1, + ) + + +def _write_skill_for(target: Path, artifact_id: str) -> None: + _write_skill_subject(target, rel=f"skills/{artifact_id}/SKILL.md", content=f"# {artifact_id}\n") + subprocess = __import__("subprocess") + subprocess.run(["git", "add", f"skills/{artifact_id}"], cwd=target, check=True, stdout=subprocess.DEVNULL) + + +def _write_ineligible_receipt(target: Path, run_id: str, *, artifact_id: str, manifest_id: str) -> None: + fingerprint = _skill_fingerprint(target, artifact_id) + binding = _fixture_binding( + fingerprint=fingerprint, + target=target, + manifest_id=manifest_id, + ) + run_dir = target / ".brigade" / "work" / "verify-runs" / run_id + run_dir.mkdir(parents=True) + receipt = { + "schema": "brigade.work.verify_run.v1", + "schema_version": 1, + "status": "failed", + "target": str(target), + "subject_binding": { + "artifact_kind": "skill", + "artifact_id": artifact_id, + "content_fingerprint": fingerprint, + }, + "verify_manifest": binding, + "commands": [], + } + (run_dir / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + + +def _run_receipt(*, skill_candidates: list[str] | None = None, routing_budget: dict | None = None) -> dict: + route: dict = { + "attached": True, + "route": ["implement", "verify"], + "size": "S", + "signals": ["code"], + "confidence": "high", + "template_version": "vertical-slice.v1", + } + if skill_candidates is not None: + route["skill_candidates"] = skill_candidates + if routing_budget is not None: + route["routing_budget"] = routing_budget + return { + "status": "ok", + "started_at": "2026-07-20T12:00:00+00:00", + "route": route, + } + + +def _route_brief(): + return aboyeur.route_brief("implement helper", template="vertical-slice") + + +def _write_prior_decision( + runs_dir: Path, + run_id: str, + *, + route_class: str, + assignments: list[dict], + started_at: str = "2026-07-19T12:00:00+00:00", +) -> Path: + run_dir = runs_dir / run_id + run_dir.mkdir(parents=True) + (run_dir / "run.json").write_text( + json.dumps( + { + "started_at": started_at, + "route": {"attached": True, "route": ["implement"], "size": "S", "signals": ["code"]}, + } + ) + + "\n" + ) + decision = { + "schema_version": "brigade.route-decision.v1", + "decided_at": started_at, + "exploration": {"route_class": route_class}, + "skill_assignments": assignments, + } + path = run_dir / "route-decision.json" + path.write_text(json.dumps(decision, indent=2) + "\n") + return path + + +def _card_with_effectiveness( + *, + artifact_id: str = "brigade-work", + helped: int, + hurt: int = 0, + trials: int | None = None, + wilson: float = 0.2, +): + trials = trials if trials is not None else helped + hurt + return scorecard.SubjectScorecard( + subject=scorecard.SubjectRef("skill", artifact_id, "fp"), + dimensions={ + "effectiveness": {"helped": helped, "hurt": hurt, "wilson": wilson, "trials": trials}, + "verifier_cost": {}, + "retry_stability": {}, + "evidence_integrity": {}, + }, + utility_guardrails={ + "required_check_ids": ["guardrail.tests-green"], + "passing_trials": 0, + "required_trials": 2, + "per_check": {}, + }, + ) + + +def _write_route_opt_in_manifest( + target: Path, + *, + manifest_id: str = _SCORECARD_MANIFEST_ID, + artifact_id: str = "brigade-work", + route_paths: list[str] | None = None, + route_classes: list[str] | None = None, + scope_globs: list[str] | None = None, + binding_mode: str | None = None, + fixture: dict | None = None, +) -> dict: + _ensure_scorecard_manifest(target, manifest_id=manifest_id) + path = target / "verify" / "manifests" / f"{manifest_id}.json" + payload = verify_manifest._load_manifest_file(path) + assert payload is not None + payload = copy.deepcopy(payload) + subject = payload.setdefault("subject", {}) + subject["artifact_id"] = artifact_id + subject["artifact_kind"] = "skill" + subject["subject_path"] = f"skills/{artifact_id}/SKILL.md" + if route_paths is not None: + payload["route_paths"] = route_paths + if route_classes is not None: + payload["route_classes"] = route_classes + if scope_globs is not None: + payload["scope_globs"] = scope_globs + if binding_mode is not None: + payload["binding_mode"] = binding_mode + if fixture is not None: + payload["fixture"] = fixture + verify_manifest.write_workspace_manifest(target, payload) + _track_workspace_manifest(target, manifest_id) + return payload + + +def _skill_fingerprint(target: Path, artifact_id: str = "brigade-work") -> str: + skill = target / "skills" / artifact_id / "SKILL.md" + return verify_trial.subject_hash_for_path(target, str(skill.relative_to(target))) + + +def _prepare_candidate_skill(scoreable_target: Path, *, scope_globs: list[str] | None = None) -> None: + _write_route_opt_in_manifest(scoreable_target, route_paths=["code"], scope_globs=scope_globs) + _write_skill_subject(scoreable_target) + fingerprint = _skill_fingerprint(scoreable_target) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt( + scoreable_target, + "one-pass", + binding=binding, + commands=[_effectiveness_command()], + ) + + +def _write_promoted_status(target: Path, artifact_id: str, *, with_policy_marker: bool) -> None: + (target / "memory" / "outcome").mkdir(parents=True, exist_ok=True) + entry: dict = {"status": "promoted"} + if with_policy_marker: + entry["route_policy"] = { + "policy_version": scorecard.SCORECARD_POLICY_VERSION, + "route_authority": "full", + } + (target / "memory" / "outcome" / "status.json").write_text(json.dumps({"artifacts": {artifact_id: entry}}) + "\n") + + +def test_classify_band_states(): + assert scorecard.classify_band(None) == "unseen" + assert scorecard.classify_band(_card_with_effectiveness(helped=0, trials=0)) == "unseen" + assert scorecard.classify_band(_card_with_effectiveness(helped=1)) == "candidate" + assert scorecard.classify_band(_card_with_effectiveness(helped=2)) == "provisional" + assert ( + scorecard.classify_band( + _card_with_effectiveness(helped=2), + persisted_status="promoted", + policy_marker=scorecard.SCORECARD_POLICY_VERSION, + ) + == "promoted" + ) + demoted_card = _card_with_effectiveness(helped=2, hurt=1) + assert ( + scorecard.classify_band( + demoted_card, + persisted_status="promoted", + policy_marker=scorecard.SCORECARD_POLICY_VERSION, + ) + == "provisional" + ) + assert scorecard.classify_band(None, persisted_status="promoted") == "unseen" + assert ( + scorecard.classify_band( + _card_with_effectiveness(helped=2), + persisted_status="promoted", + ) + == "provisional" + ) + + +def test_exploration_quota_formula(): + assert route_policy.exploration_quota(0) == 1 + assert route_policy.exploration_quota(1) == 1 + assert route_policy.exploration_quota(10) == 1 + assert route_policy.exploration_quota(20) == 2 + assert route_policy.exploration_quota(100) == 2 + + +def test_unseen_cannot_be_sole_provider(tmp_path): + card = _card_with_effectiveness(artifact_id="new-skill", helped=0, trials=1) + manifest = verify_manifest.VerifyManifest( + manifest_id="new-skill-manifest", + binding_mode="fixture_eval", + artifact_kind="skill", + artifact_id="new-skill", + verifier_id="brigade.verify.fixture", + checks=(), + route_paths=("code",), + subject_path="skills/new-skill/SKILL.md", + path=tmp_path / "verify/manifests/new-skill.json", + ) + entry = route_policy.TrustedOptInEntry( + manifest=manifest, + manifest_path="verify/manifests/new-skill.json", + card=card, + ) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(route_policy, "discover_trusted_opt_in_manifests", lambda *args, **kwargs: ([entry], [])) + decision = route_policy.decide_route_skills( + tmp_path, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert decision.policy_applied + assert decision.assignments == [] + assert any( + entry["artifact_id"] == "new-skill" and entry["reason"] == "unseen_cannot_be_sole_provider" + for entry in decision.accept_reject + ) + + +def test_promoted_skill_gets_priority_and_shadow_unseen(scoreable_target): + _ensure_scorecard_manifest(scoreable_target, manifest_id="proven-manifest") + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + route_paths=["code"], + ) + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="shadow-manifest", + artifact_id="shadow-skill", + route_paths=["code"], + binding_mode="fixture_eval", + fixture={ + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + ) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + _write_skill_for(scoreable_target, "proven-skill") + _write_skill_for(scoreable_target, "shadow-skill") + _write_ineligible_receipt( + scoreable_target, + "shadow-ineligible", + artifact_id="shadow-skill", + manifest_id="shadow-manifest", + ) + proven_fp = _skill_fingerprint(scoreable_target, "proven-skill") + _write_verify_receipt( + scoreable_target, + "proven-pass", + binding=_fixture_binding( + fingerprint=proven_fp, + target=scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + ), + commands=[_effectiveness_command()], + ) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert [item.artifact_id for item in decision.assignments] == ["proven-skill", "shadow-skill"] + assert decision.assignments[0].route_authority == "full" + shadow = decision.assignments[1] + assert shadow.route_authority == "shadow" + assert shadow.exploratory is True + assert shadow.verify_manifest_id == "shadow-manifest" + + +def test_legacy_promoted_status_without_policy_marker_is_rejected(scoreable_target): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + _write_promoted_status(scoreable_target, "brigade-work", with_policy_marker=False) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert not any(item.band == "promoted" for item in decision.assignments) + + +def test_candidate_requires_scope_globs_for_scoped_write(scoreable_target): + _prepare_candidate_skill(scoreable_target, scope_globs=[]) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert not any(item.exploratory for item in decision.assignments) + assert any( + entry["artifact_id"] == "brigade-work" and entry["reason"] == "missing_scope_globs" + for entry in decision.accept_reject + ) + + +def test_candidate_scoped_write_with_manifest_globs(scoreable_target): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**", "tests/**"]) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + exploratory = [item for item in decision.assignments if item.exploratory] + assert len(exploratory) == 1 + assert exploratory[0].scope_globs == ("src/**", "tests/**") + assert exploratory[0].manifest_path is not None + + +def test_at_most_one_exploratory_skill_includes_shadow(scoreable_target): + _ensure_scorecard_manifest(scoreable_target, manifest_id="proven-manifest") + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + route_paths=["code"], + ) + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="candidate-manifest", + artifact_id="candidate-skill", + route_paths=["code"], + scope_globs=["src/**"], + ) + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="shadow-manifest", + artifact_id="shadow-skill", + route_paths=["code"], + binding_mode="fixture_eval", + fixture={ + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + ) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + for artifact_id, manifest_id in (("proven-skill", "proven-manifest"), ("candidate-skill", "candidate-manifest")): + _write_skill_for(scoreable_target, artifact_id) + fingerprint = _skill_fingerprint(scoreable_target, artifact_id) + _write_verify_receipt( + scoreable_target, + f"{artifact_id}-pass", + binding=_fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=manifest_id, + artifact_id=artifact_id, + ), + commands=[_effectiveness_command()], + ) + _write_skill_for(scoreable_target, "shadow-skill") + _write_ineligible_receipt( + scoreable_target, + "shadow-ineligible", + artifact_id="shadow-skill", + manifest_id="shadow-manifest", + ) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + exploratory = [item for item in decision.assignments if item.exploratory] + assert len(exploratory) == 1 + assert exploratory[0].artifact_id == "candidate-skill" + assert not any(item.artifact_id == "shadow-skill" for item in decision.assignments) + + +def test_exploration_quota_blocks_after_cap(tmp_path): + runs_dir = tmp_path / ".brigade" / "runs" + route_class = outcome_cmd.route_fingerprint(route_policy.route_manifest_from_brief(_route_brief())) + assert route_class is not None + _write_prior_decision( + runs_dir, + "prior-1", + route_class=route_class, + assignments=[{"artifact_id": "skill-b", "exploratory": True}], + started_at="2026-07-19T12:00:00+00:00", + ) + card = _card_with_effectiveness(artifact_id="skill-b", helped=1) + manifest = verify_manifest.VerifyManifest( + manifest_id="skill-b-manifest", + binding_mode="patch_backed", + artifact_kind="skill", + artifact_id="skill-b", + verifier_id="brigade.verify.fixture", + checks=(), + route_paths=("code",), + scope_globs=("src/**",), + subject_path="skills/skill-b/SKILL.md", + path=tmp_path / "verify/manifests/skill-b.json", + ) + entry = route_policy.TrustedOptInEntry( + manifest=manifest, + manifest_path="verify/manifests/skill-b.json", + card=card, + ) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(route_policy, "discover_trusted_opt_in_manifests", lambda *args, **kwargs: ([entry], [])) + decision = route_policy.decide_route_skills( + tmp_path, + route_brief=_route_brief(), + runs_dir=runs_dir, + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert decision.quota == 1 + assert decision.exploratory_assignment_count_7d == 1 + assert not any(item.exploratory for item in decision.assignments) + assert any(entry["reason"] == "exploration_quota_exhausted" for entry in decision.accept_reject) + + +def test_hard_ceiling_blocks_skill_for_30_days(tmp_path): + runs_dir = tmp_path / ".brigade" / "runs" + route_class = outcome_cmd.route_fingerprint(route_policy.route_manifest_from_brief(_route_brief())) + assert route_class is not None + for index in range(route_policy.EXPLORATION_HARD_CEILING): + _write_prior_decision( + runs_dir, + f"prior-{index}", + route_class=route_class, + assignments=[{"artifact_id": "skill-a", "exploratory": True}], + started_at=f"2026-06-{20 + index:02d}T12:00:00+00:00", + ) + card_a = _card_with_effectiveness(artifact_id="skill-a", helped=1, wilson=0.9) + card_b = _card_with_effectiveness(artifact_id="skill-b", helped=1, wilson=0.5) + entries = [] + for artifact_id, card in (("skill-a", card_a), ("skill-b", card_b)): + manifest = verify_manifest.VerifyManifest( + manifest_id=f"{artifact_id}-manifest", + binding_mode="patch_backed", + artifact_kind="skill", + artifact_id=artifact_id, + verifier_id="brigade.verify.fixture", + checks=(), + route_paths=("code",), + scope_globs=("src/**",), + subject_path=f"skills/{artifact_id}/SKILL.md", + path=tmp_path / f"verify/manifests/{artifact_id}.json", + ) + entries.append( + route_policy.TrustedOptInEntry( + manifest=manifest, + manifest_path=f"verify/manifests/{artifact_id}.json", + card=card, + ) + ) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(route_policy, "discover_trusted_opt_in_manifests", lambda *args, **kwargs: (entries, [])) + decision = route_policy.decide_route_skills( + tmp_path, + route_brief=_route_brief(), + runs_dir=runs_dir, + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + exploratory = [item for item in decision.assignments if item.exploratory] + assert len(exploratory) == 1 + assert exploratory[0].artifact_id == "skill-b" + assert any( + entry["artifact_id"] == "skill-a" and entry["reason"] == "exploration_hard_ceiling" + for entry in decision.accept_reject + ) + + +def test_routing_budget_blocks_exploratory_only(tmp_path): + card = _card_with_effectiveness(artifact_id="skill-a", helped=1) + manifest = verify_manifest.VerifyManifest( + manifest_id="skill-a-manifest", + binding_mode="patch_backed", + artifact_kind="skill", + artifact_id="skill-a", + verifier_id="brigade.verify.fixture", + checks=(), + route_paths=("code",), + scope_globs=("src/**",), + subject_path="skills/skill-a/SKILL.md", + path=tmp_path / "verify/manifests/skill-a.json", + ) + entry = route_policy.TrustedOptInEntry( + manifest=manifest, + manifest_path="verify/manifests/skill-a.json", + card=card, + ) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(route_policy, "discover_trusted_opt_in_manifests", lambda *args, **kwargs: ([entry], [])) + decision = route_policy.decide_route_skills( + tmp_path, + route_brief=_route_brief(), + budget=route_policy.RouteBudget(token_budget=10, token_spent=10), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert not any(item.exploratory for item in decision.assignments) + assert any(entry["reason"] == "routing_budget_exhausted" for entry in decision.accept_reject) + + +def test_exploration_caps_do_not_mutate_scorecard_receipts(scoreable_target): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + before_cards = scorecard.build_scorecards(scoreable_target) + receipt_paths = list(scorecard.discover_verify_receipt_paths(scoreable_target)) + before_receipts = {str(path): path.read_text() for path in receipt_paths} + for _ in range(5): + route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + after_cards = scorecard.build_scorecards(scoreable_target) + after_receipts = {str(path): path.read_text() for path in receipt_paths} + assert before_receipts == after_receipts + assert before_cards[0].dimensions["effectiveness"] == after_cards[0].dimensions["effectiveness"] + + +def test_route_decision_payload_includes_policy_fields(scoreable_target, tmp_path): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + _write_promoted_status(scoreable_target, "brigade-work", with_policy_marker=True) + runs_dir = tmp_path / "runs" + output_dir = runs_dir / "current" + output_dir.mkdir(parents=True) + receipt = _run_receipt() + (output_dir / "run.json").write_text(json.dumps(receipt) + "\n") + from brigade import roster as roster_mod + + roster = roster_mod.Roster( + orchestrator="chef", + agents={"chef": roster_mod.Agent("chef", "codex", "plan"), "coder": roster_mod.Agent("coder", "codex", "code")}, + max_workers=1, + allow_models=("codex",), + ) + decision = route_policy.decide_route_skills(scoreable_target, route_brief=_route_brief()) + extensions = route_policy.route_policy_extensions_from_decision(decision) + write_route_decision(output_dir, roster, policy_extensions=extensions) + saved = json.loads((output_dir / "route-decision.json").read_text()) + assert saved["policy_version"] == route_policy.ROUTE_POLICY_VERSION + assert "decided_at" in saved + assert "exploration" in saved + assert "quota" in saved["exploration"] + assert "accept_reject" in saved["exploration"] + + +def test_route_decision_payload_omits_policy_without_opt_in(tmp_path): + from brigade import roster as roster_mod + + roster = roster_mod.Roster( + orchestrator="chef", + agents={"chef": roster_mod.Agent("chef", "codex", "plan")}, + max_workers=1, + allow_models=("codex",), + ) + payload = route_decision_payload(_run_receipt(), roster, target=tmp_path) + assert "policy_version" not in payload + assert "score_inputs" not in payload + + +def test_pre_run_decision_is_preserved_by_write_route_decision(scoreable_target, tmp_path): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + runs_dir = tmp_path / "runs" + output_dir = runs_dir / "current" + output_dir.mkdir(parents=True) + receipt = _run_receipt() + (output_dir / "run.json").write_text(json.dumps(receipt) + "\n") + from brigade import roster as roster_mod + + roster = roster_mod.Roster( + orchestrator="chef", + agents={"chef": roster_mod.Agent("chef", "codex", "plan")}, + max_workers=1, + allow_models=("codex",), + ) + decision = route_policy.decide_route_skills(scoreable_target, route_brief=_route_brief()) + extensions = route_policy.route_policy_extensions_from_decision(decision) + write_route_decision(output_dir, roster, policy_extensions=extensions) + before = json.loads((output_dir / "route-decision.json").read_text()) + write_route_decision(output_dir, roster, target=scoreable_target, runs_dir=runs_dir) + after = json.loads((output_dir / "route-decision.json").read_text()) + assert after["decided_at"] == before["decided_at"] + assert after["skill_assignments"] == before["skill_assignments"] + assert after["exploration"] == before["exploration"] + + +def test_planner_prompt_receives_selected_policy(monkeypatch, scoreable_target): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + prompts: list[str] = [] + + def fake_run_agent(cli_ref, prompt, **kwargs): + prompts.append(prompt) + return aboyeur.agents.AgentResult( + text=json.dumps( + { + "assignments": [ + { + "worker": "coder", + "task": "implement helper in src", + "covers": ["implement", "code"], + "selected_skill_ids": ["brigade-work"], + } + ] + } + ), + ok=True, + ) + + monkeypatch.setattr(aboyeur.agents, "run_agent", fake_run_agent) + route = _route_brief() + skill_policy = route_policy.decide_route_skills(scoreable_target, route_brief=route) + aboyeur.plan( + "build feature", + _plan_mode_roster("codex"), + route=route, + skill_policy=skill_policy, + ) + assert "Skill route policy" in prompts[0] + assert "scoped write only within globs" in prompts[0] + + +def test_no_opt_in_keeps_plan_prompt_unchanged(monkeypatch, tmp_path): + prompts: list[str] = [] + + def fake_run_agent(cli_ref, prompt, **kwargs): + prompts.append(prompt) + return aboyeur.agents.AgentResult(text=json.dumps({"assignments": []}), ok=True) + + monkeypatch.setattr(aboyeur.agents, "run_agent", fake_run_agent) + route = _route_brief() + skill_policy = route_policy.decide_route_skills(tmp_path, route_brief=route) + assert skill_policy.policy_applied is False + aboyeur.plan( + "build feature", + _plan_mode_roster("codex"), + route=route, + skill_policy=skill_policy, + ) + assert "Skill route policy" not in prompts[0] + + +def test_shadow_verify_directive_in_planner_prompt(monkeypatch, scoreable_target): + _ensure_scorecard_manifest(scoreable_target, manifest_id="proven-manifest") + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + route_paths=["code"], + ) + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="shadow-manifest", + artifact_id="shadow-skill", + route_paths=["code"], + binding_mode="fixture_eval", + fixture={ + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + ) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + _write_skill_for(scoreable_target, "proven-skill") + _write_skill_for(scoreable_target, "shadow-skill") + _write_ineligible_receipt( + scoreable_target, + "shadow-ineligible", + artifact_id="shadow-skill", + manifest_id="shadow-manifest", + ) + proven_fp = _skill_fingerprint(scoreable_target, "proven-skill") + _write_verify_receipt( + scoreable_target, + "proven-pass", + binding=_fixture_binding( + fingerprint=proven_fp, + target=scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + ), + commands=[_effectiveness_command()], + ) + prompts: list[str] = [] + + def fake_run_agent(cli_ref, prompt, **kwargs): + prompts.append(prompt) + return aboyeur.agents.AgentResult( + text=json.dumps( + { + "assignments": [ + { + "stage": 1, + "worker": "coder", + "task": "implement helper", + "covers": ["implement", "code"], + }, + { + "stage": 2, + "worker": "coder", + "task": "brigade work verify run --target . --manifest shadow-manifest", + "covers": ["verify"], + "selected_skill_ids": ["shadow-skill"], + }, + ] + } + ), + ok=True, + ) + + monkeypatch.setattr(aboyeur.agents, "run_agent", fake_run_agent) + route = _route_brief() + skill_policy = route_policy.decide_route_skills(scoreable_target, route_brief=route) + aboyeur.plan("build feature", _plan_mode_roster("codex"), route=route, skill_policy=skill_policy) + assert "verify-stage assignment" in prompts[0] or "verify-covering assignment" in prompts[0] + assert "brigade work verify run --target . --manifest shadow-manifest" in prompts[0] + + +def test_scope_globs_reject_whole_repo_wildcards(): + assert route_policy.validate_scope_globs(("**",)) == "invalid_scope_glob" + assert route_policy.validate_scope_globs(("**/*",)) == "invalid_scope_glob" + assert route_policy.validate_scope_globs(("../secrets",)) == "invalid_scope_glob" + assert route_policy.validate_scope_globs(("src/**",)) is None + + +def test_candidate_rejects_invalid_scope_globs(scoreable_target): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + path = scoreable_target / "verify" / "manifests" / f"{_SCORECARD_MANIFEST_ID}.json" + payload = verify_manifest._load_manifest_file(path) + assert payload is not None + payload = copy.deepcopy(payload) + payload["scope_globs"] = ["**"] + path.write_text(json.dumps(payload, indent=2) + "\n") + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert not any(item.exploratory for item in decision.assignments) + assert any( + entry["artifact_id"] == "brigade-work" and entry["reason"] == "invalid_scope_glob" + for entry in decision.accept_reject + ) + + +def test_shadow_exploration_quota_blocks_with_proven_route(scoreable_target): + _ensure_scorecard_manifest(scoreable_target, manifest_id="proven-manifest") + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + route_paths=["code"], + ) + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="shadow-manifest", + artifact_id="shadow-skill", + route_paths=["code"], + binding_mode="fixture_eval", + fixture={ + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + ) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + _write_skill_for(scoreable_target, "proven-skill") + _write_skill_for(scoreable_target, "shadow-skill") + proven_fp = _skill_fingerprint(scoreable_target, "proven-skill") + _write_verify_receipt( + scoreable_target, + "proven-pass", + binding=_fixture_binding( + fingerprint=proven_fp, + target=scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + ), + commands=[_effectiveness_command()], + ) + runs_dir = scoreable_target / ".brigade" / "runs" + route_class = outcome_cmd.route_fingerprint(route_policy.route_manifest_from_brief(_route_brief())) + assert route_class is not None + _write_prior_decision( + runs_dir, + "prior-shadow", + route_class=route_class, + assignments=[{"artifact_id": "other-shadow", "exploratory": True}], + started_at="2026-07-19T12:00:00+00:00", + ) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + runs_dir=runs_dir, + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert [item.artifact_id for item in decision.assignments] == ["proven-skill"] + assert not any(item.artifact_id == "shadow-skill" for item in decision.assignments) + assert any( + entry["artifact_id"] == "shadow-skill" and entry["reason"] == "exploration_quota_exhausted" + for entry in decision.accept_reject + ) + + +def test_zero_receipt_cold_start_shadow(scoreable_target): + _ensure_scorecard_manifest(scoreable_target, manifest_id="proven-manifest") + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + route_paths=["code"], + ) + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="shadow-manifest", + artifact_id="shadow-skill", + route_paths=["code"], + binding_mode="fixture_eval", + fixture={ + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + ) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + _write_skill_for(scoreable_target, "proven-skill") + _write_skill_for(scoreable_target, "shadow-skill") + proven_fp = _skill_fingerprint(scoreable_target, "proven-skill") + _write_verify_receipt( + scoreable_target, + "proven-pass", + binding=_fixture_binding( + fingerprint=proven_fp, + target=scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + ), + commands=[_effectiveness_command()], + ) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert [item.artifact_id for item in decision.assignments] == ["proven-skill", "shadow-skill"] + shadow = decision.assignments[1] + assert shadow.route_authority == "shadow" + assert shadow.exploratory is True + + +def test_ambiguous_route_opt_in_manifests_fail_closed(scoreable_target): + for manifest_id in ("proven-manifest", "proven-manifest-2"): + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": manifest_id, + "binding_mode": "patch_backed", + "verifier_id": "brigade.verify.test", + "patch_source": "worktree", + "route_paths": ["code"], + "subject": { + "artifact_kind": "skill", + "artifact_id": "proven-skill", + "subject_path": "skills/proven-skill/SKILL.md", + }, + "checks": [ + { + "check_id": "verify.echo-ok", + "check_role": "effectiveness", + "command": f"{__import__('sys').executable} -c \"print('ok')\"", + } + ], + } + verify_manifest.write_workspace_manifest(scoreable_target, payload) + _track_workspace_manifest(scoreable_target, manifest_id) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + _write_skill_for(scoreable_target, "proven-skill") + proven_fp = _skill_fingerprint(scoreable_target, "proven-skill") + _write_verify_receipt( + scoreable_target, + "proven-pass", + binding=_fixture_binding( + fingerprint=proven_fp, + target=scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + ), + commands=[_effectiveness_command()], + ) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert not decision.assignments + assert not any(item.band == "promoted" for item in decision.assignments) + assert any( + entry["artifact_id"] == "proven-skill" and entry["reason"] == "ambiguous_production_manifest" + for entry in decision.accept_reject + ) + + +def test_promoted_with_production_and_fixture_yields_one_assignment(scoreable_target): + path = scoreable_target / "verify" / "manifests" / "proven-manifest.json" + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": "proven-manifest", + "binding_mode": "patch_backed", + "verifier_id": "brigade.verify.test", + "patch_source": "worktree", + "route_paths": ["code"], + "subject": { + "artifact_kind": "skill", + "artifact_id": "proven-skill", + "subject_path": "skills/proven-skill/SKILL.md", + }, + "checks": [ + { + "check_id": "verify.echo-ok", + "check_role": "effectiveness", + "command": f"{__import__('sys').executable} -c \"print('ok')\"", + } + ], + } + verify_manifest.write_workspace_manifest(scoreable_target, payload) + _track_workspace_manifest(scoreable_target, "proven-manifest") + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="shadow-manifest", + artifact_id="proven-skill", + route_paths=["code"], + binding_mode="fixture_eval", + fixture={ + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + ) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + _write_skill_for(scoreable_target, "proven-skill") + proven_fp = _skill_fingerprint(scoreable_target, "proven-skill") + proven_manifest, _ = verify_manifest.resolve_manifest(scoreable_target, "proven-manifest") + assert proven_manifest is not None + _write_verify_receipt( + scoreable_target, + "proven-pass", + binding={ + "binding_mode": "patch_backed", + "artifact_kind": "skill", + "artifact_id": "proven-skill", + "content_fingerprint": proven_fp, + "manifest_binding": verify_manifest.build_manifest_binding(scoreable_target, proven_manifest), + "patch_binding": { + "subject_path": "skills/proven-skill/SKILL.md", + "patch_source": "worktree", + }, + "verifier_identity": {"verifier_id": "brigade.verify.test", "session_id": "verifier-1"}, + }, + commands=[_effectiveness_command()], + ) + decision = route_policy.decide_route_skills( + scoreable_target, + route_brief=_route_brief(), + now=datetime(2026, 7, 20, tzinfo=timezone.utc), + ) + assert [item.artifact_id for item in decision.assignments] == ["proven-skill"] + assert decision.assignments[0].manifest_id == "proven-manifest" + + +def test_direct_worker_binds_scoped_candidate_skill(monkeypatch, scoreable_target, tmp_path): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + output_dir = tmp_path / "run-dir" + output_dir.mkdir() + + def fake_dispatch(assignments, roster, **kwargs): + build_prompt = kwargs["build_prompt"] + prompt = build_prompt(roster.agents["coder"], assignments[0], direct=True) + assert "Scoped-write constraint" in prompt + return [aboyeur.WorkerResult(worker="coder", task="implement helper in src", text="done", ok=True)] + + monkeypatch.setattr(aboyeur, "dispatch", fake_dispatch) + rc = aboyeur.run( + "implement helper in src", + _plan_mode_roster("codex"), + cwd=scoreable_target, + output_dir=output_dir, + worker="coder", + evidence_enabled=False, + code_graph_enabled=False, + ) + assert rc == 0 + plan_doc = json.loads((output_dir / "plan.json").read_text()) + assert plan_doc["assignments"][0]["selected_skill_ids"] == ["brigade-work"] + route_decision = json.loads((output_dir / "route-decision.json").read_text()) + exploratory = [item for item in route_decision["skill_assignments"] if item.get("exploratory")] + assert len(exploratory) == 1 + assert exploratory[0]["artifact_id"] == "brigade-work" + + +def test_direct_worker_rejects_shadow_without_quota_consumption(monkeypatch, scoreable_target, tmp_path): + _ensure_scorecard_manifest(scoreable_target, manifest_id="proven-manifest") + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + route_paths=["code"], + ) + _write_route_opt_in_manifest( + scoreable_target, + manifest_id="shadow-manifest", + artifact_id="shadow-skill", + route_paths=["code"], + binding_mode="fixture_eval", + fixture={ + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + ) + _write_promoted_status(scoreable_target, "proven-skill", with_policy_marker=True) + _write_skill_for(scoreable_target, "proven-skill") + _write_skill_for(scoreable_target, "shadow-skill") + proven_fp = _skill_fingerprint(scoreable_target, "proven-skill") + _write_verify_receipt( + scoreable_target, + "proven-pass", + binding=_fixture_binding( + fingerprint=proven_fp, + target=scoreable_target, + manifest_id="proven-manifest", + artifact_id="proven-skill", + ), + commands=[_effectiveness_command()], + ) + output_dir = tmp_path / "run-dir" + output_dir.mkdir() + + def fake_run_agent(cli_ref, prompt, **kwargs): + return aboyeur.agents.AgentResult(text="done", ok=True) + + def fake_dispatch(*args, **kwargs): + return [aboyeur.WorkerResult(worker="coder", task="implement helper", text="done", ok=True)] + + monkeypatch.setattr(aboyeur.agents, "run_agent", fake_run_agent) + monkeypatch.setattr(aboyeur, "dispatch", fake_dispatch) + rc = aboyeur.run( + "implement helper", + _plan_mode_roster("codex"), + cwd=scoreable_target, + output_dir=output_dir, + worker="coder", + evidence_enabled=False, + code_graph_enabled=False, + ) + assert rc == 0 + route_decision = json.loads((output_dir / "route-decision.json").read_text()) + assert [item["artifact_id"] for item in route_decision["skill_assignments"]] == ["proven-skill"] + assert not any(item.get("exploratory") for item in route_decision["skill_assignments"]) + assert any( + entry["artifact_id"] == "shadow-skill" and entry["reason"] == route_policy.DIRECT_WORKER_REJECTS_SHADOW + for entry in route_decision["exploration"]["accept_reject"] + ) + plan_doc = json.loads((output_dir / "plan.json").read_text()) + assert "selected_skill_ids" not in plan_doc["assignments"][0] + + +def test_run_receipt_preserves_skill_route_policy_through_status_writes(monkeypatch, scoreable_target, tmp_path): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + output_dir = tmp_path / "run-dir" + output_dir.mkdir() + + def fake_run_agent(cli_ref, prompt, **kwargs): + return aboyeur.agents.AgentResult( + text=json.dumps( + { + "assignments": [ + { + "worker": "coder", + "task": "implement helper in src", + "covers": ["implement", "code"], + "selected_skill_ids": ["brigade-work"], + } + ] + } + ), + ok=True, + ) + + def fake_dispatch(*args, **kwargs): + return [ + aboyeur.WorkerResult( + worker="coder", + task="implement helper in src", + text="done", + ok=True, + ) + ] + + monkeypatch.setattr(aboyeur.agents, "run_agent", fake_run_agent) + monkeypatch.setattr(aboyeur, "dispatch", fake_dispatch) + rc = aboyeur.run( + "implement helper", + _plan_mode_roster("codex"), + cwd=scoreable_target, + output_dir=output_dir, + evidence_enabled=False, + code_graph_enabled=False, + ) + assert rc == 0 + run_receipt = json.loads((output_dir / "run.json").read_text()) + route_decision = json.loads((output_dir / "route-decision.json").read_text()) + assert "skill_route_policy" in run_receipt + assert run_receipt["skill_route_policy"]["policy_version"] == route_policy.ROUTE_POLICY_VERSION + assert run_receipt["skill_route_policy"]["exploration"] == route_decision["exploration"] + assert run_receipt["skill_route_policy"]["skill_assignments"] == route_decision["skill_assignments"] + plan_doc = json.loads((output_dir / "plan.json").read_text()) + assert plan_doc["assignments"][0]["selected_skill_ids"] == ["brigade-work"] + + +def test_worker_prompt_applies_scope_only_on_bound_assignment(monkeypatch, scoreable_target): + _prepare_candidate_skill(scoreable_target, scope_globs=["src/**"]) + route = _route_brief() + skill_policy = route_policy.decide_route_skills(scoreable_target, route_brief=route) + bound = aboyeur.Assignment( + worker="coder", + task="edit src/foo.py", + covers=("implement",), + selected_skill_ids=("brigade-work",), + ) + unbound = aboyeur.Assignment(worker="coder", task="run tests", covers=("verify",)) + assert "Scoped-write constraint" in aboyeur._worker_prompt( + _plan_mode_roster("codex").agents["coder"], + bound, + skill_policy=skill_policy, + ) + assert "Scoped-write constraint" not in aboyeur._worker_prompt( + _plan_mode_roster("codex").agents["coder"], + unbound, + skill_policy=skill_policy, + ) diff --git a/tests/test_scorecard.py b/tests/test_scorecard.py new file mode 100644 index 00000000..31612117 --- /dev/null +++ b/tests/test_scorecard.py @@ -0,0 +1,753 @@ +"""Receipt-only skill scorecard projection (#572).""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from brigade import cli, localio, outcome, outcome_cmd, scorecard, verify_manifest, verify_trial + +from tests.test_verify_trial import ( + _track_workspace_manifest, + _write_fixture_manifest, + _write_skill_subject, +) + +pytest_plugins = ["tests.test_verify_trial"] + +_SCORECARD_MANIFEST_ID = "scorecard-skill-fixture" +_SCORECARD_PASSING_MANIFEST_ID = "scorecard-skill-fixture-passing" +_SCORECARD_ALT_CHECK_MANIFEST_ID = "scorecard-skill-fixture-alt" +_SCORECARD_UTILITY_MANIFEST_ID = "scorecard-skill-fixture-utility" +_SCORECARD_UTILITY_PASSING_MANIFEST_ID = "scorecard-skill-fixture-utility-passing" + + +def _write_scorecard_manifest(target: Path, *, manifest_id: str, case_id: str, check_id: str) -> dict: + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": manifest_id, + "binding_mode": "fixture_eval", + "verifier_id": "brigade.verify.fixture", + "subject": { + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "subject_path": "skills/brigade-work/SKILL.md", + }, + "fixture": { + "manifest_id": "adversarial-failed-verification", + "case_id": case_id, + "check_id": check_id, + }, + "checks": [ + { + "check_id": check_id, + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('ok')\"", + } + ], + } + verify_manifest.write_workspace_manifest(target, payload) + _track_workspace_manifest(target, manifest_id) + return payload + + +def _ensure_scorecard_manifest(target: Path, *, manifest_id: str = _SCORECARD_MANIFEST_ID) -> dict: + if manifest_id not in verify_manifest.registered_manifest_ids(target): + if manifest_id == _SCORECARD_PASSING_MANIFEST_ID: + _write_scorecard_manifest( + target, + manifest_id=manifest_id, + case_id="report-passing-tests", + check_id="fixture.echo-ok", + ) + elif manifest_id == _SCORECARD_ALT_CHECK_MANIFEST_ID: + _write_scorecard_manifest( + target, + manifest_id=manifest_id, + case_id="report-failing-tests", + check_id="fixture.echo-alt", + ) + elif manifest_id in {_SCORECARD_UTILITY_MANIFEST_ID, _SCORECARD_UTILITY_PASSING_MANIFEST_ID}: + case_id = ( + "report-passing-tests" + if manifest_id == _SCORECARD_UTILITY_PASSING_MANIFEST_ID + else "report-failing-tests" + ) + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": manifest_id, + "binding_mode": "fixture_eval", + "verifier_id": "brigade.verify.fixture", + "subject": { + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "subject_path": "skills/brigade-work/SKILL.md", + }, + "fixture": { + "manifest_id": "adversarial-failed-verification", + "case_id": case_id, + "check_id": "fixture.echo-ok", + }, + "required_utility_check_ids": ["guardrail.tests-green", "guardrail.lint-clean"], + "checks": [ + { + "check_id": "fixture.echo-ok", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + { + "check_id": "guardrail.tests-green", + "check_role": "utility_guardrail", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + { + "check_id": "guardrail.lint-clean", + "check_role": "utility_guardrail", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + ], + } + verify_manifest.write_workspace_manifest(target, payload) + _track_workspace_manifest(target, manifest_id) + else: + _write_fixture_manifest(target, manifest_id=manifest_id) + manifest, _error = verify_manifest.resolve_manifest(target, manifest_id) + assert manifest is not None + return verify_manifest.build_manifest_binding(target, manifest) + + +def _fixture_binding( + *, + artifact_id: str = "brigade-work", + fingerprint: str | None = None, + target: Path, + manifest_id: str = _SCORECARD_MANIFEST_ID, + case_id: str = "report-failing-tests", + check_id: str = "fixture.echo-ok", +) -> dict: + manifest_binding = _ensure_scorecard_manifest(target, manifest_id=manifest_id) + return { + "binding_mode": "fixture_eval", + "artifact_kind": "skill", + "artifact_id": artifact_id, + "content_fingerprint": fingerprint or "sha256:" + "a" * 64, + "fixture_binding": { + "manifest_id": "adversarial-failed-verification", + "case_id": case_id, + "check_id": check_id, + }, + "manifest_binding": manifest_binding, + "verifier_identity": {"verifier_id": "brigade.verify.fixture", "session_id": "verifier-1"}, + } + + +def _write_verify_receipt( + target, + run_id: str, + *, + binding: dict | None = None, + status: str = "completed", + commands: list[dict] | None = None, + started_at: str = "2026-06-20T00:00:00+00:00", + duration_seconds: float = 10.0, + reused_from: str | None = None, + required_utility_check_ids: list[str] | None = None, + extra: dict | None = None, +) -> str: + run_dir = target / ".brigade" / "work" / "verify-runs" / run_id + run_dir.mkdir(parents=True, exist_ok=True) + receipt = { + "schema_version": 2, + "run_id": run_id, + "target": str(target), + "status": status, + "started_at": started_at, + "completed_at": started_at, + "duration_seconds": duration_seconds, + "path": str(run_dir), + "commands": commands or [], + } + if binding is not None: + receipt["subject_binding"] = binding + manifest_binding = binding.get("manifest_binding") + if isinstance(manifest_binding, dict) and isinstance(manifest_binding.get("manifest_id"), str): + receipt["verify_manifest_id"] = manifest_binding["manifest_id"] + if reused_from is not None: + receipt["reused_from"] = reused_from + if required_utility_check_ids is not None: + receipt["required_utility_check_ids"] = required_utility_check_ids + if extra: + receipt.update(extra) + verify_trial.stamp_verify_receipt_failure_taxonomy(receipt) + for command in receipt["commands"]: + if isinstance(command, dict): + verify_trial.stamp_verify_command_failure_taxonomy(command) + receipt["digests"] = { + "algorithm": "sha256", + "logs": {}, + "receipt_sha256": localio.canonical_json_digest(receipt, exclude_keys={"digests"}), + } + localio.write_json(run_dir / "receipt.json", receipt) + return str(run_dir / "receipt.json") + + +def _effectiveness_command( + *, + ok: bool = True, + check_id: str = "fixture.echo-ok", + duration_seconds: float | None = None, +) -> dict: + command = { + "command": f"{sys.executable} -c \"print('ok')\"", + "check_role": "effectiveness", + "check_id": check_id, + "status": "completed" if ok else "failed", + "exit_code": 0 if ok else 1, + } + if duration_seconds is not None: + command["duration_seconds"] = duration_seconds + verify_trial.stamp_verify_command_failure_taxonomy(command) + return command + + +def _utility_command( + *, + ok: bool = True, + check_id: str = "guardrail.tests-green", + duration_seconds: float | None = None, +) -> dict: + command = { + "command": f"{sys.executable} -c \"print('ok')\"", + "check_role": "utility_guardrail", + "check_id": check_id, + "status": "completed" if ok else "failed", + "exit_code": 0 if ok else 1, + } + if duration_seconds is not None: + command["duration_seconds"] = duration_seconds + verify_trial.stamp_verify_command_failure_taxonomy(command) + return command + + +def test_build_scorecard_projects_all_dimensions(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding_a = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + binding_b = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_PASSING_MANIFEST_ID, + case_id="report-passing-tests", + ) + _write_verify_receipt( + scoreable_target, + "run-a", + binding=binding_a, + commands=[_effectiveness_command()], + duration_seconds=12.0, + ) + _write_verify_receipt( + scoreable_target, + "run-b", + binding=binding_b, + commands=[_effectiveness_command()], + started_at="2026-06-20T01:00:00+00:00", + duration_seconds=20.0, + ) + + cards = scorecard.build_scorecards(scoreable_target) + assert len(cards) == 1 + card = cards[0] + assert card.subject.artifact_id == "brigade-work" + assert card.dimensions["effectiveness"]["helped"] == 2 + assert card.dimensions["effectiveness"]["hurt"] == 0 + assert card.dimensions["effectiveness"]["trials"] == 2 + assert card.dimensions["verifier_cost"]["median_s"] == 16.0 + assert card.dimensions["verifier_cost"]["receipt_median_s"] == 16.0 + assert card.dimensions["verifier_cost"]["receipt_samples"] == 2 + assert card.dimensions["evidence_integrity"]["eligible"] == 2 + assert card.dimensions["evidence_integrity"]["audit"] == 2 + + +def test_utility_only_failure_does_not_hurt_effectiveness(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_UTILITY_MANIFEST_ID, + ) + _write_verify_receipt( + scoreable_target, + "utility-fail", + binding=binding, + status="failed", + commands=[ + _effectiveness_command(), + _utility_command(ok=True, check_id="guardrail.tests-green"), + _utility_command(ok=False, check_id="guardrail.lint-clean"), + ], + required_utility_check_ids=["guardrail.tests-green", "guardrail.lint-clean"], + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["helped"] == 1 + assert card.dimensions["effectiveness"]["hurt"] == 0 + + +def test_reused_and_duplicate_evidence_units_do_not_inflate_effectiveness(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + commands = [_effectiveness_command()] + _write_verify_receipt(scoreable_target, "run-1", binding=binding, commands=commands) + _write_verify_receipt( + scoreable_target, + "run-2", + binding=binding, + commands=commands, + started_at="2026-06-20T01:00:00+00:00", + ) + _write_verify_receipt( + scoreable_target, + "run-3", + binding=binding, + commands=commands, + started_at="2026-06-20T02:00:00+00:00", + reused_from="run-1", + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["trials"] == 1 + assert card.dimensions["effectiveness"]["helped"] == 1 + + +def test_ineligible_only_subject_stays_visible_with_reason_counts(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt( + scoreable_target, + "missing-role", + binding=binding, + commands=[{"status": "completed", "exit_code": 0, "check_id": "fixture.echo-ok"}], + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["trials"] == 0 + assert card.ineligible_summary["missing_check_role"] == 1 + + +def test_rank_json_emits_scorecard_dimensions(scoreable_target, capsys): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt(scoreable_target, "run-a", binding=binding, commands=[_effectiveness_command()]) + + assert outcome_cmd.rank(target=scoreable_target, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + entry = payload["ranking"][0] + assert entry["artifact_id"] == "brigade-work" + assert entry["policy_version"] == scorecard.SCORECARD_POLICY_VERSION + assert "effectiveness" in entry["dimensions"] + assert "utility_guardrails" in entry + + +def test_explain_json_includes_receipt_trail_keyed_by_subject_binding(scoreable_target, capsys): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt(scoreable_target, "run-a", binding=binding, commands=[_effectiveness_command()]) + + assert outcome_cmd.explain(target=scoreable_target, artifact_id="brigade-work", json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + assert "scorecard" in payload + trail = payload["scorecard"]["receipt_trail"] + assert len(trail) == 1 + assert trail[0]["subject_binding"]["artifact_id"] == "brigade-work" + assert trail[0]["eligible"] is True + + +def test_poisoned_non_verifier_inputs_do_not_change_dimensions(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt(scoreable_target, "run-a", binding=binding, commands=[_effectiveness_command()]) + + baseline = scorecard.subject_scorecard_to_dict(scorecard.scorecard_for_artifact(scoreable_target, "brigade-work")) + + # Ledger rows with a different caller artifact id must not affect scorecard values. + outcome_cmd.append_records( + scoreable_target, + [ + outcome.OutcomeRecord( + "poison-skill", + "skill", + "t1", + "verify", + 1, + "fake", + "2026-06-20T00:00:00+00:00", + ) + ], + ) + run_dir = scoreable_target / ".brigade" / "runs" / "poison-run" + run_dir.mkdir(parents=True) + localio.write_json( + run_dir / "run.json", + { + "schema": "brigade.run.v1", + "schema_version": 1, + "status": "ok", + "suspected_noop": False, + "context_eval": {"brief_hit_rate": 1.0}, + "duration_seconds": 9999, + }, + ) + closeout_dir = scoreable_target / ".brigade" / "work" / "closeouts" / "poison-closeout" + closeout_dir.mkdir(parents=True) + localio.write_json(closeout_dir / "closeout.json", {"status": "ready", "ready": True}) + + after = scorecard.subject_scorecard_to_dict(scorecard.scorecard_for_artifact(scoreable_target, "brigade-work")) + assert after == baseline + + +def test_cli_outcome_explain_dispatch(scoreable_target, capsys): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + _write_verify_receipt(scoreable_target, "run-a", binding=binding, commands=[_effectiveness_command()]) + + assert cli.main(["outcome", "explain", "brigade-work", "--target", str(scoreable_target), "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["scorecard"]["subject"]["artifact_id"] == "brigade-work" + + +def test_different_check_ids_with_same_command_do_not_dedupe(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding_a = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + binding_b = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_ALT_CHECK_MANIFEST_ID, + check_id="fixture.echo-alt", + ) + shared_command = f"{sys.executable} -c \"print('ok')\"" + cmd_a = { + "command": shared_command, + "check_role": "effectiveness", + "check_id": "fixture.echo-ok", + "status": "completed", + "exit_code": 0, + } + cmd_b = { + "command": shared_command, + "check_role": "effectiveness", + "check_id": "fixture.echo-alt", + "status": "completed", + "exit_code": 0, + } + verify_trial.stamp_verify_command_failure_taxonomy(cmd_a) + verify_trial.stamp_verify_command_failure_taxonomy(cmd_b) + _write_verify_receipt(scoreable_target, "check-a", binding=binding_a, commands=[cmd_a]) + _write_verify_receipt( + scoreable_target, + "check-b", + binding=binding_b, + commands=[cmd_b], + started_at="2026-06-20T01:00:00+00:00", + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["trials"] == 2 + assert card.dimensions["effectiveness"]["helped"] == 2 + + +def test_same_check_retry_sequence_records_transitions_and_flips(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + commands_fail = [_effectiveness_command(ok=False)] + commands_pass = [_effectiveness_command(ok=True)] + _write_verify_receipt(scoreable_target, "retry-1", binding=binding, commands=commands_fail) + _write_verify_receipt( + scoreable_target, + "retry-2", + binding=binding, + commands=commands_pass, + started_at="2026-06-20T01:00:00+00:00", + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + stability = card.dimensions["retry_stability"] + assert stability["sequences"] == 1 + assert stability["fail_to_pass"] == 1 + assert stability["flips"] == 1 + assert stability["first_pass_yield"] == 0.0 + + +def test_digest_missing_is_ineligible_not_hurt(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + run_dir = scoreable_target / ".brigade" / "work" / "verify-runs" / "digest-missing" + run_dir.mkdir(parents=True, exist_ok=True) + receipt = { + "schema_version": 2, + "run_id": "digest-missing", + "target": str(scoreable_target), + "status": "completed", + "started_at": "2026-06-20T00:00:00+00:00", + "completed_at": "2026-06-20T00:00:00+00:00", + "duration_seconds": 10.0, + "path": str(run_dir), + "commands": [_effectiveness_command()], + "verify_manifest_id": _SCORECARD_MANIFEST_ID, + "subject_binding": binding, + } + verify_trial.stamp_verify_receipt_failure_taxonomy(receipt) + localio.write_json(run_dir / "receipt.json", receipt) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["hurt"] == 0 + assert card.dimensions["effectiveness"]["trials"] == 0 + assert card.ineligible_summary["receipt_digest_missing"] == 1 + + +def test_digest_mismatch_is_ineligible_not_hurt(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + run_dir = scoreable_target / ".brigade" / "work" / "verify-runs" / "digest-bad" + run_dir.mkdir(parents=True, exist_ok=True) + receipt = { + "schema_version": 2, + "run_id": "digest-bad", + "target": str(scoreable_target), + "status": "completed", + "started_at": "2026-06-20T00:00:00+00:00", + "completed_at": "2026-06-20T00:00:00+00:00", + "duration_seconds": 10.0, + "path": str(run_dir), + "commands": [_effectiveness_command()], + "verify_manifest_id": _SCORECARD_MANIFEST_ID, + "subject_binding": binding, + "digests": { + "algorithm": "sha256", + "logs": {}, + "receipt_sha256": "c" * 64, + }, + } + verify_trial.stamp_verify_receipt_failure_taxonomy(receipt) + localio.write_json(run_dir / "receipt.json", receipt) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["hurt"] == 0 + assert card.dimensions["effectiveness"]["trials"] == 0 + assert card.ineligible_summary["receipt_digest_mismatch"] == 1 + + +def test_verifier_cost_includes_command_duration_stats(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding_a = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + binding_b = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_PASSING_MANIFEST_ID, + case_id="report-passing-tests", + ) + _write_verify_receipt( + scoreable_target, + "cmd-cost-a", + binding=binding_a, + commands=[_effectiveness_command(duration_seconds=4.0)], + duration_seconds=12.0, + ) + _write_verify_receipt( + scoreable_target, + "cmd-cost-b", + binding=binding_b, + commands=[_effectiveness_command(duration_seconds=8.0)], + started_at="2026-06-20T01:00:00+00:00", + duration_seconds=20.0, + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + cost = card.dimensions["verifier_cost"] + assert cost["receipt_median_s"] == 16.0 + assert cost["receipt_samples"] == 2 + assert cost["command_median_s"] == 6.0 + assert cost["command_samples"] == 2 + assert cost["median_s"] == cost["receipt_median_s"] + + +def test_utility_guardrails_track_per_check_passing_and_failing_units(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding_a = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_UTILITY_MANIFEST_ID, + ) + binding_b = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_UTILITY_PASSING_MANIFEST_ID, + case_id="report-passing-tests", + ) + _write_verify_receipt( + scoreable_target, + "utility-pass", + binding=binding_a, + commands=[ + _effectiveness_command(), + _utility_command(ok=True, check_id="guardrail.tests-green"), + _utility_command(ok=True, check_id="guardrail.lint-clean"), + ], + required_utility_check_ids=["guardrail.tests-green", "guardrail.lint-clean"], + ) + _write_verify_receipt( + scoreable_target, + "utility-fail", + binding=binding_b, + commands=[ + _effectiveness_command(), + _utility_command(ok=True, check_id="guardrail.tests-green"), + _utility_command(ok=False, check_id="guardrail.lint-clean"), + ], + status="failed", + started_at="2026-06-20T01:00:00+00:00", + required_utility_check_ids=["guardrail.tests-green", "guardrail.lint-clean"], + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + guardrails = card.utility_guardrails + assert guardrails["passing_trials"] == 1 + per_check = guardrails["per_check"] + assert per_check["guardrail.tests-green"]["passing_units"] == 2 + assert per_check["guardrail.tests-green"]["failing_units"] == 0 + assert per_check["guardrail.lint-clean"]["passing_units"] == 1 + assert per_check["guardrail.lint-clean"]["failing_units"] == 1 + + +def test_forged_receipt_is_audit_visible_but_ineligible(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + binding = _fixture_binding(fingerprint=fingerprint, target=scoreable_target) + binding.pop("manifest_binding") + _write_verify_receipt(scoreable_target, "forged", binding=binding, commands=[_effectiveness_command()]) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["trials"] == 0 + assert card.ineligible_summary["verifier_manifest_missing"] == 1 + assert len(card.receipt_trail) == 1 + assert card.receipt_trail[0].eligible is False + + +def test_utility_evidence_is_independent_of_effectiveness_plan(scoreable_target): + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + utility_manifest_b = "scorecard-skill-fixture-utility-alt-plan" + if utility_manifest_b not in verify_manifest.registered_manifest_ids(scoreable_target): + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": utility_manifest_b, + "binding_mode": "fixture_eval", + "verifier_id": "brigade.verify.fixture", + "subject": { + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "subject_path": "skills/brigade-work/SKILL.md", + }, + "fixture": { + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + "required_utility_check_ids": ["guardrail.tests-green", "guardrail.lint-clean"], + "checks": [ + { + "check_id": "fixture.echo-ok", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('alt')\"", + }, + { + "check_id": "guardrail.tests-green", + "check_role": "utility_guardrail", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + { + "check_id": "guardrail.lint-clean", + "check_role": "utility_guardrail", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + ], + } + verify_manifest.write_workspace_manifest(scoreable_target, payload) + _track_workspace_manifest(scoreable_target, utility_manifest_b) + binding_a = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_UTILITY_MANIFEST_ID, + ) + binding_b = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=utility_manifest_b, + ) + _write_verify_receipt( + scoreable_target, + "utility-plan-a", + binding=binding_a, + commands=[ + _effectiveness_command(), + _utility_command(ok=True, check_id="guardrail.tests-green"), + _utility_command(ok=True, check_id="guardrail.lint-clean"), + ], + required_utility_check_ids=["guardrail.tests-green", "guardrail.lint-clean"], + ) + _write_verify_receipt( + scoreable_target, + "utility-plan-b", + binding=binding_b, + commands=[ + _effectiveness_command(check_id="fixture.echo-ok"), + _utility_command(ok=True, check_id="guardrail.tests-green"), + _utility_command(ok=True, check_id="guardrail.lint-clean"), + ], + started_at="2026-06-20T01:00:00+00:00", + required_utility_check_ids=["guardrail.tests-green", "guardrail.lint-clean"], + ) + alt_effectiveness = _effectiveness_command(check_id="fixture.echo-ok") + alt_effectiveness["command"] = f"{sys.executable} -c \"print('alt')\"" + receipt_path = scoreable_target / ".brigade" / "work" / "verify-runs" / "utility-plan-b" / "receipt.json" + receipt = json.loads(receipt_path.read_text()) + receipt["commands"][0] = alt_effectiveness + receipt["digests"] = { + "algorithm": "sha256", + "logs": {}, + "receipt_sha256": localio.canonical_json_digest(receipt, exclude_keys={"digests"}), + } + localio.write_json(receipt_path, receipt) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + per_check = card.utility_guardrails["per_check"] + assert per_check["guardrail.tests-green"]["passing_units"] == 1 + assert per_check["guardrail.lint-clean"]["passing_units"] == 1 diff --git a/tests/test_scorecard_reconcile.py b/tests/test_scorecard_reconcile.py new file mode 100644 index 00000000..e5bf6ca9 --- /dev/null +++ b/tests/test_scorecard_reconcile.py @@ -0,0 +1,636 @@ +"""Dual-criterion scorecard promotion gate for outcome reconcile/fork (#503).""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +from brigade import cli, localio, outcome, outcome_cmd, scorecard, verify_manifest, verify_trial + +from tests.test_scorecard import ( + _SCORECARD_ALT_CHECK_MANIFEST_ID, + _SCORECARD_UTILITY_MANIFEST_ID, + _SCORECARD_UTILITY_PASSING_MANIFEST_ID, + _effectiveness_command, + _ensure_scorecard_manifest, + _fixture_binding, + _utility_command, + _write_verify_receipt, +) +from tests.test_verify_trial import _write_skill_subject + +pytest_plugins = ["tests.test_scorecard"] + +_REQUIRED_UTILITY = ["guardrail.tests-green", "guardrail.lint-clean"] + + +def _ensure_git_repo(target: Path) -> None: + if (target / ".git").is_dir(): + return + subprocess.run(["git", "init"], cwd=target, check=True, stdout=subprocess.DEVNULL) + subprocess.run( + [ + "git", + "-c", + "user.name=Test User", + "-c", + "user.email=test@example.invalid", + "commit", + "--allow-empty", + "-m", + "init", + ], + cwd=target, + check=True, + stdout=subprocess.DEVNULL, + ) + + +def _status_file(target: Path) -> Path: + return target / "memory" / "outcome" / "status.json" + + +def _decisions_dir(target: Path) -> Path: + return target / "memory" / "outcome" / "decisions" + + +def _stub_execute(monkeypatch, *, install: str = "installed") -> None: + def _fake(target, artifact_id, action): + return install if action == "install" else "reverted:claude:uninstall" + + monkeypatch.setattr(outcome_cmd, "_execute_skill_decision", _fake) + + +def _skill_fingerprint(target: Path, artifact_id: str = "brigade-work") -> str: + skill = target / "skills" / artifact_id / "SKILL.md" + return verify_trial.subject_hash_for_path(target, str(skill.relative_to(target))) + + +def _utility_pass_commands(*, lint_ok: bool = True) -> list[dict]: + return [ + _effectiveness_command(), + _utility_command(ok=True, check_id="guardrail.tests-green"), + _utility_command(ok=lint_ok, check_id="guardrail.lint-clean"), + ] + + +def _write_independent_utility_pass( + target: Path, + run_id: str, + *, + fingerprint: str, + case_id: str, + manifest_id: str = _SCORECARD_UTILITY_PASSING_MANIFEST_ID, + started_at: str = "2026-06-20T00:00:00+00:00", + lint_ok: bool = True, + reused_from: str | None = None, +) -> None: + _ensure_scorecard_manifest(target, manifest_id=manifest_id) + binding = _fixture_binding( + fingerprint=fingerprint, + target=target, + manifest_id=manifest_id, + case_id=case_id, + ) + _write_verify_receipt( + target, + run_id, + binding=binding, + commands=_utility_pass_commands(lint_ok=lint_ok), + required_utility_check_ids=_REQUIRED_UTILITY, + started_at=started_at, + reused_from=reused_from, + ) + + +def _write_registry_skill_utility_manifest( + target: Path, + skill_id: str, + *, + manifest_id: str, + case_id: str, +) -> None: + from tests.test_verify_trial import _track_workspace_manifest + + subject_path = f".brigade/skills/registry/{skill_id}/SKILL.md" + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": manifest_id, + "binding_mode": "fixture_eval", + "verifier_id": "brigade.verify.fixture", + "subject": { + "artifact_kind": "skill", + "artifact_id": skill_id, + "subject_path": subject_path, + }, + "fixture": { + "manifest_id": "adversarial-failed-verification", + "case_id": case_id, + "check_id": "fixture.echo-ok", + }, + "required_utility_check_ids": _REQUIRED_UTILITY, + "checks": [ + { + "check_id": "fixture.echo-ok", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + { + "check_id": "guardrail.tests-green", + "check_role": "utility_guardrail", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + { + "check_id": "guardrail.lint-clean", + "check_role": "utility_guardrail", + "command": f"{sys.executable} -c \"print('ok')\"", + }, + ], + } + verify_manifest.write_workspace_manifest(target, payload) + _track_workspace_manifest(target, manifest_id) + + +def _registry_binding( + target: Path, + skill_id: str, + fingerprint: str, + *, + manifest_id: str, + case_id: str, +) -> dict: + manifest, error = verify_manifest.resolve_manifest(target, manifest_id) + assert manifest is not None, error + return { + "binding_mode": "fixture_eval", + "artifact_kind": "skill", + "artifact_id": skill_id, + "content_fingerprint": fingerprint, + "fixture_binding": { + "manifest_id": "adversarial-failed-verification", + "case_id": case_id, + "check_id": "fixture.echo-ok", + }, + "manifest_binding": verify_manifest.build_manifest_binding(target, manifest), + "verifier_identity": {"verifier_id": "brigade.verify.fixture", "session_id": "verifier-1"}, + } + + +def seed_registry_skill_scorecard_promotion(target: Path, skill_id: str) -> str: + """Write dual-gate scorecard receipts for a registry skill under test.""" + _ensure_git_repo(target) + passing_id = f"{skill_id}-utility-passing" + utility_id = f"{skill_id}-utility" + for manifest_id, case_id in ( + (passing_id, "report-passing-tests"), + (utility_id, "report-failing-tests"), + ): + if manifest_id not in verify_manifest.registered_manifest_ids(target): + _write_registry_skill_utility_manifest( + target, + skill_id, + manifest_id=manifest_id, + case_id=case_id, + ) + fingerprint = outcome_cmd.artifact_fingerprint(target, skill_id, "skill") + assert fingerprint is not None + for run_id, manifest_id, case_id, started_at in ( + (f"{skill_id}-pass-a", passing_id, "report-passing-tests", "2026-06-20T00:00:00+00:00"), + (f"{skill_id}-pass-b", utility_id, "report-failing-tests", "2026-06-20T01:00:00+00:00"), + ): + binding = _registry_binding( + target, + skill_id, + fingerprint, + manifest_id=manifest_id, + case_id=case_id, + ) + _write_verify_receipt( + target, + run_id, + binding=binding, + commands=_utility_pass_commands(), + required_utility_check_ids=_REQUIRED_UTILITY, + started_at=started_at, + ) + return fingerprint + + +def seed_registry_skill_scorecard_hurt(target: Path, skill_id: str) -> None: + _ensure_git_repo(target) + hurt_id = f"{skill_id}-utility-alt" + if hurt_id not in verify_manifest.registered_manifest_ids(target): + from tests.test_verify_trial import _track_workspace_manifest + + subject_path = f".brigade/skills/registry/{skill_id}/SKILL.md" + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": hurt_id, + "binding_mode": "fixture_eval", + "verifier_id": "brigade.verify.fixture", + "subject": { + "artifact_kind": "skill", + "artifact_id": skill_id, + "subject_path": subject_path, + }, + "fixture": { + "manifest_id": "adversarial-failed-verification", + "case_id": "report-passing-tests", + "check_id": "fixture.echo-alt", + }, + "checks": [ + { + "check_id": "fixture.echo-alt", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('alt')\"", + } + ], + } + verify_manifest.write_workspace_manifest(target, payload) + _track_workspace_manifest(target, hurt_id) + fingerprint = outcome_cmd.artifact_fingerprint(target, skill_id, "skill") + assert fingerprint is not None + binding = _registry_binding( + target, + skill_id, + fingerprint, + manifest_id=hurt_id, + case_id="report-passing-tests", + ) + binding["fixture_binding"]["check_id"] = "fixture.echo-alt" + hurt_command = { + "command": f"{sys.executable} -c \"print('alt')\"", + "check_role": "effectiveness", + "check_id": "fixture.echo-alt", + "status": "failed", + "exit_code": 1, + } + verify_trial.stamp_verify_command_failure_taxonomy(hurt_command) + _write_verify_receipt( + target, + f"{skill_id}-hurt", + binding=binding, + commands=[hurt_command], + started_at="2026-06-20T02:00:00+00:00", + ) + + +def _seed_dual_gate_promotion_ready(target: Path) -> str: + skill = _write_skill_subject(target) + fingerprint = verify_trial.subject_hash_for_path(target, str(skill.relative_to(target))) + _write_independent_utility_pass( + target, + "pass-a", + fingerprint=fingerprint, + case_id="report-passing-tests", + manifest_id=_SCORECARD_UTILITY_PASSING_MANIFEST_ID, + ) + _write_independent_utility_pass( + target, + "pass-b", + fingerprint=fingerprint, + case_id="report-failing-tests", + manifest_id=_SCORECARD_UTILITY_MANIFEST_ID, + started_at="2026-06-20T01:00:00+00:00", + ) + return fingerprint + + +def test_effectiveness_pass_utility_fail_holds_promotion(scoreable_target, capsys, monkeypatch): + _stub_execute(monkeypatch) + _ensure_scorecard_manifest(scoreable_target, manifest_id=_SCORECARD_UTILITY_PASSING_MANIFEST_ID) + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + _write_independent_utility_pass( + scoreable_target, + "eff-only-a", + fingerprint=fingerprint, + case_id="report-passing-tests", + lint_ok=False, + ) + _write_independent_utility_pass( + scoreable_target, + "eff-only-b", + fingerprint=fingerprint, + case_id="report-failing-tests", + manifest_id=_SCORECARD_UTILITY_MANIFEST_ID, + started_at="2026-06-20T01:00:00+00:00", + lint_ok=True, + ) + + assert outcome_cmd.reconcile(target=scoreable_target, apply=False, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + decision = payload["decisions"][0] + assert decision["action"] == "hold" + assert decision["reason"] == "withheld: utility_guardrail guardrail.lint-clean" + assert not _status_file(scoreable_target).exists() + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.dimensions["effectiveness"]["helped"] == 2 + decision = scorecard.decide_scorecard( + card, + artifact_id="brigade-work", + current_status="candidate", + last_action_ts=None, + now=localio.utc_now(), + config=outcome.ReconcileConfig(), + ) + assert decision.action == "hold" + assert decision.reason == "withheld: utility_guardrail guardrail.lint-clean" + + +def test_duplicate_utility_evidence_does_not_satisfy_gate(scoreable_target, capsys, monkeypatch): + _stub_execute(monkeypatch) + _ensure_scorecard_manifest(scoreable_target, manifest_id=_SCORECARD_UTILITY_PASSING_MANIFEST_ID) + skill = _write_skill_subject(scoreable_target) + fingerprint = verify_trial.subject_hash_for_path(scoreable_target, str(skill.relative_to(scoreable_target))) + _write_independent_utility_pass( + scoreable_target, + "orig", + fingerprint=fingerprint, + case_id="report-passing-tests", + ) + _write_independent_utility_pass( + scoreable_target, + "reuse", + fingerprint=fingerprint, + case_id="report-passing-tests", + started_at="2026-06-20T01:00:00+00:00", + reused_from="orig", + ) + + card = scorecard.scorecard_for_artifact(scoreable_target, "brigade-work") + assert card is not None + assert card.utility_guardrails["per_check"]["guardrail.tests-green"]["passing_units"] == 1 + + assert outcome_cmd.reconcile(target=scoreable_target, apply=False, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + decision = payload["decisions"][0] + assert decision["action"] == "hold" + assert decision["reason"] in { + "insufficient verified evidence", + "withheld: utility_guardrail guardrail.tests-green", + "withheld: utility_guardrail guardrail.lint-clean", + } + + +def test_dual_gate_promotion_persists_route_policy_marker(scoreable_target, capsys, monkeypatch): + _stub_execute(monkeypatch) + _seed_dual_gate_promotion_ready(scoreable_target) + + assert outcome_cmd.reconcile(target=scoreable_target, apply=True, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["applied"] == ["brigade-work"] + decision = payload["decisions"][0] + assert decision["action"] == "install" + assert decision["policy_version"] == scorecard.SCORECARD_POLICY_VERSION + assert decision["new_status"] == "promoted" + + status = json.loads(_status_file(scoreable_target).read_text()) + entry = status["artifacts"]["brigade-work"] + assert entry["status"] == "promoted" + assert entry["route_policy"]["policy_version"] == scorecard.SCORECARD_POLICY_VERSION + + receipts = list(_decisions_dir(scoreable_target).glob("*.json")) + assert len(receipts) == 1 + receipt = json.loads(receipts[0].read_text()) + assert receipt["route_policy"]["policy_version"] == scorecard.SCORECARD_POLICY_VERSION + + +def test_missing_scorecard_holds_skill_promotion(scoreable_target, capsys, monkeypatch): + _stub_execute(monkeypatch) + from tests.test_outcome_cmd import _helped, _seed, _write_registry_skill + + _write_registry_skill(scoreable_target, "skill-x") + _seed(scoreable_target, _helped("skill-x", 2)) + + assert outcome_cmd.reconcile(target=scoreable_target, apply=False, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + decision = payload["decisions"][0] + assert decision["artifact_id"] == "skill-x" + assert decision["action"] == "hold" + assert decision["reason"] == "withheld: missing scorecard" + assert not _status_file(scoreable_target).exists() + + +def test_promoted_regression_demotes_inside_cooldown(scoreable_target, capsys, monkeypatch): + _stub_execute(monkeypatch, install="installed") + _seed_dual_gate_promotion_ready(scoreable_target) + cfg = outcome.ReconcileConfig(cooldown_seconds=86_400, revert_min_hurt=99) + assert outcome_cmd.reconcile(target=scoreable_target, apply=True, config=cfg, json_output=True) == 0 + capsys.readouterr() + + fingerprint = _skill_fingerprint(scoreable_target) + binding = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_ALT_CHECK_MANIFEST_ID, + case_id="report-failing-tests", + check_id="fixture.echo-alt", + ) + _write_verify_receipt( + scoreable_target, + "hurt", + binding=binding, + commands=[_effectiveness_command(ok=False, check_id="fixture.echo-alt")], + started_at="2026-06-20T02:00:00+00:00", + ) + + assert outcome_cmd.reconcile(target=scoreable_target, apply=True, config=cfg, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + decision = payload["decisions"][0] + assert decision["action"] == "rollback" + assert decision["new_status"] == "demoted" + assert decision["reason"] == "verified regression measured" + + status = json.loads(_status_file(scoreable_target).read_text()) + assert status["artifacts"]["brigade-work"]["status"] == "demoted" + + +def test_promoted_regression_demotes_even_when_rollback_fails(scoreable_target, capsys, monkeypatch): + _stub_execute(monkeypatch, install="installed") + + def _rollback_always_fails(target, artifact_id, action): + return "reverted:claude:noop" if action == "rollback" else "installed" + + monkeypatch.setattr(outcome_cmd, "_execute_skill_decision", _rollback_always_fails) + + _seed_dual_gate_promotion_ready(scoreable_target) + cfg = outcome.ReconcileConfig(cooldown_seconds=0) + assert outcome_cmd.reconcile(target=scoreable_target, apply=True, config=cfg, json_output=True) == 0 + capsys.readouterr() + + fingerprint = _skill_fingerprint(scoreable_target) + binding = _fixture_binding( + fingerprint=fingerprint, + target=scoreable_target, + manifest_id=_SCORECARD_ALT_CHECK_MANIFEST_ID, + case_id="report-failing-tests", + check_id="fixture.echo-alt", + ) + _write_verify_receipt( + scoreable_target, + "hurt", + binding=binding, + commands=[_effectiveness_command(ok=False, check_id="fixture.echo-alt")], + started_at="2026-06-20T02:00:00+00:00", + ) + + assert outcome_cmd.reconcile(target=scoreable_target, apply=True, config=cfg, json_output=True) == 0 + payload = json.loads(capsys.readouterr().out) + decision = payload["decisions"][0] + assert decision["action"] == "rollback" + assert decision["new_status"] == "demoted" + assert "noop" in decision["execution"] + + status = json.loads(_status_file(scoreable_target).read_text()) + entry = status["artifacts"]["brigade-work"] + assert entry["status"] == "demoted" + assert "route_policy" not in entry + + +def test_fork_parity_with_reconcile_dual_gate(scoreable_target, capsys, tmp_path): + _seed_dual_gate_promotion_ready(scoreable_target) + + assert outcome_cmd.reconcile(target=scoreable_target, apply=False, json_output=True) == 0 + reconcile_payload = json.loads(capsys.readouterr().out) + reconcile_decision = reconcile_payload["decisions"][0] + + fork_out = tmp_path / "fork.json" + assert outcome_cmd.fork(target=scoreable_target, out=fork_out, json_output=True) == 0 + capsys.readouterr() + fork_entry = json.loads(fork_out.read_text())["artifacts"]["brigade-work"] + + assert fork_entry["action"] == reconcile_decision["action"] + assert fork_entry["new_status"] == reconcile_decision["new_status"] + assert fork_entry["policy_version"] == scorecard.SCORECARD_POLICY_VERSION + + +def test_fork_utility_threshold_override(scoreable_target, capsys, tmp_path): + _seed_dual_gate_promotion_ready(scoreable_target) + + fork_strict = tmp_path / "strict.json" + assert ( + outcome_cmd.fork( + target=scoreable_target, + out=fork_strict, + config=outcome.ReconcileConfig(utility_min_passing_units=3), + json_output=True, + ) + == 0 + ) + capsys.readouterr() + strict = json.loads(fork_strict.read_text())["artifacts"]["brigade-work"] + assert strict["new_status"] != "promoted" + + fork_loose = tmp_path / "loose.json" + assert ( + outcome_cmd.fork( + target=scoreable_target, + out=fork_loose, + config=outcome.ReconcileConfig(utility_min_passing_units=1, install_min_helped=1), + json_output=True, + ) + == 0 + ) + capsys.readouterr() + loose = json.loads(fork_loose.read_text())["artifacts"]["brigade-work"] + assert loose["new_status"] == "promoted" + + +def test_cli_fork_utility_check_min_passing_units_alias(scoreable_target, capsys, tmp_path): + _seed_dual_gate_promotion_ready(scoreable_target) + fork_out = tmp_path / "alias.json" + assert ( + cli.main( + [ + "outcome", + "fork", + "--target", + str(scoreable_target), + "--out", + str(fork_out), + "--utility-check-min-passing-units", + "3", + "--json", + ] + ) + == 0 + ) + capsys.readouterr() + entry = json.loads(fork_out.read_text())["artifacts"]["brigade-work"] + assert entry["new_status"] != "promoted" + + +def test_cli_fork_effective_wilson_override(scoreable_target, capsys, tmp_path): + _seed_dual_gate_promotion_ready(scoreable_target) + fork_out = tmp_path / "wilson.json" + assert ( + cli.main( + [ + "outcome", + "fork", + "--target", + str(scoreable_target), + "--out", + str(fork_out), + "--effective-wilson-min", + "0.99", + "--json", + ] + ) + == 0 + ) + capsys.readouterr() + entry = json.loads(fork_out.read_text())["artifacts"]["brigade-work"] + assert entry["new_status"] != "promoted" + assert "wilson" in entry["reason"].lower() or entry["helped"] < 2 + + +def test_fork_z_override_controls_effectiveness_gate(scoreable_target, capsys, tmp_path): + _seed_dual_gate_promotion_ready(scoreable_target) + high_z_out = tmp_path / "high-z.json" + low_z_out = tmp_path / "low-z.json" + assert ( + cli.main( + [ + "outcome", + "fork", + "--target", + str(scoreable_target), + "--out", + str(high_z_out), + "--z", + "5.0", + "--json", + ] + ) + == 0 + ) + assert ( + cli.main( + [ + "outcome", + "fork", + "--target", + str(scoreable_target), + "--out", + str(low_z_out), + "--z", + "0.5", + "--json", + ] + ) + == 0 + ) + capsys.readouterr() + high_z_entry = json.loads(high_z_out.read_text())["artifacts"]["brigade-work"] + low_z_entry = json.loads(low_z_out.read_text())["artifacts"]["brigade-work"] + assert high_z_entry["new_status"] != "promoted" + assert "wilson" in high_z_entry["reason"].lower() + assert low_z_entry["new_status"] == "promoted" diff --git a/tests/test_verify_trial.py b/tests/test_verify_trial.py new file mode 100644 index 00000000..f57ea98f --- /dev/null +++ b/tests/test_verify_trial.py @@ -0,0 +1,715 @@ +"""Scoreable verify receipt binding and project_trial eligibility (#571).""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from brigade import localio, verify_manifest, verify_trial, work_cmd +from brigade.verify_manifest import _WORKSPACE_MANIFESTS_REL +from brigade.work_cmd import verification as verify_mod + +from tests.work_cmd_test_helpers import _init_git_repo + + +def _init_verify_git_repo(path: Path) -> None: + _init_git_repo(path) + (path / ".gitignore").write_text(".brigade/\n") + subprocess.run(["git", "add", ".gitignore"], cwd=path, check=True, stdout=subprocess.DEVNULL) + subprocess.run( + ["git", "-c", "user.name=Test User", "-c", "user.email=test@example.invalid", "commit", "-m", "init"], + cwd=path, + check=True, + stdout=subprocess.DEVNULL, + ) + + +def _track_workspace_manifest(path: Path, manifest_id: str) -> None: + rel = _WORKSPACE_MANIFESTS_REL / f"{manifest_id}.json" + subprocess.run(["git", "add", str(rel)], cwd=path, check=True, stdout=subprocess.DEVNULL) + subprocess.run( + [ + "git", + "-c", + "user.name=Test User", + "-c", + "user.email=test@example.invalid", + "commit", + "-m", + f"add verify manifest {manifest_id}", + ], + cwd=path, + check=True, + stdout=subprocess.DEVNULL, + ) + + +def _write_skill_subject( + path: Path, *, rel: str = "skills/brigade-work/SKILL.md", content: str = "# brigade-work\n" +) -> Path: + skill_path = path / rel + skill_path.parent.mkdir(parents=True, exist_ok=True) + skill_path.write_text(content) + return skill_path + + +def _stamp_receipt_digest(receipt: dict) -> None: + receipt["digests"] = { + "algorithm": "sha256", + "logs": {}, + "receipt_sha256": localio.canonical_json_digest(receipt, exclude_keys={"digests"}), + } + + +def _minimal_manifest_binding( + *, + manifest_id: str = "structural-fixture", + payload_sha256: str = "sha256:" + "b" * 64, +) -> dict: + return { + "manifest_id": manifest_id, + "payload_sha256": payload_sha256, + "source_path": f"verify/manifests/{manifest_id}.json", + } + + +def _structural_fixture_binding(**overrides) -> dict: + binding = { + "binding_mode": "fixture_eval", + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "content_fingerprint": "sha256:" + "a" * 64, + "fixture_binding": { + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + "manifest_binding": _minimal_manifest_binding(), + "verifier_identity": {"verifier_id": "test", "session_id": "verifier-1"}, + } + binding.update(overrides) + return binding + + +def _write_patch_manifest(path: Path, *, manifest_id: str = "test-skill-patch") -> dict: + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": manifest_id, + "binding_mode": "patch_backed", + "verifier_id": "brigade.verify.test", + "patch_source": "worktree", + "subject": { + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "subject_path": "skills/brigade-work/SKILL.md", + }, + "checks": [ + { + "check_id": "verify.echo-ok", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('ok')\"", + } + ], + } + verify_manifest.write_workspace_manifest(path, payload) + _track_workspace_manifest(path, manifest_id) + return payload + + +def _write_fixture_manifest(path: Path, *, manifest_id: str = "test-skill-fixture") -> dict: + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": manifest_id, + "binding_mode": "fixture_eval", + "verifier_id": "brigade.verify.fixture", + "subject": { + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "subject_path": "skills/brigade-work/SKILL.md", + }, + "fixture": { + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + "checks": [ + { + "check_id": "fixture.echo-ok", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('ok')\"", + } + ], + } + verify_manifest.write_workspace_manifest(path, payload) + _track_workspace_manifest(path, manifest_id) + return payload + + +def _run_manifest_verify( + tmp_path: Path, + manifest_id: str, + *, + monkeypatch: pytest.MonkeyPatch, + start_session: bool = True, +) -> dict: + monkeypatch.setattr(verify_mod.graphtrail_delta, "capture_before", lambda *_a, **_k: None) + monkeypatch.setattr( + verify_mod.graphtrail_delta, + "capture_after_and_diff", + lambda *_a, **_k: verify_mod.graphtrail_delta._status("disabled", "graph disabled in test"), + ) + if start_session: + assert work_cmd.start(target=tmp_path, title="verify-trial") == 0 + import io + from contextlib import redirect_stdout + + buffer = io.StringIO() + with redirect_stdout(buffer): + rc = verify_mod.verify_run( + target=tmp_path, + manifest_id=manifest_id, + reuse=False, + json_output=True, + ) + assert rc == 0, buffer.getvalue() + return json.loads(buffer.getvalue()) + + +@pytest.fixture +def scoreable_target(tmp_path): + target = tmp_path / "ws" + target.mkdir() + _init_verify_git_repo(target) + _write_skill_subject(target) + subprocess.run(["git", "add", "skills"], cwd=target, check=True, stdout=subprocess.DEVNULL) + subprocess.run( + ["git", "-c", "user.name=Test User", "-c", "user.email=test@example.invalid", "commit", "-m", "add skill"], + cwd=target, + check=True, + stdout=subprocess.DEVNULL, + ) + return target + + +def test_registered_manifest_is_discoverable(scoreable_target): + _write_patch_manifest(scoreable_target, manifest_id="local-patch") + assert "local-patch" in verify_manifest.registered_manifest_ids(scoreable_target) + + +def test_resolve_manifest_rejects_untracked_workspace_manifest(scoreable_target): + payload = { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": "untracked-patch", + "binding_mode": "patch_backed", + "verifier_id": "brigade.verify.test", + "patch_source": "worktree", + "subject": { + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "subject_path": "skills/brigade-work/SKILL.md", + }, + "checks": [ + { + "check_id": "verify.echo-ok", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('ok')\"", + } + ], + } + verify_manifest.write_workspace_manifest(scoreable_target, payload) + manifest_path = scoreable_target / _WORKSPACE_MANIFESTS_REL / "untracked-patch.json" + assert manifest_path.is_file() + + manifest, error = verify_manifest.resolve_manifest(scoreable_target, "untracked-patch") + assert manifest is None + assert error == "verify manifest not tracked: untracked-patch" + assert "untracked-patch" not in verify_manifest.registered_manifest_ids(scoreable_target) + + +def test_adhoc_verify_receipt_is_unattributed(scoreable_target, monkeypatch): + monkeypatch.setattr(verify_mod.graphtrail_delta, "capture_before", lambda *_a, **_k: None) + monkeypatch.setattr( + verify_mod.graphtrail_delta, + "capture_after_and_diff", + lambda *_a, **_k: verify_mod.graphtrail_delta._status("disabled", "graph disabled in test"), + ) + import io + from contextlib import redirect_stdout + + buffer = io.StringIO() + with redirect_stdout(buffer): + rc = verify_mod.verify_run( + target=scoreable_target, + commands=[f"{sys.executable} -c \"print('ok')\""], + reuse=False, + json_output=True, + ) + assert rc == 0 + receipt = json.loads(buffer.getvalue()) + assert "subject_binding" not in receipt + assert all("check_role" not in command for command in receipt["commands"]) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "unattributed" + assert projection.attributed is False + + +def test_manifest_verify_writes_subject_binding_and_check_roles(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch) + assert receipt["verify_manifest_id"] == "test-skill-patch" + binding = receipt["subject_binding"] + assert binding["binding_mode"] == "patch_backed" + assert binding["manifest_binding"]["manifest_id"] == "test-skill-patch" + assert binding["artifact_id"] == "brigade-work" + assert binding["patch_binding"]["subject_path"] == "skills/brigade-work/SKILL.md" + assert receipt["commands"][0]["check_role"] == "effectiveness" + assert receipt["commands"][0]["check_id"] == "verify.echo-ok" + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "empty_patch" + + +def test_manifest_verify_rejects_cli_command_override(scoreable_target): + from brigade.cli.work import dispatching + + class Parser: + @staticmethod + def error(msg: str) -> None: + raise SystemExit(msg) + + class Args: + work_command = "verify" + verify_command = "run" + target = scoreable_target + verify_commands = ["true"] + verify_argv_json = None + verify_manifest_id = "test-skill-patch" + timeout = 900 + graphtrail_timeout = None + json = False + capture = None + capture_kind = "skill" + no_reuse = True + _brigade_parser = Parser() + + _write_patch_manifest(scoreable_target) + with pytest.raises(SystemExit, match="mutually exclusive"): + dispatching.dispatch(Args()) + + +def test_project_trial_empty_patch_is_ineligible(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "empty_patch" + + +def test_project_trial_patch_digest_mismatch_is_ineligible(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch) + run_dir = Path(receipt["path"]) + patch_path = run_dir / "changes.patch" + patch_path.write_bytes(b"diff --git a/x b/x\n") + projection = verify_trial.project_trial(receipt, target=scoreable_target, patch_path=patch_path) + assert projection.eligible is False + assert projection.reason == "patch_digest_mismatch" + + +def test_project_trial_no_owned_work_without_subject_delta(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + assert work_cmd.start(target=scoreable_target, title="verify-trial") == 0 + readme = scoreable_target / "README.md" + readme.write_text("changed\n") + subprocess.run(["git", "add", "README.md"], cwd=scoreable_target, check=True) + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch, start_session=False) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "no_owned_work" + + +def test_project_trial_no_session_dirty_work_is_ineligible(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + skill = _write_skill_subject(scoreable_target, content="# brigade-work\npatched\n") + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch, start_session=False) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "producer_session_missing" + + +def test_project_trial_pre_existing_dirty_subject_is_ineligible(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + skill = _write_skill_subject(scoreable_target, content="# brigade-work\npre-existing\n") + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + assert work_cmd.start(target=scoreable_target, title="verify-trial") == 0 + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch, start_session=False) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "subject_pre_existing_dirty" + + +def test_project_trial_concurrent_session_ownership_is_ineligible(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + skill = _write_skill_subject(scoreable_target, content="# brigade-work\npre-existing\n") + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + assert work_cmd.start(target=scoreable_target, title="producer") == 0 + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch, start_session=False) + producer = receipt["subject_binding"]["producer_binding"] + producer["subject_clean_at_start"] = True + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "concurrent_session_ownership" + + +def test_project_trial_owned_patch_is_eligible(scoreable_target, monkeypatch): + _write_patch_manifest(scoreable_target) + assert work_cmd.start(target=scoreable_target, title="verify-trial") == 0 + skill = _write_skill_subject(scoreable_target, content="# brigade-work\npatched\n") + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + receipt = _run_manifest_verify(scoreable_target, "test-skill-patch", monkeypatch=monkeypatch, start_session=False) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is True + + +def test_project_trial_fixture_eval_without_patch(scoreable_target, monkeypatch): + _write_fixture_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-fixture", monkeypatch=monkeypatch, start_session=False) + binding = receipt["subject_binding"] + assert binding["binding_mode"] == "fixture_eval" + assert binding["fixture_binding"]["case_id"] == "report-failing-tests" + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is True + + +def test_project_trial_generated_patch_requires_independent_verifier(scoreable_target, monkeypatch): + payload = _write_patch_manifest(scoreable_target, manifest_id="generated-patch") + payload["patch_source"] = "generated" + verify_manifest.write_workspace_manifest(scoreable_target, payload) + _track_workspace_manifest(scoreable_target, "generated-patch") + assert work_cmd.start(target=scoreable_target, title="producer") == 0 + skill = _write_skill_subject(scoreable_target, content="# brigade-work\ngenerated\n") + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + monkeypatch.setenv("BRIGADE_CLAUDE_SESSION", "aaaaaaaaaaaaaaaa") + receipt = _run_manifest_verify( + scoreable_target, + "generated-patch", + monkeypatch=monkeypatch, + start_session=False, + ) + producer_session = receipt["subject_binding"]["producer_binding"]["work_session_id"] + receipt["subject_binding"]["verifier_identity"]["session_id"] = producer_session + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "verifier_not_independent" + receipt["subject_binding"]["verifier_identity"]["session_id"] = f"{producer_session}-verifier" + _stamp_receipt_digest(receipt) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is True + + +def test_project_trial_infrastructure_failure_is_ineligible(scoreable_target): + receipt = { + "status": "failed", + "failure_class": "infrastructure", + "failure_kind": "timeout", + "commands": [ + { + "status": "timed_out", + "exit_code": None, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + "failure_class": "infrastructure", + "failure_kind": "timeout", + } + ], + "subject_binding": _structural_fixture_binding(), + } + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "failure_taxonomy_infrastructure" + assert projection.infrastructure_excluded is True + + +def test_project_trial_missing_taxonomy_on_failure_is_ineligible(scoreable_target): + receipt = { + "status": "failed", + "commands": [ + { + "status": "failed", + "exit_code": 1, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + } + ], + "subject_binding": _structural_fixture_binding(), + } + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "failure_taxonomy_missing" + + +def test_project_trial_skill_failure_with_taxonomy_is_eligible(scoreable_target): + receipt = { + "status": "failed", + "failure_class": "verification", + "failure_kind": "receipt_failed", + "commands": [ + { + "status": "failed", + "exit_code": 1, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + "failure_class": "verification", + "failure_kind": "nonzero_exit", + } + ], + "subject_binding": _structural_fixture_binding(), + } + _stamp_receipt_digest(receipt) + projection = verify_trial.project_trial(receipt) + assert projection.eligible is True + assert projection.reason == "eligible" + + +def test_project_trial_missing_receipt_digest_is_ineligible(scoreable_target): + receipt = { + "status": "completed", + "commands": [ + { + "status": "completed", + "exit_code": 0, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + } + ], + "subject_binding": _structural_fixture_binding(), + } + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "receipt_digest_missing" + assert projection.attributed is True + + +def test_project_trial_receipt_digest_mismatch_is_ineligible(scoreable_target): + receipt = { + "status": "completed", + "commands": [ + { + "status": "completed", + "exit_code": 0, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + } + ], + "subject_binding": _structural_fixture_binding(), + "digests": { + "algorithm": "sha256", + "logs": {}, + "receipt_sha256": "b" * 64, + }, + } + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "receipt_digest_mismatch" + assert projection.attributed is True + + +def test_project_trial_missing_check_role_is_ineligible(scoreable_target): + receipt = { + "status": "completed", + "commands": [{"status": "completed", "exit_code": 0, "check_id": "verify.echo-ok"}], + "subject_binding": _structural_fixture_binding(), + } + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "missing_check_role" + + +def test_verify_finalize_stamps_failure_taxonomy(tmp_path): + receipt = { + "schema_version": 2, + "run_id": "test", + "target": str(tmp_path), + "status": "failed", + "started_at": "2026-01-01T00:00:00+00:00", + "path": str(tmp_path), + "commands": [{"command": "false", "status": "failed", "exit_code": 1}], + } + verify_trial.stamp_verify_receipt_failure_taxonomy(receipt) + assert receipt["commands"][0]["failure_class"] == "verification" + assert receipt["failure_class"] == "verification" + + +def test_manifest_required_utility_check_ids_must_reference_guardrail_checks(scoreable_target): + with pytest.raises(ValueError, match="utility_guardrail"): + verify_manifest.write_workspace_manifest( + scoreable_target, + { + "schema": "brigade.verify_manifest.v1", + "schema_version": 1, + "manifest_id": "bad-utility", + "binding_mode": "fixture_eval", + "verifier_id": "brigade.verify.test", + "subject": { + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "subject_path": "skills/brigade-work/SKILL.md", + }, + "fixture": { + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + "required_utility_check_ids": ["guardrail.missing"], + "checks": [ + { + "check_id": "fixture.echo-ok", + "check_role": "effectiveness", + "command": f"{sys.executable} -c \"print('ok')\"", + } + ], + }, + ) + + +def test_project_trial_missing_required_utility_check_is_ineligible(scoreable_target): + receipt = { + "status": "completed", + "required_utility_check_ids": ["guardrail.tests-green"], + "commands": [ + { + "status": "completed", + "exit_code": 0, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + } + ], + "subject_binding": _structural_fixture_binding(), + } + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "missing_required_utility_check" + + +def test_manifest_verify_stamps_required_utility_check_ids(scoreable_target, monkeypatch): + payload = _write_patch_manifest(scoreable_target, manifest_id="utility-manifest") + payload["checks"].append( + { + "check_id": "guardrail.tests-green", + "check_role": "utility_guardrail", + "command": f"{sys.executable} -c \"print('ok')\"", + } + ) + payload["required_utility_check_ids"] = ["guardrail.tests-green"] + verify_manifest.write_workspace_manifest(scoreable_target, payload) + _track_workspace_manifest(scoreable_target, "utility-manifest") + assert work_cmd.start(target=scoreable_target, title="verify-trial") == 0 + skill = _write_skill_subject(scoreable_target, content="# brigade-work\nutility\n") + subprocess.run(["git", "add", str(skill.relative_to(scoreable_target))], cwd=scoreable_target, check=True) + receipt = _run_manifest_verify(scoreable_target, "utility-manifest", monkeypatch=monkeypatch, start_session=False) + assert receipt["required_utility_check_ids"] == ["guardrail.tests-green"] + assert receipt["commands"][1]["check_role"] == "utility_guardrail" + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is True + + +def test_project_trial_missing_manifest_binding_is_ineligible(scoreable_target): + receipt = { + "status": "completed", + "commands": [ + { + "status": "completed", + "exit_code": 0, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + } + ], + "subject_binding": { + "binding_mode": "fixture_eval", + "artifact_kind": "skill", + "artifact_id": "brigade-work", + "content_fingerprint": "sha256:" + "a" * 64, + "fixture_binding": { + "manifest_id": "adversarial-failed-verification", + "case_id": "report-failing-tests", + "check_id": "fixture.echo-ok", + }, + "verifier_identity": {"verifier_id": "test", "session_id": "verifier-1"}, + }, + } + _stamp_receipt_digest(receipt) + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "verifier_manifest_missing" + + +def test_project_trial_missing_verifier_identity_is_ineligible(scoreable_target): + binding = _structural_fixture_binding() + binding.pop("verifier_identity") + receipt = { + "status": "completed", + "commands": [ + { + "status": "completed", + "exit_code": 0, + "check_role": "effectiveness", + "check_id": "verify.echo-ok", + } + ], + "subject_binding": binding, + } + _stamp_receipt_digest(receipt) + projection = verify_trial.project_trial(receipt) + assert projection.eligible is False + assert projection.reason == "verifier_identity_missing" + + +def test_project_trial_forged_manifest_binding_mismatch_is_ineligible(scoreable_target, monkeypatch): + _write_fixture_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-fixture", monkeypatch=monkeypatch, start_session=False) + receipt["subject_binding"]["manifest_binding"]["payload_sha256"] = "sha256:" + "f" * 64 + _stamp_receipt_digest(receipt) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "verifier_manifest_mismatch" + + +def test_project_trial_forged_subject_artifact_id_is_ineligible(scoreable_target, monkeypatch): + _write_fixture_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-fixture", monkeypatch=monkeypatch, start_session=False) + receipt["subject_binding"]["artifact_id"] = "forged-skill" + _stamp_receipt_digest(receipt) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "verifier_manifest_mismatch" + + +def test_project_trial_forged_check_plan_is_ineligible(scoreable_target, monkeypatch): + _write_fixture_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-fixture", monkeypatch=monkeypatch, start_session=False) + receipt["commands"][0]["check_id"] = "forged.check" + _stamp_receipt_digest(receipt) + projection = verify_trial.project_trial(receipt, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "verifier_manifest_mismatch" + + +def test_project_trial_copied_binding_without_tracked_manifest_is_ineligible(scoreable_target, monkeypatch): + _write_fixture_manifest(scoreable_target) + receipt = _run_manifest_verify(scoreable_target, "test-skill-fixture", monkeypatch=monkeypatch, start_session=False) + forged = dict(receipt) + forged.pop("verify_manifest_id", None) + forged["subject_binding"] = dict(receipt["subject_binding"]) + _stamp_receipt_digest(forged) + projection = verify_trial.project_trial(forged, target=scoreable_target) + assert projection.eligible is False + assert projection.reason == "verifier_manifest_missing" diff --git a/tests/test_work_cmd_verification.py b/tests/test_work_cmd_verification.py index 1f165e9f..c56dd8e6 100644 --- a/tests/test_work_cmd_verification.py +++ b/tests/test_work_cmd_verification.py @@ -204,8 +204,9 @@ def test_outcome_health_flags_dormant_then_half_fed(tmp_path): _init_git_repo(tmp_path) assert work_cmd.verify_run(target=tmp_path, commands=["python3 -c \"print('ok')\""]) == 0 half_fed = outcome_cmd.health(tmp_path) - assert half_fed["verify_run_count"] >= 1 and half_fed["record_count"] == 0 + assert half_fed["verify_run_count"] >= 1 and half_fed["eligible_receipt_count"] == 0 assert half_fed["top_issue"]["name"] == "outcome_loop_half_fed" + assert "subject_binding" in half_fed["top_issue"]["detail"] def test_work_acceptance_rollup_covers_completion_review_and_closeout(tmp_path, capsys): @@ -1872,6 +1873,7 @@ def fake_verify_run(**kwargs): { "target": tmp_path, "commands": ["python3 -m pytest -q"], + "manifest_id": None, "timeout": 900, "graphtrail_timeout": 45, "json_output": True, @@ -1989,12 +1991,16 @@ def fake_closeout(**kwargs): assert cli.main(["work", "verify", "show", "latest", "--target", str(tmp_path), "--json"]) == 0 assert cli.main(["work", "closeout", "latest", "--target", str(tmp_path), "--json"]) == 0 assert seen == [ - ("verify-plan", {"target": tmp_path, "commands": ["python3 -m pytest -q"], "json_output": True}), + ( + "verify-plan", + {"target": tmp_path, "commands": ["python3 -m pytest -q"], "manifest_id": None, "json_output": True}, + ), ( "verify-run", { "target": tmp_path, "commands": ["python3 -m pytest -q"], + "manifest_id": None, "timeout": 12, "graphtrail_timeout": None, "json_output": True,