diff --git a/docs/command-inventory.md b/docs/command-inventory.md index 189e29b5..5c99b994 100644 --- a/docs/command-inventory.md +++ b/docs/command-inventory.md @@ -42,7 +42,7 @@ enabled: run `brigade extras on` once, or set `BRIGADE_EXTRAS=1`. - `brigade notifications` (extras): 4 command path(s) - `brigade openclaw-fragments` (extras): 1 command path(s) - `brigade operator`: 24 command path(s) -- `brigade outcome`: 11 command path(s) +- `brigade outcome`: 12 command path(s) - `brigade pantry` (extras): 5 command path(s) - `brigade profiles`: 2 command path(s) - `brigade projects` (extras): 10 command path(s) @@ -292,6 +292,7 @@ enabled: run `brigade extras on` once, or set `BRIGADE_EXTRAS=1`. - `brigade outcome rebuild-status` - `brigade outcome reconcile` - `brigade outcome record` +- `brigade outcome repair` - `brigade outcome score` - `brigade pantry doctor` (extras) - `brigade pantry expiry-alert` (extras) diff --git a/src/brigade/cli/outcome.py b/src/brigade/cli/outcome.py index d96e8e4d..46c004af 100644 --- a/src/brigade/cli/outcome.py +++ b/src/brigade/cli/outcome.py @@ -72,6 +72,19 @@ def register(sub: argparse._SubParsersAction) -> None: p_doctor.add_argument("--json", action="store_true", help="Emit machine-readable JSON instead of text.") p_doctor.set_defaults(func=_dispatch_doctor) + p_repair = outcome_sub.add_parser( + "repair", + help="Quarantine a broken completed outcome ledger and re-chain from the last valid record.", + ) + p_repair.add_argument("--target", "-t", type=Path, default=Path(".")) + p_repair.add_argument( + "--operator-confirm", + action="store_true", + help="Required confirmation; repair never runs silently.", + ) + p_repair.add_argument("--json", action="store_true", help="Emit machine-readable JSON instead of text.") + p_repair.set_defaults(func=_dispatch_repair) + p_rebuild = outcome_sub.add_parser( "rebuild-status", help="Rebuild status.json from decision receipts and report any drift." ) @@ -199,6 +212,16 @@ def _dispatch_doctor(args) -> int: return outcome_cmd.doctor(target=args.target, json_output=args.json) +def _dispatch_repair(args) -> int: + from .. import outcome_repair + + return outcome_repair.repair( + target=args.target, + operator_confirmed=args.operator_confirm, + json_output=args.json, + ) + + def _dispatch_rebuild_status(args) -> int: from .. import outcome_cmd diff --git a/src/brigade/outcome.py b/src/brigade/outcome.py index 19e8be41..145cab20 100644 --- a/src/brigade/outcome.py +++ b/src/brigade/outcome.py @@ -33,7 +33,7 @@ } # Sources whose every status is advisory/neutral regardless of value. -NEUTRAL_SOURCES = frozenset({"aboyeur", "replay"}) +NEUTRAL_SOURCES = frozenset({"aboyeur", "replay", "ledger-repair"}) @dataclass(frozen=True) diff --git a/src/brigade/outcome_cmd.py b/src/brigade/outcome_cmd.py index dcf6fe99..625e6ac5 100644 --- a/src/brigade/outcome_cmd.py +++ b/src/brigade/outcome_cmd.py @@ -592,34 +592,57 @@ def _last_record_digest(path: Path) -> str | None: return _validate_completed_ledger(path) +def _ledger_corrupt_message(line_no: int, kind: str, path: Path, *, detail: str = "") -> str: + """Bounded capture/append error that points operators at ``outcome repair``.""" + suffix = f" ({detail})" if detail else "" + return f"ledger corrupt at line {line_no}: {kind}{suffix}; run `brigade outcome repair --operator-confirm`: {path}" + + def _validate_completed_ledger_bytes(raw: bytes, path: Path) -> str | None: """Validate every completed ledger row in ``raw`` and return the last signed digest.""" if not raw: return None if not raw.endswith(b"\n"): - raise OutcomeLedgerError(f"outcome ledger has incomplete trailing record: {path}") + raise OutcomeLedgerError( + _ledger_corrupt_message( + raw.count(b"\n") + 1, + "incomplete trailing record", + path, + ) + ) previous_digest: str | None = None for line_no, line in enumerate(raw.splitlines(), start=1): if not line.strip(): - raise OutcomeLedgerError(f"outcome ledger line {line_no} is empty: {path}") + raise OutcomeLedgerError(_ledger_corrupt_message(line_no, "empty", path, detail="empty ledger line")) try: row = json.loads(line) except json.JSONDecodeError as exc: - raise OutcomeLedgerError(f"outcome ledger line {line_no} is not valid JSON: {path}: {exc}") from exc + raise OutcomeLedgerError( + _ledger_corrupt_message(line_no, "is not valid JSON", path, detail=str(exc)) + ) from exc if not isinstance(row, dict): - raise OutcomeLedgerError(f"outcome ledger line {line_no} is not an object: {path}") + raise OutcomeLedgerError( + _ledger_corrupt_message(line_no, "is not an object", path, detail="ledger line is not an object") + ) recorded_digest = row.get("digest") if not isinstance(recorded_digest, str) or not recorded_digest: continue recomputed = localio.canonical_json_digest(row, exclude_keys={"digest"}) if recomputed != recorded_digest: - raise OutcomeLedgerError(f"outcome ledger line {line_no} digest mismatch: {path}") + raise OutcomeLedgerError(_ledger_corrupt_message(line_no, "digest mismatch", path)) if "prev_digest" not in row: previous_digest = recorded_digest continue actual_prev = row.get("prev_digest") if actual_prev != previous_digest: - raise OutcomeLedgerError(f"outcome ledger line {line_no} digest chain break: {path}") + raise OutcomeLedgerError( + _ledger_corrupt_message( + line_no, + "digest chain break", + path, + detail=f"expected prev_digest={previous_digest!r}, actual={actual_prev!r}", + ) + ) previous_digest = recorded_digest return previous_digest @@ -1579,7 +1602,11 @@ def capture( route=route, route_fingerprint=route_fingerprint(route), ) - append_records(target, [record]) + try: + append_records(target, [record]) + except OutcomeLedgerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 if json_output: print(json.dumps({"target": str(target), "record": _record_payload(record)}, indent=2, sort_keys=True)) return 0 @@ -2054,9 +2081,43 @@ def sort_key(item: tuple[str, core.FingerprintCohorts]) -> tuple: def doctor(*, target: Path, json_output: bool = False) -> int: - from . import receipts_cmd + from . import outcome_repair, receipts_cmd - return receipts_cmd.doctor(target=target, json_output=json_output) + target = target.expanduser().resolve() + payload = receipts_cmd.verify_payload(target) + break_info = outcome_repair.diagnose_completed_ledger(_records_path(target)) + completed_ledger: dict[str, Any] + if break_info is None: + completed_ledger = {"status": "ok", "path": str(_records_path(target))} + else: + completed_ledger = { + "status": "corrupt", + "path": str(break_info.path), + "kind": break_info.kind, + "line_no": break_info.line_no, + "expected_prev": break_info.expected_prev, + "actual_prev": break_info.actual_prev, + "suspected_cause": break_info.suspected_cause, + "invalid_segment_start": break_info.invalid_segment_start, + "invalid_segment_end": break_info.invalid_segment_end, + "repair_command": "brigade outcome repair --operator-confirm", + } + payload["completed_ledger"] = completed_ledger + if json_output: + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + print(f"outcome doctor: {target}") + print(f"receipts: {receipts_cmd.summary_detail(target)}") + if break_info is None: + print("completed_ledger: ok") + else: + print( + f"completed_ledger: CORRUPT line={break_info.line_no} kind={break_info.kind} " + f"expected_prev={break_info.expected_prev!r} actual_prev={break_info.actual_prev!r}" + ) + print(f"suspected_cause: {break_info.suspected_cause}") + print("repair: brigade outcome repair --operator-confirm") + return 0 def record( @@ -2089,7 +2150,11 @@ def record( context=manifest, capability_fingerprint=capability_fingerprint(manifest), ) - append_records(target, [new_record]) + try: + append_records(target, [new_record]) + except OutcomeLedgerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 if json_output: print(json.dumps({"target": str(target), "record": _record_payload(new_record)}, indent=2, sort_keys=True)) return 0 diff --git a/src/brigade/outcome_repair.py b/src/brigade/outcome_repair.py new file mode 100644 index 00000000..80290d9f --- /dev/null +++ b/src/brigade/outcome_repair.py @@ -0,0 +1,578 @@ +"""Operator-only repair for a broken completed outcome ledger digest chain. + +Normal ledger writes remain append-only. This module is the exceptional +incident procedure when a completed ``records.jsonl`` fails chain validation: + +1. require explicit operator confirmation; +2. diagnose the first break (line, expected/actual digests, suspected cause); +3. quarantine the original ledger write-once and preserve the invalid segment; +4. replace the active ledger with the valid prefix plus a neutral repair record; +5. re-verify the full chain before returning. + +The quarantine is retained for audit. Standard library only. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import os +import sys +from pathlib import Path +from typing import Any + +from brigade import localio, outcome as core, outcome_cmd + +REPAIR_SCHEMA = "brigade.outcome_ledger_repair.v1" +REPAIR_SCHEMA_VERSION = 1 +REPAIR_SOURCE = "ledger-repair" +REPAIR_ARTIFACT_ID = "outcome-ledger" + + +@dataclasses.dataclass(frozen=True) +class LedgerBreak: + """First integrity failure in a completed outcome ledger.""" + + path: Path + kind: str + line_no: int + expected_prev: str | None + actual_prev: str | None + suspected_cause: str + valid_prefix_bytes: bytes + invalid_segment_bytes: bytes + valid_prefix_lines: int + invalid_segment_start: int + invalid_segment_end: int + last_valid_digest: str | None + + +@dataclasses.dataclass(frozen=True) +class LedgerRepairReport: + """Paths and bounds for one ledger repair.""" + + operation_id: str + break_line: int + kind: str + suspected_cause: str + quarantine_path: Path + invalid_segment_path: Path + record_path: Path + valid_prefix_lines: int + invalid_segment_start: int + invalid_segment_end: int + re_chained_record_count: int + re_chained_line_ranges: tuple[tuple[int, int], ...] + + +@dataclasses.dataclass(frozen=True) +class RechainedTail: + """Verified records salvaged from the raw segment after the first break.""" + + rows: list[dict[str, Any]] + line_ranges: tuple[tuple[int, int], ...] + last_digest: str | None + + +def _payload_without_digests(row: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in row.items() if key not in {"digest", "prev_digest"}} + + +def _suspect_duplicate_writer(previous_row: dict[str, Any] | None, broken_row: dict[str, Any]) -> bool: + if previous_row is None: + return False + return _payload_without_digests(previous_row) == _payload_without_digests(broken_row) + + +def _break( + *, + path: Path, + kind: str, + line_no: int, + expected_prev: str | None, + actual_prev: str | None, + suspected_cause: str, + raw: bytes, + offset: int, + end_line: int, + last_valid_digest: str | None, +) -> LedgerBreak: + return LedgerBreak( + path=path, + kind=kind, + line_no=line_no, + expected_prev=expected_prev, + actual_prev=actual_prev, + suspected_cause=suspected_cause, + valid_prefix_bytes=raw[:offset], + invalid_segment_bytes=raw[offset:], + valid_prefix_lines=raw[:offset].count(b"\n"), + invalid_segment_start=line_no, + invalid_segment_end=end_line, + last_valid_digest=last_valid_digest, + ) + + +def _diagnose_completed_lines(path: Path, raw: bytes) -> LedgerBreak | None: + """Diagnose newline-terminated ledger bytes; return None when the chain is healthy.""" + previous_digest: str | None = None + previous_row: dict[str, Any] | None = None + offset = 0 + lines = raw.splitlines(keepends=True) + end_line = len(lines) + for line_no, line in enumerate(lines, start=1): + line_bytes = line if line.endswith(b"\n") else line + b"\n" + if not line.strip(): + return _break( + path=path, + kind="empty_line", + line_no=line_no, + expected_prev=previous_digest, + actual_prev=None, + suspected_cause="empty ledger line", + raw=raw, + offset=offset, + end_line=end_line, + last_valid_digest=previous_digest, + ) + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + return _break( + path=path, + kind="invalid_json", + line_no=line_no, + expected_prev=previous_digest, + actual_prev=None, + suspected_cause=f"invalid JSON ({exc})", + raw=raw, + offset=offset, + end_line=end_line, + last_valid_digest=previous_digest, + ) + if not isinstance(row, dict): + return _break( + path=path, + kind="invalid_json", + line_no=line_no, + expected_prev=previous_digest, + actual_prev=None, + suspected_cause="ledger line is not an object", + raw=raw, + offset=offset, + end_line=end_line, + last_valid_digest=previous_digest, + ) + recorded_digest = row.get("digest") + if not isinstance(recorded_digest, str) or not recorded_digest: + previous_row = row + offset += len(line_bytes) + continue + recomputed = localio.canonical_json_digest(row, exclude_keys={"digest"}) + if recomputed != recorded_digest: + return _break( + path=path, + kind="digest_mismatch", + line_no=line_no, + expected_prev=previous_digest, + actual_prev=str(row.get("prev_digest")) if "prev_digest" in row else None, + suspected_cause="tampered or rewritten record", + raw=raw, + offset=offset, + end_line=end_line, + last_valid_digest=previous_digest, + ) + if "prev_digest" not in row: + previous_digest = recorded_digest + previous_row = row + offset += len(line_bytes) + continue + actual_prev = row.get("prev_digest") + if actual_prev != previous_digest: + suspected = ( + "duplicate-writer records" + if _suspect_duplicate_writer(previous_row, row) + else "digest chain discontinuity" + ) + return _break( + path=path, + kind="digest_chain_break", + line_no=line_no, + expected_prev=previous_digest, + actual_prev=None if actual_prev is None else str(actual_prev), + suspected_cause=suspected, + raw=raw, + offset=offset, + end_line=end_line, + last_valid_digest=previous_digest, + ) + previous_digest = recorded_digest + previous_row = row + offset += len(line_bytes) + return None + + +def diagnose_completed_ledger(path: Path) -> LedgerBreak | None: + """Return the first completed-ledger integrity break, or None when healthy.""" + if not path.is_file(): + return None + try: + raw = path.read_bytes() + except OSError as exc: + raise outcome_cmd.OutcomeLedgerError(f"could not read outcome ledger: {path}: {exc}") from exc + if not raw: + return None + + if raw.endswith(b"\n"): + return _diagnose_completed_lines(path, raw) + + last_newline = raw.rfind(b"\n") + if last_newline < 0: + return _break( + path=path, + kind="incomplete_trailing", + line_no=1, + expected_prev=None, + actual_prev=None, + suspected_cause="incomplete trailing record", + raw=raw, + offset=0, + end_line=1, + last_valid_digest=None, + ) + + prefix = raw[: last_newline + 1] + prefix_break = _diagnose_completed_lines(path, prefix) + if prefix_break is not None: + # Keep the incomplete tail inside the preserved invalid segment. + return LedgerBreak( + path=path, + kind=prefix_break.kind, + line_no=prefix_break.line_no, + expected_prev=prefix_break.expected_prev, + actual_prev=prefix_break.actual_prev, + suspected_cause=prefix_break.suspected_cause, + valid_prefix_bytes=prefix_break.valid_prefix_bytes, + invalid_segment_bytes=raw[len(prefix_break.valid_prefix_bytes) :], + valid_prefix_lines=prefix_break.valid_prefix_lines, + invalid_segment_start=prefix_break.invalid_segment_start, + invalid_segment_end=prefix.count(b"\n") + 1, + last_valid_digest=prefix_break.last_valid_digest, + ) + + try: + last_digest = outcome_cmd._validate_completed_ledger_bytes(prefix, path) + except outcome_cmd.OutcomeLedgerError: + last_digest = None + start_line = prefix.count(b"\n") + 1 + return _break( + path=path, + kind="incomplete_trailing", + line_no=start_line, + expected_prev=last_digest, + actual_prev=None, + suspected_cause="incomplete trailing record", + raw=raw, + offset=len(prefix), + end_line=start_line, + last_valid_digest=last_digest, + ) + + +def _operation_id(original: bytes, break_info: LedgerBreak) -> str: + material = { + "sha256": hashlib.sha256(original).hexdigest(), + "line_no": break_info.line_no, + "kind": break_info.kind, + } + encoded = json.dumps(material, sort_keys=True, separators=(",", ":")).encode("utf-8") + return f"repair-{hashlib.sha256(encoded).hexdigest()[:16]}" + + +def _repair_paths(target: Path, operation_id: str) -> tuple[Path, Path, Path, Path]: + # Keep quarantine under gitignored .brigade/ so operators can retain the + # original corrupted ledger for audit without committing it. + operation_dir = target / ".brigade" / "outcome" / "repairs" / operation_id + return ( + operation_dir, + operation_dir / "original.jsonl", + operation_dir / "invalid-segment.jsonl", + operation_dir / "record.json", + ) + + +def _line_ranges(line_numbers: list[int]) -> tuple[tuple[int, int], ...]: + if not line_numbers: + return () + ranges: list[tuple[int, int]] = [] + start = end = line_numbers[0] + for line_no in line_numbers[1:]: + if line_no == end + 1: + end = line_no + continue + ranges.append((start, end)) + start = end = line_no + ranges.append((start, end)) + return tuple(ranges) + + +def _rechain_tail(break_info: LedgerBreak) -> RechainedTail: + """Re-sign self-consistent records after the first break onto the valid prefix.""" + previous_digest = break_info.last_valid_digest + rows: list[dict[str, Any]] = [] + line_numbers: list[int] = [] + for line_no, line in enumerate( + break_info.invalid_segment_bytes.splitlines(keepends=True), + start=break_info.invalid_segment_start, + ): + if not line.strip(): + continue + try: + row = json.loads(line) + except (UnicodeDecodeError, json.JSONDecodeError): + continue + if not isinstance(row, dict): + continue + recorded_digest = row.get("digest") + if recorded_digest is not None and ( + not isinstance(recorded_digest, str) + or not recorded_digest + or localio.canonical_json_digest(row, exclude_keys={"digest"}) != recorded_digest + ): + continue + re_chained = _payload_without_digests(row) + re_chained["prev_digest"] = previous_digest + re_chained["digest"] = localio.canonical_json_digest(re_chained, exclude_keys={"digest"}) + previous_digest = re_chained["digest"] + rows.append(re_chained) + line_numbers.append(line_no) + return RechainedTail(rows, _line_ranges(line_numbers), previous_digest) + + +def _build_repair_record( + break_info: LedgerBreak, + *, + operation_id: str, + quarantine_path: Path, + re_chained_tail: RechainedTail, +) -> core.OutcomeRecord: + return core.OutcomeRecord( + artifact_id=REPAIR_ARTIFACT_ID, + artifact_kind="skill", + task_id=f"{operation_id}:lines-{break_info.invalid_segment_start}-{break_info.invalid_segment_end}", + source=REPAIR_SOURCE, + signal_value=0, + evidence_ref=str(quarantine_path), + ts=localio.utc_now_iso(), + context={ + "schema": REPAIR_SCHEMA, + "schema_version": REPAIR_SCHEMA_VERSION, + "kind": break_info.kind, + "break_line": break_info.line_no, + "suspected_cause": break_info.suspected_cause, + "invalid_segment_start": break_info.invalid_segment_start, + "invalid_segment_end": break_info.invalid_segment_end, + "expected_prev": break_info.expected_prev, + "actual_prev": break_info.actual_prev, + "re_chained_record_count": len(re_chained_tail.rows), + "re_chained_line_ranges": re_chained_tail.line_ranges, + }, + ) + + +def _write_repaired_ledger( + path: Path, + prefix: bytes, + re_chained_tail: RechainedTail, + repair_row: dict[str, Any], +) -> None: + rows = [*re_chained_tail.rows, repair_row] + body = b"".join((json.dumps(row, sort_keys=True) + "\n").encode("utf-8") for row in rows) + localio.write_bytes_atomic(path, prefix + body) + + +def _publish_exclusive_bytes(path: Path, data: bytes) -> None: + """Publish byte-identical quarantine evidence without replacing an existing file.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + except BaseException: + path.unlink(missing_ok=True) + raise + + +def repair_ledger( + *, + target: Path, + operator_confirmed: bool = False, + json_output: bool = False, +) -> int: + """Quarantine a broken completed ledger, keep the valid prefix, append a repair record.""" + if operator_confirmed is not True: + print("error: operator confirmation is required (--operator-confirm)", file=sys.stderr) + return 2 + target = target.expanduser().resolve() + path = outcome_cmd._records_path(target) + lock_path = outcome_cmd._records_lock_path(target) + if not path.is_file(): + print(f"outcome repair: no ledger at {path}") + return 0 + + with outcome_cmd._records_append_lock(lock_path): + try: + original = path.read_bytes() + except OSError as exc: + print(f"error: could not read outcome ledger: {path}: {exc}", file=sys.stderr) + return 1 + break_info = diagnose_completed_ledger(path) + if break_info is None: + if json_output: + print(json.dumps({"target": str(target), "status": "healthy"}, indent=2, sort_keys=True)) + else: + print(f"outcome repair: ledger healthy ({path})") + return 0 + + operation_id = _operation_id(original, break_info) + operation_dir, quarantine_path, invalid_path, record_path = _repair_paths(target, operation_id) + operation_dir.mkdir(parents=True, exist_ok=True) + + if quarantine_path.exists(): + try: + existing = quarantine_path.read_bytes() + except OSError as exc: + print(f"error: could not read quarantine: {quarantine_path}: {exc}", file=sys.stderr) + return 1 + if existing != original: + print( + f"error: repair quarantine already exists with different bytes: {quarantine_path}", + file=sys.stderr, + ) + return 1 + else: + try: + _publish_exclusive_bytes(quarantine_path, original) + except FileExistsError: + print(f"error: repair quarantine already exists: {quarantine_path}", file=sys.stderr) + return 1 + except OSError as exc: + print(f"error: could not quarantine ledger: {exc}", file=sys.stderr) + return 1 + + if not invalid_path.exists(): + try: + _publish_exclusive_bytes(invalid_path, break_info.invalid_segment_bytes) + except FileExistsError: + pass + except OSError as exc: + print(f"error: could not preserve invalid segment: {exc}", file=sys.stderr) + return 1 + + try: + localio._fsync_parent_directory(operation_dir) + except OSError as exc: + print(f"error: could not durably publish repair quarantine: {exc}", file=sys.stderr) + return 1 + + if break_info.kind == "incomplete_trailing": + # A partial append can now be recovered because its original bytes + # are safely quarantined. A prior chain break remains authoritative, + # including any trailing bytes it already placed in the evidence. + outcome_cmd._recover_interrupted_append(path) + break_info = diagnose_completed_ledger(path) + if break_info is None: + if json_output: + print(json.dumps({"target": str(target), "status": "recovered"}, indent=2, sort_keys=True)) + else: + print(f"outcome repair: ledger healthy after preserving interrupted bytes ({path})") + return 0 + + re_chained_tail = _rechain_tail(break_info) + repair_record = _build_repair_record( + break_info, + operation_id=operation_id, + quarantine_path=quarantine_path, + re_chained_tail=re_chained_tail, + ) + row = outcome_cmd._record_payload(repair_record) + row["prev_digest"] = re_chained_tail.last_digest + row["digest"] = localio.canonical_json_digest(row, exclude_keys={"digest"}) + + audit = { + "schema": REPAIR_SCHEMA, + "schema_version": REPAIR_SCHEMA_VERSION, + "operation_id": operation_id, + "break_line": break_info.line_no, + "kind": break_info.kind, + "suspected_cause": break_info.suspected_cause, + "expected_prev": break_info.expected_prev, + "actual_prev": break_info.actual_prev, + "valid_prefix_lines": break_info.valid_prefix_lines, + "invalid_segment_start": break_info.invalid_segment_start, + "invalid_segment_end": break_info.invalid_segment_end, + "quarantine_path": str(quarantine_path), + "invalid_segment_path": str(invalid_path), + "repair_digest": row["digest"], + "re_chained_record_count": len(re_chained_tail.rows), + "re_chained_line_ranges": [list(bounds) for bounds in re_chained_tail.line_ranges], + } + if not record_path.exists(): + try: + localio.write_json_exclusive(record_path, audit) + except FileExistsError: + pass + except OSError as exc: + print(f"error: could not write repair record: {exc}", file=sys.stderr) + return 1 + + try: + _write_repaired_ledger(path, break_info.valid_prefix_bytes, re_chained_tail, row) + outcome_cmd._validate_completed_ledger(path) + except outcome_cmd.OutcomeLedgerError as exc: + print(f"error: repaired ledger failed verification: {exc}", file=sys.stderr) + return 1 + except OSError as exc: + print(f"error: could not write repaired ledger: {exc}", file=sys.stderr) + return 1 + + report = LedgerRepairReport( + operation_id=operation_id, + break_line=break_info.line_no, + kind=break_info.kind, + suspected_cause=break_info.suspected_cause, + quarantine_path=quarantine_path, + invalid_segment_path=invalid_path, + record_path=record_path, + valid_prefix_lines=break_info.valid_prefix_lines, + invalid_segment_start=break_info.invalid_segment_start, + invalid_segment_end=break_info.invalid_segment_end, + re_chained_record_count=len(re_chained_tail.rows), + re_chained_line_ranges=re_chained_tail.line_ranges, + ) + if json_output: + print(json.dumps(dataclasses.asdict(report), indent=2, sort_keys=True, default=str)) + return 0 + print(f"outcome repair: repaired {path}") + print(f"operation: {report.operation_id}") + print(f"break: line {report.break_line} [{report.kind}] suspected={report.suspected_cause}") + print(f"quarantine: {report.quarantine_path}") + print(f"invalid_segment: {report.invalid_segment_path}") + print(f"record: {report.record_path}") + print(f"kept_prefix_lines: {report.valid_prefix_lines}") + print(f"re_chained_records: {report.re_chained_record_count}") + print(f"re_chained_line_ranges: {list(report.re_chained_line_ranges)}") + return 0 + + +def repair( + *, + target: Path, + operator_confirmed: bool = False, + json_output: bool = False, +) -> int: + """CLI entry point alias.""" + return repair_ledger(target=target, operator_confirmed=operator_confirmed, json_output=json_output) diff --git a/tests/test_outcome_repair.py b/tests/test_outcome_repair.py new file mode 100644 index 00000000..e313a9fa --- /dev/null +++ b/tests/test_outcome_repair.py @@ -0,0 +1,380 @@ +"""Regression coverage for completed-ledger diagnosis and sanctioned repair (#639).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from brigade import cli, localio, outcome, outcome_cmd, outcome_repair + +from tests.work_cmd_test_helpers import _init_git_repo + + +def _signed_row(record: outcome.OutcomeRecord, prev_digest: str | None) -> dict: + row = outcome_cmd._record_payload(record) + row["prev_digest"] = prev_digest + row["digest"] = localio.canonical_json_digest(row, exclude_keys={"digest"}) + return row + + +def _write_rows(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows)) + + +def _seed_two_valid(tmp_path: Path) -> tuple[Path, list[dict]]: + first = outcome.OutcomeRecord("skill-x", "skill", "t0", "verify", 1, "ref-0", "2026-06-20T00:00:00+00:00") + second = outcome.OutcomeRecord("skill-x", "skill", "t1", "verify", 1, "ref-1", "2026-06-20T01:00:00+00:00") + outcome_cmd.append_records(tmp_path, [first, second]) + path = tmp_path / "memory" / "outcome" / "records.jsonl" + rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + return path, rows + + +def test_diagnose_duplicate_record_chain_break(tmp_path): + path, rows = _seed_two_valid(tmp_path) + # Competing writers: append a near-identical payload that still points at the + # first digest instead of the second (classic duplicate-writer break). + duplicate = dict(rows[1]) + duplicate["prev_digest"] = rows[0]["digest"] + duplicate["evidence_ref"] = rows[1]["evidence_ref"] + duplicate["digest"] = localio.canonical_json_digest(duplicate, exclude_keys={"digest"}) + _write_rows(path, [rows[0], rows[1], duplicate]) + + break_info = outcome_repair.diagnose_completed_ledger(path) + assert break_info is not None + assert break_info.kind == "digest_chain_break" + assert break_info.line_no == 3 + assert break_info.expected_prev == rows[1]["digest"] + assert break_info.actual_prev == rows[0]["digest"] + assert break_info.suspected_cause == "duplicate-writer records" + + +def test_diagnose_truncated_line_break(tmp_path): + path, rows = _seed_two_valid(tmp_path) + path.write_bytes(path.read_bytes() + b'{"artifact_id":"skill-x","digest":"pending"') + + break_info = outcome_repair.diagnose_completed_ledger(path) + assert break_info is not None + assert break_info.kind == "incomplete_trailing" + assert break_info.line_no == 3 + assert ( + break_info.valid_prefix_bytes + == (json.dumps(rows[0], sort_keys=True) + "\n").encode() + (json.dumps(rows[1], sort_keys=True) + "\n").encode() + ) + assert break_info.suspected_cause == "incomplete trailing record" + + +def test_doctor_reports_completed_ledger_chain_break(tmp_path, capsys): + path, rows = _seed_two_valid(tmp_path) + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="not-the-previous-digest", + ) + _write_rows(path, [rows[0], rows[1], broken]) + + assert cli.main(["outcome", "doctor", "--target", str(tmp_path), "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["completed_ledger"]["status"] == "corrupt" + assert payload["completed_ledger"]["line_no"] == 3 + assert payload["completed_ledger"]["kind"] == "digest_chain_break" + assert "outcome repair" in payload["completed_ledger"]["repair_command"] + + assert cli.main(["outcome", "doctor", "--target", str(tmp_path)]) == 0 + text = capsys.readouterr().out + assert "completed_ledger: CORRUPT line=3" in text + assert "repair: brigade outcome repair --operator-confirm" in text + + +def test_repair_requires_operator_confirmation(tmp_path, capsys): + path, rows = _seed_two_valid(tmp_path) + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="wrong", + ) + _write_rows(path, [rows[0], rows[1], broken]) + before = path.read_bytes() + + assert cli.main(["outcome", "repair", "--target", str(tmp_path)]) == 2 + err = capsys.readouterr().err + assert "operator confirmation is required" in err + assert path.read_bytes() == before + + +def test_repair_quarantines_rechains_tail_and_reverify(tmp_path): + path, rows = _seed_two_valid(tmp_path) + original_prefix = path.read_bytes() + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest=rows[0]["digest"], + ) + # Make the broken payload match row 1 so diagnosis flags duplicate-writer. + for key in ("artifact_id", "artifact_kind", "task_id", "source", "signal_value", "evidence_ref", "ts"): + broken[key] = rows[1][key] + broken["digest"] = localio.canonical_json_digest(broken, exclude_keys={"digest"}) + _write_rows(path, [rows[0], rows[1], broken]) + original = path.read_bytes() + + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True, json_output=False) == 0 + + repaired_rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert len(repaired_rows) == 4 + assert repaired_rows[0] == rows[0] + assert repaired_rows[1] == rows[1] + assert repaired_rows[2]["task_id"] == broken["task_id"] + assert repaired_rows[2]["prev_digest"] == rows[1]["digest"] + assert repaired_rows[2]["digest"] == localio.canonical_json_digest(repaired_rows[2], exclude_keys={"digest"}) + assert repaired_rows[3]["source"] == "ledger-repair" + assert repaired_rows[3]["signal_value"] == 0 + assert repaired_rows[3]["prev_digest"] == repaired_rows[2]["digest"] + assert repaired_rows[3]["digest"] == localio.canonical_json_digest(repaired_rows[3], exclude_keys={"digest"}) + assert path.read_bytes().startswith(original_prefix) + + quarantine_dirs = list((tmp_path / ".brigade" / "outcome" / "repairs").iterdir()) + assert len(quarantine_dirs) == 1 + quarantine = quarantine_dirs[0] / "original.jsonl" + invalid = quarantine_dirs[0] / "invalid-segment.jsonl" + record = quarantine_dirs[0] / "record.json" + assert quarantine.read_bytes() == original + assert invalid.read_bytes() == (json.dumps(broken, sort_keys=True) + "\n").encode() + audit = json.loads(record.read_text()) + assert audit["kind"] == "digest_chain_break" + assert audit["suspected_cause"] == "duplicate-writer records" + assert audit["break_line"] == 3 + assert audit["re_chained_record_count"] == 1 + assert audit["re_chained_line_ranges"] == [[3, 3]] + + assert outcome_cmd._validate_completed_ledger(path) == repaired_rows[3]["digest"] + assert outcome_repair.diagnose_completed_ledger(path) is None + + +def test_repair_rechains_valid_records_after_break(tmp_path): + path, rows = _seed_two_valid(tmp_path) + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="wrong", + ) + tail = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t3", "verify", 1, "ref-3", "2026-06-20T03:00:00+00:00"), + prev_digest=broken["digest"], + ) + _write_rows(path, [rows[0], rows[1], broken, tail]) + + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True) == 0 + + repaired_rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert [row["task_id"] for row in repaired_rows] == ["t0", "t1", "t2", "t3", repaired_rows[-1]["task_id"]] + assert repaired_rows[-1]["source"] == "ledger-repair" + assert repaired_rows[2]["prev_digest"] == rows[1]["digest"] + assert repaired_rows[3]["prev_digest"] == repaired_rows[2]["digest"] + audit_path = next((tmp_path / ".brigade" / "outcome" / "repairs").glob("*/record.json")) + audit = json.loads(audit_path.read_text()) + assert audit["re_chained_record_count"] == 2 + assert audit["re_chained_line_ranges"] == [[3, 4]] + assert outcome_cmd._validate_completed_ledger(path) == repaired_rows[-1]["digest"] + + +def test_repair_rechains_from_a_first_line_break(tmp_path): + path, rows = _seed_two_valid(tmp_path) + first = dict(rows[0]) + first["prev_digest"] = "wrong" + first["digest"] = localio.canonical_json_digest(first, exclude_keys={"digest"}) + third = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest=rows[1]["digest"], + ) + _write_rows(path, [first, rows[1], third]) + + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True) == 0 + + repaired_rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert [row["task_id"] for row in repaired_rows[:-1]] == ["t0", "t1", "t2"] + assert repaired_rows[0]["prev_digest"] is None + assert repaired_rows[1]["prev_digest"] == repaired_rows[0]["digest"] + assert repaired_rows[2]["prev_digest"] == repaired_rows[1]["digest"] + assert outcome_cmd._validate_completed_ledger(path) == repaired_rows[-1]["digest"] + + +def test_repair_rechains_across_multiple_chain_breaks(tmp_path): + path, rows = _seed_two_valid(tmp_path) + first_break = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="wrong-a", + ) + second_break = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t3", "verify", 1, "ref-3", "2026-06-20T03:00:00+00:00"), + prev_digest="wrong-b", + ) + tail = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t4", "verify", 1, "ref-4", "2026-06-20T04:00:00+00:00"), + prev_digest=second_break["digest"], + ) + _write_rows(path, [rows[0], rows[1], first_break, second_break, tail]) + + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True) == 0 + + repaired_rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert [row["task_id"] for row in repaired_rows[:-1]] == ["t0", "t1", "t2", "t3", "t4"] + assert all( + repaired_rows[index]["prev_digest"] == repaired_rows[index - 1]["digest"] + for index in range(1, len(repaired_rows)) + ) + audit_path = next((tmp_path / ".brigade" / "outcome" / "repairs").glob("*/record.json")) + audit = json.loads(audit_path.read_text()) + assert audit["re_chained_record_count"] == 3 + assert audit["re_chained_line_ranges"] == [[3, 5]] + assert outcome_cmd._validate_completed_ledger(path) == repaired_rows[-1]["digest"] + + +def test_repair_skips_binary_bytes_and_rechains_later_valid_records(tmp_path): + path, rows = _seed_two_valid(tmp_path) + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="wrong", + ) + tail = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t3", "verify", 1, "ref-3", "2026-06-20T03:00:00+00:00"), + prev_digest=broken["digest"], + ) + path.write_bytes( + b"".join( + [ + *(json.dumps(row, sort_keys=True).encode() + b"\n" for row in rows), + json.dumps(broken, sort_keys=True).encode() + b"\n", + b"\xff\n", + json.dumps(tail, sort_keys=True).encode() + b"\n", + ] + ) + ) + + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True) == 0 + + repaired_rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert [row["task_id"] for row in repaired_rows[:-1]] == ["t0", "t1", "t2", "t3"] + record_path = next((tmp_path / ".brigade" / "outcome" / "repairs").glob("*/record.json")) + audit = json.loads(record_path.read_text()) + assert audit["re_chained_line_ranges"] == [[3, 3], [5, 5]] + assert outcome_cmd._validate_completed_ledger(path) == repaired_rows[-1]["digest"] + + +def test_repair_keeps_the_original_tail_range_when_chain_break_precedes_partial_bytes(tmp_path): + path, rows = _seed_two_valid(tmp_path) + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="wrong", + ) + partial = b'{"artifact_id":"partial"' + _write_rows(path, [rows[0], rows[1], broken]) + path.write_bytes(path.read_bytes() + partial) + + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True) == 0 + + record_path = next((tmp_path / ".brigade" / "outcome" / "repairs").glob("*/record.json")) + audit = json.loads(record_path.read_text()) + quarantine = record_path.parent / "original.jsonl" + assert audit["invalid_segment_end"] == 4 + assert quarantine.read_bytes().endswith(partial) + + +def test_capture_degrades_with_bounded_error_and_writes_nothing(tmp_path, capsys): + _init_git_repo(tmp_path) + path, rows = _seed_two_valid(tmp_path) + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="wrong", + ) + _write_rows(path, [rows[0], rows[1], broken]) + before = path.read_bytes() + _write_verify_receipt = tmp_path / ".brigade" / "work" / "verify-runs" / "v1" + _write_verify_receipt.mkdir(parents=True) + (_write_verify_receipt / "receipt.json").write_text( + json.dumps( + { + "run_id": "v1", + "target": str(tmp_path), + "status": "completed", + "started_at": "2026-06-20T03:00:00+00:00", + "completed_at": "2026-06-20T03:00:00+00:00", + "commands": [], + "path": str(_write_verify_receipt), + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + rc = outcome_cmd.capture(target=tmp_path, artifact_id="skill-x", run_id="v1") + captured = capsys.readouterr() + assert rc == 1 + assert "ledger corrupt at line 3" in captured.err + assert "brigade outcome repair --operator-confirm" in captured.err + assert path.read_bytes() == before + + +def test_post_repair_capture_succeeds(tmp_path, capsys): + _init_git_repo(tmp_path) + path, rows = _seed_two_valid(tmp_path) + broken = _signed_row( + outcome.OutcomeRecord("skill-x", "skill", "t2", "verify", 1, "ref-2", "2026-06-20T02:00:00+00:00"), + prev_digest="wrong", + ) + _write_rows(path, [rows[0], rows[1], broken]) + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True) == 0 + + run_dir = tmp_path / ".brigade" / "work" / "verify-runs" / "v2" + run_dir.mkdir(parents=True) + (run_dir / "receipt.json").write_text( + json.dumps( + { + "run_id": "v2", + "target": str(tmp_path), + "status": "completed", + "started_at": "2026-06-20T04:00:00+00:00", + "completed_at": "2026-06-20T04:00:00+00:00", + "commands": [], + "path": str(run_dir), + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + assert outcome_cmd.capture(target=tmp_path, artifact_id="skill-x", run_id="v2") == 0 + out = capsys.readouterr().out + assert "outcome capture: skill-x" in out + assert outcome_repair.diagnose_completed_ledger(path) is None + final_rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert final_rows[-1]["evidence_ref"].endswith("receipt.json") + assert final_rows[-1]["prev_digest"] == final_rows[-2]["digest"] + + +def test_repair_quarantines_incomplete_trailing_bytes_before_recovery(tmp_path, capsys): + path, rows = _seed_two_valid(tmp_path) + prefix = path.read_bytes() + path.write_bytes(prefix + b'{"artifact_id":"truncated"') + + assert outcome_repair.repair(target=tmp_path, operator_confirmed=True) == 0 + out = capsys.readouterr().out + assert "ledger healthy" in out + assert path.read_bytes() == prefix + assert outcome_repair.diagnose_completed_ledger(path) is None + rows_after = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + assert rows_after == rows + quarantine = next((tmp_path / ".brigade" / "outcome" / "repairs").glob("*/original.jsonl")) + assert quarantine.read_bytes() == prefix + b'{"artifact_id":"truncated"' + + +def test_publish_exclusive_bytes_preserves_non_utf8_bytes(tmp_path): + path = tmp_path / "quarantine" / "original.jsonl" + original = b'\xff\r\n{"partial":"tail"}' + + outcome_repair._publish_exclusive_bytes(path, original) + + assert path.read_bytes() == original + with pytest.raises(FileExistsError): + outcome_repair._publish_exclusive_bytes(path, b"replacement") + assert path.read_bytes() == original