feat(roster): add capability suggestions and receipt stats (#463)

* feat(roster): add capability suggestions and receipt stats

Co-Authored-By: Codex <codex@openai.com>

* fix(ci): stabilize Ruff and command inventory checks

Co-Authored-By: Codex <codex@openai.com>

* fix(ci): support Python 3.10 in Ruff config test

Co-Authored-By: Codex <codex@openai.com>

---------

Co-authored-by: Codex <codex@openai.com>

Author: solomonneas

docs/command-inventory.md

@@ -52,7 +52,7 @@ enabled: run `brigade extras on` once, or set `BRIGADE_EXTRAS=1`.
 - `brigade repos` (extras): 74 command path(s)
 - `brigade research` (extras): 11 command path(s)
 - `brigade roadmap` (extras): 4 command path(s)
-- `brigade roster`: 2 command path(s)
+- `brigade roster`: 4 command path(s)
 - `brigade route`: 1 command path(s)
 - `brigade run`: 1 command path(s)
 - `brigade runbook` (extras): 5 command path(s)
@@ -428,6 +428,8 @@ enabled: run `brigade extras on` once, or set `BRIGADE_EXTRAS=1`.
 - `brigade roadmap patterns` (extras)
 - `brigade roster doctor`
 - `brigade roster init`
+- `brigade roster stats`
+- `brigade roster suggest`
 - `brigade route`
 - `brigade run`
 - `brigade runbook closeout` (extras)

pyproject.toml

@@ -63,7 +63,7 @@ addopts = "-ra"
 [tool.ruff]
 line-length = 120
 src = ["src", "tests"]
-extend-exclude = ["engines"]
+extend-exclude = ["engines", "*.md"]
 force-exclude = true
 
 [tool.ruff.lint]

src/brigade/cli/roster.py

@@ -35,6 +35,18 @@ def register(sub: argparse._SubParsersAction) -> None:
         default=None,
         help="Path to roster.toml. Defaults to .brigade/roster.toml under --target.",
     )
+    p_roster_suggest = roster_sub.add_parser(
+        "suggest",
+        help="Assemble a preset roster from installed host capabilities.",
+    )
+    p_roster_suggest.add_argument(
+        "--preset",
+        required=True,
+        help="Packaged preset name or path (with or without .toml).",
+    )
+    p_roster_suggest.add_argument("--target", "-t", type=Path, default=Path("."))
+    p_roster_stats = roster_sub.add_parser("stats", help="Summarize per-seat stats from local worker receipts.")
+    p_roster_stats.add_argument("--target", "-t", type=Path, default=Path("."))
     p_roster.set_defaults(func=dispatch)
 
 
@@ -51,5 +63,9 @@ def dispatch(args) -> int:
         )
     if args.roster_command == "doctor":
         return roster_cmd.doctor(target=args.target, roster_path=args.roster)
+    if args.roster_command == "suggest":
+        return roster_cmd.suggest(target=args.target, preset=args.preset)
+    if args.roster_command == "stats":
+        return roster_cmd.stats(target=args.target)
     args._brigade_parser.error(f"unknown roster command: {args.roster_command}")
     return 2

src/brigade/roster.py

@@ -3,7 +3,9 @@
 from __future__ import annotations
 
 import fnmatch
+import json
 import re
+import statistics
 from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Literal, Protocol
@@ -92,6 +94,13 @@ def usable(self) -> bool:
         return self.roster.orchestrator in self.roster.agents
 
 
+@dataclass(frozen=True)
+class SeatReceiptStats:
+    sample_count: int
+    median_duration_seconds: float
+    failure_rate: float
+
+
 @dataclass(frozen=True)
 class HostCapabilityProbe:
     def lookup(self, cli_ref: str) -> Capability:
@@ -695,3 +704,76 @@ def resolve_capabilities(
         roster=replace(roster, agents=resolved_agents),
         report=tuple(report),
     )
+
+
+def _worker_result_failed(row: dict[str, object]) -> bool:
+    ok = row.get("ok")
+    if ok is False:
+        return True
+    if ok is True:
+        return False
+    status = row.get("status")
+    if isinstance(status, str):
+        normalized = status.strip().lower()
+        if normalized in {"failed", "error", "fail", "failure"}:
+            return True
+        if normalized in {"ok", "success", "passed", "complete", "completed"}:
+            return False
+    exit_code = row.get("exit_code")
+    if isinstance(exit_code, int) and not isinstance(exit_code, bool) and exit_code != 0:
+        return True
+    return False
+
+
+def collect_seat_receipt_stats(runs_root: Path) -> dict[str, SeatReceiptStats]:
+    """Aggregate per-seat worker receipt durations and failure rates from local runs."""
+
+    runs_root = runs_root.expanduser()
+    if not runs_root.is_dir():
+        return {}
+
+    durations: dict[str, list[float]] = {}
+    samples: dict[str, int] = {}
+    failures: dict[str, int] = {}
+
+    for run_dir in sorted(runs_root.iterdir()):
+        if not run_dir.is_dir():
+            continue
+        path = run_dir / "worker-results.json"
+        if not path.is_file():
+            continue
+        try:
+            payload = json.loads(path.read_text())
+        except (OSError, json.JSONDecodeError, UnicodeDecodeError):
+            continue
+        if not isinstance(payload, dict):
+            continue
+        results = payload.get("results")
+        if not isinstance(results, list):
+            continue
+        for row in results:
+            if not isinstance(row, dict):
+                continue
+            seat = row.get("worker")
+            if not isinstance(seat, str) or not seat.strip():
+                continue
+            seat_name = seat.strip()
+            samples[seat_name] = samples.get(seat_name, 0) + 1
+            if _worker_result_failed(row):
+                failures[seat_name] = failures.get(seat_name, 0) + 1
+            duration = row.get("duration_seconds")
+            if not isinstance(duration, (int, float)) or isinstance(duration, bool):
+                continue
+            durations.setdefault(seat_name, []).append(float(duration))
+
+    stats: dict[str, SeatReceiptStats] = {}
+    for seat_name, seat_durations in durations.items():
+        sample_count = samples[seat_name]
+        if sample_count <= 0:
+            continue
+        stats[seat_name] = SeatReceiptStats(
+            sample_count=sample_count,
+            median_duration_seconds=float(statistics.median(seat_durations)),
+            failure_rate=failures.get(seat_name, 0) / sample_count,
+        )
+    return stats

src/brigade/roster_cmd.py

@@ -10,6 +10,7 @@
 from . import model_inventory
 from . import roster as roster_mod
 from . import templates
+from . import toml_compat
 
 DEFAULT_ROSTER_REL = ".brigade/roster.toml"
 
@@ -94,6 +95,193 @@ def preset_roster_paths() -> tuple[Path, ...]:
     return tuple(sorted(rosters_dir.glob("*.toml")))
 
 
+def _resolve_preset_path(preset: Path | str) -> Path:
+    if isinstance(preset, Path):
+        path = preset.expanduser().resolve()
+    else:
+        name = str(preset).strip()
+        if not name:
+            raise ValueError("preset name must be non-empty")
+        if not name.endswith(".toml"):
+            name = f"{name}.toml"
+        path = (templates.template_root() / "rosters" / name).resolve()
+    if not path.is_file():
+        raise FileNotFoundError(f"preset not found: {path}")
+    return path
+
+
+def _local_receipt_stats(target: Path) -> dict[str, roster_mod.SeatReceiptStats]:
+    return roster_mod.collect_seat_receipt_stats(target / ".brigade" / "runs")
+
+
+def _format_resolved_seat(resolved: str | None) -> str:
+    return "-" if resolved is None else resolved
+
+
+def _print_seat_resolutions(report: tuple[roster_mod.SeatResolution, ...]) -> None:
+    for entry in report:
+        print(
+            f"requested={entry.requested} outcome={entry.outcome} "
+            f"resolved={_format_resolved_seat(entry.resolved)} reason={entry.reason}"
+        )
+
+
+def _stats_detail(
+    agent_name: str,
+    agent: roster_mod.Agent,
+    local_stats: dict[str, roster_mod.SeatReceiptStats],
+) -> str:
+    receipt = local_stats.get(agent_name)
+    if receipt is not None:
+        return (
+            f"source=local-receipts sample_count={receipt.sample_count} "
+            f"median_duration={receipt.median_duration_seconds:g} "
+            f"failure_rate={receipt.failure_rate:.3f}"
+        )
+    parts = ["source=author-default"]
+    if agent.stats:
+        for key, value in sorted(agent.stats.items()):
+            if key == "source":
+                continue
+            parts.append(f"{key}={value}")
+    return " ".join(parts)
+
+
+def _format_inline_table(values: dict[str, str]) -> str:
+    inner = ", ".join(f"{key} = {toml_compat.format_toml_value(value)}" for key, value in values.items())
+    return "{" + inner + "}"
+
+
+def _format_string_list(values: tuple[str, ...]) -> str:
+    return "[" + ", ".join(toml_compat.format_toml_value(item) for item in values) + "]"
+
+
+def _render_agent_stats(
+    agent: roster_mod.Agent,
+    agent_name: str,
+    local_stats: dict[str, roster_mod.SeatReceiptStats],
+) -> dict[str, str]:
+    receipt = local_stats.get(agent_name)
+    if receipt is not None:
+        rendered: dict[str, str] = {}
+        rendered["source"] = "local-receipts"
+        rendered["median_duration_seconds"] = f"{receipt.median_duration_seconds:g}"
+        rendered["failure_rate"] = f"{receipt.failure_rate:.3f}"
+        rendered["sample_count"] = str(receipt.sample_count)
+        return rendered
+    rendered = dict(agent.stats or {})
+    rendered["source"] = "author-default"
+    return rendered
+
+
+def _render_roster_toml(
+    roster: roster_mod.Roster,
+    local_stats: dict[str, roster_mod.SeatReceiptStats],
+) -> str:
+    lines: list[str] = [f"orchestrator = {toml_compat.format_toml_value(roster.orchestrator)}"]
+    if roster.codex_transport != "exec":
+        lines.append(f"codex_transport = {toml_compat.format_toml_value(roster.codex_transport)}")
+    lines.append("")
+
+    agent_names = [roster.orchestrator] + sorted(name for name in roster.agents if name != roster.orchestrator)
+    for name in agent_names:
+        agent = roster.agents[name]
+        lines.append(f"[agents.{name}]")
+        if agent.cli is not None:
+            lines.append(f"cli = {toml_compat.format_toml_value(agent.cli)}")
+        if agent.endpoint is not None:
+            lines.append(f"endpoint = {toml_compat.format_toml_value(agent.endpoint)}")
+        if agent.model is not None:
+            lines.append(f"model = {toml_compat.format_toml_value(agent.model)}")
+        if agent.reasoning is not None:
+            lines.append(f"reasoning = {toml_compat.format_toml_value(agent.reasoning)}")
+        lines.append(f"role = {toml_compat.format_toml_value(agent.role)}")
+        if agent.purpose is not None:
+            lines.append(f"purpose = {toml_compat.format_toml_value(agent.purpose)}")
+        if agent.requires is not None:
+            lines.append(f"requires = {_format_inline_table(agent.requires)}")
+        if agent.fallback:
+            lines.append(f"fallback = {_format_string_list(agent.fallback)}")
+        stats = _render_agent_stats(agent, name, local_stats)
+        if stats:
+            lines.append(f"stats = {_format_inline_table(stats)}")
+        if agent.caveats:
+            lines.append(f"caveats = {_format_string_list(agent.caveats)}")
+        if agent.transport != "direct":
+            lines.append(f"transport = {toml_compat.format_toml_value(agent.transport)}")
+        if agent.transport_version is not None:
+            lines.append(f"transport_version = {toml_compat.format_toml_value(agent.transport_version)}")
+        if agent.timeout_seconds is not None:
+            lines.append(f"timeout_seconds = {toml_compat.format_toml_value(agent.timeout_seconds)}")
+        if not agent.read_only_capable:
+            lines.append("read_only_capable = false")
+        if agent.invalid_final_fallback is not None:
+            lines.append(f"invalid_final_fallback = {toml_compat.format_toml_value(agent.invalid_final_fallback)}")
+        if agent.env is not None:
+            lines.append(f"env = {_format_inline_table(agent.env)}")
+        lines.append("")
+
+    lines.append("[limits]")
+    lines.append(f"max_workers = {toml_compat.format_toml_value(roster.max_workers)}")
+    lines.append(f"timeout_seconds = {toml_compat.format_toml_value(roster.timeout_seconds)}")
+    if roster.allow_models:
+        lines.append(f"allow_models = {_format_string_list(roster.allow_models)}")
+    if roster.sandbox is not None:
+        lines.append(f"sandbox = {toml_compat.format_toml_value(roster.sandbox)}")
+    return "\n".join(lines) + "\n"
+
+
+def suggest(
+    target: Path,
+    *,
+    preset: Path | str,
+    probe: roster_mod.CapabilityProbe | None = None,
+) -> int:
+    target = target.expanduser()
+    try:
+        preset_path = _resolve_preset_path(preset)
+    except (FileNotFoundError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    try:
+        loaded = roster_mod.load_roster(preset_path)
+    except ValueError as exc:
+        print(f"error: invalid preset {preset_path}: {exc}", file=sys.stderr)
+        return 2
+
+    active_probe = probe if probe is not None else roster_mod.HostCapabilityProbe()
+    result = roster_mod.resolve_capabilities(loaded, active_probe)
+    local_stats = _local_receipt_stats(target)
+
+    _print_seat_resolutions(result.report)
+    for name, agent in result.roster.agents.items():
+        print(f"stats seat={name} {_stats_detail(name, agent, local_stats)}")
+
+    if not result.usable:
+        print("roster is not adoptable: orchestrator seat is unavailable")
+        return 1
+
+    print("\n# Adoptable roster")
+    print(_render_roster_toml(result.roster, local_stats), end="")
+    return 0
+
+
+def stats(target: Path) -> int:
+    target = target.expanduser()
+    local_stats = _local_receipt_stats(target)
+    if not local_stats:
+        print("no local worker receipt stats found")
+        return 0
+    for seat_name in sorted(local_stats):
+        receipt = local_stats[seat_name]
+        print(
+            f"seat={seat_name} source=local-receipts sample_count={receipt.sample_count} "
+            f"median_duration={receipt.median_duration_seconds:g} "
+            f"failure_rate={receipt.failure_rate:.3f}"
+        )
+    return 0
+
+
 def init(
     target: Path,
     *,
@@ -130,7 +318,12 @@ def init(
     return 0
 
 
-def doctor(target: Path, *, roster_path: Path | None = None) -> int:
+def doctor(
+    target: Path,
+    *,
+    roster_path: Path | None = None,
+    probe: roster_mod.CapabilityProbe | None = None,
+) -> int:
     target = target.expanduser()
 
     checks: list[doctor_mod.CheckResult] = []
@@ -144,6 +337,14 @@ def doctor(target: Path, *, roster_path: Path | None = None) -> int:
         checks.append((doctor_mod.FAIL, "roster: file", f"invalid {path}: {exc}"))
         return doctor_mod._report(checks)
 
+    local_stats = _local_receipt_stats(target)
+    active_probe = probe if probe is not None else roster_mod.HostCapabilityProbe()
+    capability = roster_mod.resolve_capabilities(loaded, active_probe)
+    for entry in capability.report:
+        if entry.outcome == "self":
+            continue
+        checks.append((doctor_mod.WARN, f"roster: capability {entry.requested}", entry.reason))
+
     checks.append((doctor_mod.OK, "roster: file", str(path)))
     checks.append((doctor_mod.OK, "roster: orchestrator", loaded.orchestrator))
     checks.append((doctor_mod.OK, "roster: max_workers", str(loaded.max_workers)))
@@ -155,6 +356,10 @@ def doctor(target: Path, *, roster_path: Path | None = None) -> int:
     else:
         checks.append((doctor_mod.WARN, "roster: allow_models", "not set; explicit model allow-list recommended"))
 
+    for name, agent in loaded.agents.items():
+        if agent.stats is not None or name in local_stats:
+            checks.append((doctor_mod.INFO, f"roster: stats {name}", _stats_detail(name, agent, local_stats)))
+
     inventory_inspector = model_inventory.ModelInventoryInspector()
     for name, agent in loaded.agents.items():
         timeout = roster_mod.timeout_for(agent, loaded)