feat: LLM-as-judge scorer, dataset auto-sampling, eval --ci baseline (#78)

- Add score_llm_judge() scorer: calls any OpenAI-compatible endpoint,
  parses JSON {score, reason}, clamps to [0,1], strips markdown fences
- Dispatch llm_judge through run_scorer() alongside existing scorers
- Add auto_populate() to dataset: 6 signal filters (has-errors,
  high-retry, cost-above, wide-blast, long-duration, low-eval-score),
  since_days window, dedup, optional label
- Extend cmd_eval_ci() with --baseline, --save-baseline, --tolerance,
  --github-summary flags; _load_baseline/_save_baseline/_write_github_summary
- GitHub summary writes PR-comment-ready Markdown with delta vs baseline
- 25 new tests covering all three features (700 total, all passing)

Co-authored-by: Ona <no-reply@ona.com>

Author: Siddhant-K-code

src/agent_trace/cli.py

@@ -575,6 +575,14 @@ def build_parser() -> argparse.ArgumentParser:
     p_eval_ci = eval_sub.add_parser("ci", help="run evals and exit 1 if any scorer fails")
     p_eval_ci.add_argument("session_id", nargs="?", help="session ID or prefix (default: latest)")
     p_eval_ci.add_argument("--config", default=".agent-evals.yaml")
+    p_eval_ci.add_argument("--baseline", metavar="FILE",
+                           help="compare scores against a saved baseline JSON")
+    p_eval_ci.add_argument("--save-baseline", dest="save_baseline", metavar="FILE",
+                           help="save current scores as a baseline and exit")
+    p_eval_ci.add_argument("--tolerance", type=float, default=0.0, metavar="N",
+                           help="allow up to N regression vs baseline before failing (default: 0)")
+    p_eval_ci.add_argument("--github-summary", dest="github_summary", action="store_true",
+                           help="write PR-comment Markdown to .agent-traces/eval-summary.md")
 
     p_eval_dataset = eval_sub.add_parser("dataset", help="manage eval datasets")
     dataset_sub = p_eval_dataset.add_subparsers(dest="dataset_command")
@@ -587,6 +595,15 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_ds_export = dataset_sub.add_parser("export", help="export dataset to JSONL")
     p_ds_export.add_argument("--dataset", default=".agent-traces/datasets/default.jsonl")
+    p_ds_auto = dataset_sub.add_parser("auto", help="auto-populate dataset from sessions by signal filter")
+    p_ds_auto.add_argument("--name", default="default", help="dataset name (default: default)")
+    p_ds_auto.add_argument("--dataset", default="", help="explicit dataset path (overrides --name)")
+    p_ds_auto.add_argument("--filter", default="has-errors",
+                           help="filter: has-errors, high-retry, cost-above:N, wide-blast, "
+                                "long-duration:Ns, low-eval-score:N (default: has-errors)")
+    p_ds_auto.add_argument("--since", default="7d", metavar="Nd",
+                           help="look back N days (default: 7d)")
+    p_ds_auto.add_argument("--label", default="", help="label for added entries")
 
     # watch
     p_watch = sub.add_parser("watch", help="monitor a live session with circuit breakers")

src/agent_trace/eval/dataset.py

@@ -13,6 +13,10 @@
 import uuid
 from dataclasses import asdict, dataclass, field
 from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..store import TraceStore
 
 
 
@@ -68,6 +72,138 @@ def export_entries(dataset_path: str | Path, out=sys.stdout) -> None:
         out.write(entry.to_json() + "\n")
 
 
+# ---------------------------------------------------------------------------
+# Auto-sampling: populate a dataset from stored sessions by signal filter
+# ---------------------------------------------------------------------------
+
+def _session_passes_filter(
+    store: "TraceStore",
+    session_id: str,
+    filter_spec: str,
+    eval_threshold: float = 0.8,
+) -> bool:
+    """Return True if the session matches the given filter spec.
+
+    Supported filters:
+      has-errors          — session has at least one ERROR event
+      high-retry          — retry rate > 30%
+      cost-above:N        — estimated cost > $N
+      wide-blast          — distinct files written > 10
+      long-duration:Ns    — session duration > N seconds
+      low-eval-score:N    — eval.json overall score < N
+    """
+    from ..models import EventType
+
+    try:
+        events = store.load_events(session_id)
+        meta = store.load_meta(session_id)
+    except Exception:
+        return False
+
+    spec = filter_spec.strip().lower()
+
+    if spec == "has-errors":
+        return any(e.event_type == EventType.ERROR for e in events)
+
+    if spec == "high-retry":
+        tool_calls = [e for e in events if e.event_type == EventType.TOOL_CALL]
+        if not tool_calls:
+            return False
+        retries = 0
+        prev = None
+        run = 0
+        for ev in tool_calls:
+            name = ev.data.get("tool_name", "")
+            if name == prev:
+                run += 1
+                if run >= 2:
+                    retries += 1
+            else:
+                prev = name
+                run = 0
+        return retries / len(tool_calls) > 0.30
+
+    if spec.startswith("cost-above:"):
+        try:
+            threshold_dollars = float(spec.split(":", 1)[1])
+        except ValueError:
+            return False
+        cost = meta.total_tokens / 1_000_000 * 3.0
+        return cost > threshold_dollars
+
+    if spec == "wide-blast":
+        files: set[str] = set()
+        for ev in events:
+            if ev.event_type == EventType.FILE_WRITE:
+                p = ev.data.get("path") or ev.data.get("file_path") or ""
+                if p:
+                    files.add(p)
+        return len(files) > 10
+
+    if spec.startswith("long-duration:"):
+        try:
+            max_s = float(spec.split(":", 1)[1].rstrip("s"))
+        except ValueError:
+            return False
+        duration = meta.total_duration_ms / 1000 if meta.total_duration_ms else 0.0
+        return duration > max_s
+
+    if spec.startswith("low-eval-score:"):
+        try:
+            score_threshold = float(spec.split(":", 1)[1])
+        except ValueError:
+            score_threshold = eval_threshold
+        eval_path = store.base_dir / session_id / "eval.json"
+        if not eval_path.exists():
+            return False
+        try:
+            data = json.loads(eval_path.read_text())
+            results = data.get("results") or data.get("judges") or []
+            if not results:
+                return False
+            avg = sum(float(r.get("score", 0)) for r in results) / len(results)
+            return avg < score_threshold
+        except Exception:
+            return False
+
+    return False
+
+
+def auto_populate(
+    store: "TraceStore",
+    dataset_path: str | Path,
+    filter_spec: str,
+    since_days: float = 7.0,
+    label: str = "",
+    limit: int = 500,
+) -> int:
+    """Auto-populate a dataset from sessions matching a filter.
+
+    Returns the number of entries added.
+    """
+    cutoff = time.time() - since_days * 86400
+    added = 0
+
+    existing = {e.session_id for e in list_entries(dataset_path)}
+
+    for meta in store.list_sessions():
+        if meta.started_at < cutoff:
+            continue
+        if meta.session_id in existing:
+            continue
+        if added >= limit:
+            break
+        if _session_passes_filter(store, meta.session_id, filter_spec):
+            entry = DatasetEntry(
+                session_id=meta.session_id,
+                label=label or filter_spec,
+            )
+            add_entry(dataset_path, entry)
+            added += 1
+
+    return added
+
+
 # ---------------------------------------------------------------------------
 # CLI handler
 # ---------------------------------------------------------------------------
@@ -104,5 +240,17 @@ def cmd_dataset(args: argparse.Namespace) -> int:
         export_entries(dataset_path)
         return 0
 
-    sys.stderr.write("Usage: agent-strace eval dataset <add|list|export>\n")
+    if dataset_command == "auto":
+        from ..store import TraceStore
+        filter_spec = getattr(args, "filter", "has-errors") or "has-errors"
+        since_raw = getattr(args, "since", "7d") or "7d"
+        since_days = float(since_raw.rstrip("d"))
+        label = getattr(args, "label", "") or filter_spec
+        trace_dir = getattr(args, "trace_dir", ".agent-traces")
+        store = TraceStore(trace_dir)
+        added = auto_populate(store, dataset_path, filter_spec, since_days=since_days, label=label)
+        sys.stdout.write(f"Added {added} session(s) to {dataset_path} (filter: {filter_spec})\n")
+        return 0
+
+    sys.stderr.write("Usage: agent-strace eval dataset <add|list|export|auto>\n")
     return 1

src/agent_trace/eval/runner.py

@@ -8,6 +8,7 @@
 import argparse
 import json
 import sys
+from pathlib import Path
 from dataclasses import dataclass, field
 
 from ..store import TraceStore
@@ -211,8 +212,74 @@ def cmd_eval_compare(args: argparse.Namespace) -> int:
     return 0
 
 
+def _load_baseline(path: str) -> dict[str, float]:
+    """Load a saved baseline: {scorer_name: score}."""
+    p = Path(path)
+    if not p.exists():
+        return {}
+    try:
+        return json.loads(p.read_text())
+    except Exception:
+        return {}
+
+
+def _save_baseline(path: str, report: "EvalReport") -> None:
+    """Save current scores as a baseline file."""
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    data = {r.scorer: r.score for r in report.results}
+    p.write_text(json.dumps(data, indent=2))
+
+
+def _write_github_summary(report: "EvalReport", baseline: dict[str, float], tolerance: float) -> None:
+    """Write a PR-comment-ready Markdown summary to .agent-traces/eval-summary.md."""
+    lines = ["## agent-strace eval\n"]
+    lines.append("| Judge | Pass rate | Baseline | Delta | Status |")
+    lines.append("|---|---|---|---|---|")
+    for r in report.results:
+        base_score = baseline.get(r.scorer)
+        if base_score is not None:
+            delta = r.score - base_score
+            delta_str = f"{delta:+.0%}"
+            regressed = delta < -tolerance
+            status = "❌" if regressed else "✅"
+            base_str = f"{base_score:.0%}"
+        else:
+            delta_str = "—"
+            status = "✅" if r.passed else "❌"
+            base_str = "—"
+        lines.append(f"| `{r.scorer}` | {r.score:.0%} | {base_str} | {delta_str} | {status} |")
+
+    lines.append("")
+    if report.overall_passed:
+        lines.append("**Result: PASS**")
+    else:
+        lines.append(f"**Result: FAIL** — {report.failed} scorer(s) below threshold.")
+
+    failing = [r for r in report.results if not r.passed]
+    if failing:
+        lines.append("")
+        lines.append("<details>")
+        lines.append("<summary>Failing scorers</summary>")
+        lines.append("")
+        for r in failing:
+            lines.append(f"- `{r.scorer}` — score {r.score:.2f} (threshold {r.threshold:.2f}): {r.reason}")
+        lines.append("")
+        lines.append("</details>")
+
+    summary_path = Path(".agent-traces/eval-summary.md")
+    summary_path.parent.mkdir(parents=True, exist_ok=True)
+    summary_path.write_text("\n".join(lines) + "\n")
+    sys.stderr.write(f"GitHub summary written to {summary_path}\n")
+
+
 def cmd_eval_ci(args: argparse.Namespace) -> int:
-    """Run evals and exit 1 if any scorer fails (for CI integration)."""
+    """Run evals and exit 1 if any scorer fails (for CI integration).
+
+    Supports baseline comparison (--baseline), saving baselines
+    (--save-baseline), regression tolerance (--tolerance), and
+    GitHub Actions PR comment output (--github-summary).
+    """
     store = TraceStore(args.trace_dir)
     config = load_config(getattr(args, "config", ".agent-evals.yaml"))
 
@@ -222,12 +289,44 @@ def cmd_eval_ci(args: argparse.Namespace) -> int:
         return 1
 
     report = run_eval(store, session_id, config)
-    # Route table to stderr so CI output is pipeable without noise
     format_report_table(report, out=sys.stderr)
 
-    if report.overall_passed:
-        sys.stderr.write("CI: all scorers passed\n")
+    # Save baseline if requested
+    save_baseline_path = getattr(args, "save_baseline", None)
+    if save_baseline_path:
+        _save_baseline(save_baseline_path, report)
+        sys.stderr.write(f"Baseline saved to {save_baseline_path}\n")
         return 0
-    else:
-        sys.stderr.write(f"CI: {report.failed} scorer(s) failed\n")
+
+    # Load baseline for comparison
+    baseline_path = getattr(args, "baseline", None)
+    baseline: dict[str, float] = {}
+    if baseline_path:
+        baseline = _load_baseline(baseline_path)
+
+    tolerance = float(getattr(args, "tolerance", 0.0) or 0.0)
+
+    # GitHub summary
+    if getattr(args, "github_summary", False):
+        _write_github_summary(report, baseline, tolerance)
+
+    # Determine pass/fail with optional baseline regression check
+    failed = False
+    if not report.overall_passed:
+        failed = True
+    elif baseline:
+        for r in report.results:
+            base_score = baseline.get(r.scorer)
+            if base_score is not None and (r.score - base_score) < -tolerance:
+                sys.stderr.write(
+                    f"CI: {r.scorer} regressed {r.score:.2f} vs baseline {base_score:.2f} "
+                    f"(tolerance {tolerance:.2f})\n"
+                )
+                failed = True
+
+    if failed:
+        sys.stderr.write(f"CI: FAIL — {report.failed} scorer(s) failed\n")
         return 1
+
+    sys.stderr.write("CI: PASS — all scorers passed\n")
+    return 0