feat(sample): add dataset auto-sampler for regression suite export (#107)

Adds agent-strace sample command to surface worst/diverse/random/recent
sessions and export them as JSONL for use in eval frameworks. Supports
four sampling strategies, deduplication by tool call sequence, and
reproducible random sampling via --seed.

Closes #94

Co-authored-by: Ona <no-reply@ona.com>

Author: Siddhant-K-code

README.md

@@ -218,6 +218,7 @@ agent-strace token-budget <session-id>          Check token usage against model
 agent-strace replay [session-id] [--limit N]    Replay a session (--limit caps events shown)
 agent-strace retention status                   Show session count, size, and what policy would delete
 agent-strace retention clean [--dry-run]        Delete sessions that exceed retention limits
+agent-strace sample --strategy worst --n 20     Export worst/diverse/random/recent sessions as JSONL
 agent-strace watch [--timeout DURATION] [--budget $] [--on-death CMD] [--rules file]
                                                 Watch a live session; kill/pause on rule breach
 agent-strace share <session-id> [-o file]       Export a self-contained HTML report
@@ -797,6 +798,29 @@ agent-strace dashboard --html report.html # self-contained HTML export
 
 The terminal view shows total tool calls, errors, tokens, and estimated cost, plus ASCII sparkline charts for each metric over time and a top-tools frequency table. The HTML export is self-contained. No server needed.
 
+### Dataset auto-sampler
+
+Export the sessions most useful for regression suites and eval datasets — without manual inspection.
+
+```bash
+# Export the 20 worst-performing sessions (highest error/retry/cost)
+agent-strace sample --strategy worst --n 20 --output regression.jsonl
+
+# Export 10 sessions that maximise behavioral variety
+agent-strace sample --strategy diverse --n 10 --output diverse.jsonl
+
+# Export the 5 most recent sessions
+agent-strace sample --strategy recent --n 5 --output recent.jsonl
+
+# Random sample, reproducible with a seed
+agent-strace sample --strategy random --n 15 --seed 42 --output random.jsonl
+
+# Skip sessions with identical tool call sequences
+agent-strace sample --strategy worst --n 20 --deduplicate --output regression.jsonl
+```
+
+Output is JSONL — one session per line — with full event data and a score breakdown. Compatible with LangSmith, Braintrust, and any custom eval framework.
+
 ### Eval trend dashboard
 
 See whether your agent is getting better or worse over time. Reads eval scores and behavioral metrics from session events, then renders a self-contained HTML report with inline SVG charts.

src/agent_trace/__init__.py

@@ -1,3 +1,3 @@
 """agent-trace: strace for AI agents."""
 
-__version__ = "0.40.0"
+__version__ = "0.41.0"

src/agent_trace/cli.py

@@ -46,6 +46,7 @@
 from .share import cmd_share
 from .token_budget import cmd_token_budget
 from .retention import cmd_retention
+from .sample import cmd_sample
 from .watch import cmd_watch
 from .why import cmd_why
 from .models import EventType, SessionMeta, TraceEvent
@@ -773,6 +774,26 @@ def build_parser() -> argparse.ArgumentParser:
         help="transport protocol (default: stdio)",
     )
 
+    # sample
+    p_sample = sub.add_parser(
+        "sample",
+        help="export worst/diverse/random/recent sessions as a JSONL regression suite",
+    )
+    p_sample.add_argument(
+        "--strategy",
+        choices=["worst", "diverse", "random", "recent"],
+        default="worst",
+        help="sampling strategy (default: worst)",
+    )
+    p_sample.add_argument("--n", type=int, default=20, metavar="N",
+                          help="number of sessions to sample (default: 20)")
+    p_sample.add_argument("--output", "-o", default="sample.jsonl",
+                          help="output JSONL file path (default: sample.jsonl)")
+    p_sample.add_argument("--deduplicate", action="store_true",
+                          help="skip sessions with identical tool call sequences")
+    p_sample.add_argument("--seed", type=int, default=None,
+                          help="random seed for reproducible random sampling")
+
     # retention
     p_ret = sub.add_parser("retention", help="manage session data retention")
     ret_sub = p_ret.add_subparsers(dest="retention_command")
@@ -852,6 +873,7 @@ def main() -> None:
         "standup": cmd_standup,
         "mcp": cmd_mcp,
         "retention": cmd_retention,
+        "sample": cmd_sample,
     }
 
     handler = handlers.get(args.command)