|
12 | 12 | # Run single scenario directly |
13 | 13 | agingbench run --scenario s1_research_literature --sut <path> |
14 | 14 |
|
15 | | - # Run with oracle-memory ablation (S6) |
16 | | - agingbench run --scenario s1_research_literature --sut <path> --oracle memory |
17 | | -
|
18 | 15 | # Run with multiple seeds for confidence intervals |
19 | 16 | agingbench run --suite core --seeds 3 |
20 | 17 |
|
@@ -67,16 +64,16 @@ def cmd_run(suite_id: Optional[str], scenario_id: Optional[str], |
67 | 64 | adapter_spec: Optional[str] = None, |
68 | 65 | memory_policy_spec: Optional[str] = None, |
69 | 66 | generated: bool = False, gen_sessions: int = 0, |
70 | | - emit_card: bool = False) -> None: |
| 67 | + emit_card: bool = True) -> None: |
71 | 68 | """Run one or more scenarios from a suite against one or more SUTs. |
72 | 69 |
|
73 | 70 | Parameters |
74 | 71 | ---------- |
75 | 72 | emit_card : bool |
76 | | - When True, emit a consolidated aging_card.json alongside the |
77 | | - existing metrics.json after each run completes. Default False |
78 | | - so existing CI scripts that don't pass `--card` produce |
79 | | - unchanged output. |
| 73 | + When True (the default), emit a consolidated aging_card.json |
| 74 | + alongside the existing metrics.json after each run completes. |
| 75 | + The card is the canonical four-mechanism view. Pass --no-card |
| 76 | + (emit_card=False) to skip it, e.g. CI that only reads metrics.json. |
80 | 77 | """ |
81 | 78 |
|
82 | 79 | # --sessions / --cycles only have meaning with the programmatic generator. |
@@ -158,8 +155,8 @@ def cmd_run(suite_id: Optional[str], scenario_id: Optional[str], |
158 | 155 | runner_cls = getattr(mod, runner_cfg["class"]) |
159 | 156 | print(f"[info] Loaded runner {runner_cfg['class']} from manifest") |
160 | 157 | # Wrap in a function matching _run_sX signature |
161 | | - def _dynamic_runner(sut, scen_cfg, out, n, oracle=False, **kw): |
162 | | - return _run_dynamic(runner_cls, sut, scen_cfg, out, n, oracle, **kw) |
| 158 | + def _dynamic_runner(sut, scen_cfg, out, n, **kw): |
| 159 | + return _run_dynamic(runner_cls, sut, scen_cfg, out, n, **kw) |
163 | 160 | runner_fn = _dynamic_runner |
164 | 161 | except Exception as e: |
165 | 162 | print(f"[warn] Failed to load runner from manifest: {e}") |
@@ -210,8 +207,8 @@ def _dynamic_runner(sut, scen_cfg, out, n, oracle=False, **kw): |
210 | 207 | stats["seed"] = seed_val |
211 | 208 | all_results.append(stats) |
212 | 209 |
|
213 | | - # Emit AgingCard JSON when opted in via --card. Pure |
214 | | - # post-processor: never modifies metrics.json or |
| 210 | + # Emit AgingCard JSON by default (disable with --no-card). |
| 211 | + # Pure post-processor: never modifies metrics.json or |
215 | 212 | # dependency_metrics.json. |
216 | 213 | if emit_card: |
217 | 214 | try: |
@@ -428,11 +425,13 @@ def _build_parser(): |
428 | 425 | "N as 'N sessions' (exclusive); S7+ treats N as 'N agent blocks'. " |
429 | 426 | "Suite-driven runs (`--suite lite/full/core/...`) use n_cycles " |
430 | 427 | "from the suite YAML.") |
431 | | - # AgingCard emission. Default OFF so existing CI scripts that don't |
432 | | - # pass --card produce unchanged output. |
433 | | - run_p.add_argument("--card", action="store_true", default=False, |
434 | | - help="Emit a consolidated aging_card.json alongside metrics.json. " |
435 | | - "Schema validated against agingbench/metrics/aging_card_schema.json.") |
| 428 | + # AgingCard emission. Default ON — the consolidated aging_card.json is the |
| 429 | + # canonical four-mechanism view; emit it for every run. Opt out with |
| 430 | + # --no-card (e.g. CI that only consumes metrics.json). |
| 431 | + run_p.add_argument("--card", action=argparse.BooleanOptionalAction, default=True, |
| 432 | + help="Emit a consolidated aging_card.json alongside metrics.json " |
| 433 | + "(default: on; use --no-card to disable). Schema validated " |
| 434 | + "against agingbench/metrics/aging_card_schema.json.") |
436 | 435 |
|
437 | 436 | # ---- compare ---- |
438 | 437 | cmp_p = sub.add_parser("compare", help="Compare results from multiple result directories") |
|
0 commit comments