diff --git a/cli/README.md b/cli/README.md index 74f5335a4..f270832f2 100644 --- a/cli/README.md +++ b/cli/README.md @@ -207,6 +207,7 @@ Options: - `--model` / `-m`: Models to test against (default: `openai:gpt-4.1`). Can be specified multiple times. - `--threads` / `-t`: Number of parallel threads (default: `1`) +- `--k` / `-k`: Number of times to run each test case, used to compute pass@k and pass^k (default: `1`) - `--select` / `-s`: Run only selected tests by name, yaml stem, or subfolder. Comma-separated. - `--username` / `-u`, `--password`: Credentials for the nao backend. Fall back to `NAO_USERNAME` / `NAO_PASSWORD`. @@ -216,9 +217,10 @@ Examples: nao test -m openai:gpt-4.1 nao test -m openai:gpt-4.1 -m anthropic:claude-sonnet-4-20250514 nao test --threads 4 +nao test --k 5 ``` -Defaults for every run live in the `test` block of `nao_config.yaml`, and the `--model` / `--threads` flags override them: +Defaults for every run live in the `test` block of `nao_config.yaml`, and the `--model` / `--threads` / `--k` flags override them: ```yaml test: @@ -226,6 +228,7 @@ test: - openai:gpt-4.1 - anthropic:claude-sonnet-4-5 threads: 4 + # k: 5 # run each case 5 times to compute pass@k / pass^k comparison: rtol: 0.00001 atol: 0.00000001 diff --git a/cli/nao_core/commands/test/runner.py b/cli/nao_core/commands/test/runner.py index d0a35da45..cf5c710a5 100644 --- a/cli/nao_core/commands/test/runner.py +++ b/cli/nao_core/commands/test/runner.py @@ -21,7 +21,13 @@ from .case import TESTS_FOLDER, TestCase, discover_tests from .client import BACKEND_URL, AgentClientError, VerificationResult, get_client from .compare import normalize_dataframe_numbers -from .summary import ModelSummary, summarize, summarize_by_model +from .summary import ( + group_by_test_and_model, + pass_metrics_for_group, + summarize, + summarize_by_model, + summarize_pass_metrics, +) @dataclass @@ -70,6 +76,7 @@ class TestRunResult: tool_call_count: int | None = None error: str | None = None details: TestRunDetails | None = None + attempt: int | None = None def check_dataframe( @@ -295,6 +302,7 @@ def save_results(results: list[TestRunResult], output_dir: Path) -> Path: "results": runs, "summary": summarize(runs), "by_model": [asdict(s) for s in summarize_by_model(runs)], + "pass_metrics": summarize_pass_metrics(runs), } output_file.write_text(json.dumps(data, indent=2)) @@ -302,52 +310,54 @@ def save_results(results: list[TestRunResult], output_dir: Path) -> Path: def print_run_table(results: list[TestRunResult]) -> None: - """Print one row per run, i.e. per (test, model) pair.""" - df = pd.DataFrame( - [ + """Print one row per (test, model), aggregating all attempts for that pair.""" + run_dicts = [asdict(r) for r in results] + rows = [] + for (name, model), group in sorted(group_by_test_and_model(run_dicts).items()): + metrics = pass_metrics_for_group(group) + rows.append( { - "Test": r.name, - "Model": r.model, - "Status": status_icon(r.passed), - "Message": r.message, - "Tokens": r.tokens or 0, - "Cost": r.cost or 0, - "Time (s)": round((r.duration_ms or 0) / 1000, 1), - "Tools": r.tool_call_count or 0, + "Test": name, + "Model": model, + "Status": status_icon(float(metrics["pass_hat_k"]) == 1.0), + "Success %": format_pass_fraction(float(metrics["pass_at_1"])), + "Tokens": sum(run.get("tokens") or 0 for run in group), + "Cost": sum(run.get("cost") or 0 for run in group), + "Time (s)": round(sum(run.get("duration_ms") or 0 for run in group) / 1000, 1), + "Tools": sum(run.get("tool_call_count") or 0 for run in group), } - for r in results - ] - ) - - UI.table(df, title="Test Results", sum_columns={"Tokens": "", "Cost": "$", "Time (s)": "", "Tools": ""}) - - -def print_model_table(summaries: list[ModelSummary]) -> None: - """Print one row per model, ranked from best to worst pass rate.""" - df = pd.DataFrame( - [ - { - "Model": column_label(s.model), - "Pass Rate": format_pass_rate(s.pass_rate), - "Passed": f"{s.passed}/{s.total}", - "Tokens": s.total_tokens, - "Cost": s.total_cost, - "Avg Time (s)": round(s.avg_duration_ms / 1000, 1), - "Avg Tools": s.avg_tool_calls, - } - for s in summaries - ] - ) + ) - UI.table(df, title="Performance by Model", sum_columns={"Tokens": "", "Cost": "$"}, fixed_columns={"Model"}) + UI.table(pd.DataFrame(rows), title="Test Results") + + +def print_summary_table(results: list[TestRunResult]) -> None: + """Print one totals row across all test/model pairs and attempts.""" + run_dicts = [asdict(r) for r in results] + groups = group_by_test_and_model(run_dicts) + total = len(run_dicts) + passed = sum(1 for run in run_dicts if run.get("passed")) + always_passed = sum(1 for group in groups.values() if all(run.get("passed") for run in group)) + total_duration_ms = sum(run.get("duration_ms") or 0 for run in run_dicts) + + row = { + "Tests": len(groups), + "Success %": format_pass_fraction(passed / total if total else 0.0), + "Always Pass %": format_pass_fraction(always_passed / len(groups) if groups else 0.0), + "Tokens": sum(run.get("tokens") or 0 for run in run_dicts), + "Cost": f"${sum(run.get('cost') or 0 for run in run_dicts):.4f}", + "Time (s)": round(total_duration_ms / 1000, 1), + "Tools": sum(run.get("tool_call_count") or 0 for run in run_dicts), + } + UI.table(pd.DataFrame([row]), title="Summary") def print_model_matrix(results: list[TestRunResult]) -> None: """Print a test × model grid to show which model passes which test.""" models = list(dict.fromkeys(r.model for r in results)) statuses: dict[tuple[str, str], list[str]] = {} - for result in results: - statuses.setdefault((result.name, result.model), []).append(status_icon(result.passed)) + for key, group in group_by_test_and_model([asdict(result) for result in results]).items(): + statuses[key] = [status_icon(all(bool(run.get("passed")) for run in group))] rows = [ {"Test": name} @@ -368,10 +378,11 @@ def status_icon(passed: bool) -> str: return "[green]✓[/green]" if passed else "[red]✗[/red]" -def format_pass_rate(pass_rate: float) -> str: - """Colour a pass rate from green (all passing) to red (mostly failing).""" - color = "green" if pass_rate == 100 else "red" if pass_rate < 50 else "yellow" - return f"[{color}]{pass_rate}%[/{color}]" +def format_pass_fraction(rate: float) -> str: + """Colour a 0-1 pass metric (pass@k / pass^k) for terminal tables.""" + pct = round(rate * 100, 1) + color = "green" if pct == 100 else "red" if pct < 50 else "yellow" + return f"[{color}]{pct}%[/{color}]" def filter_test_cases( @@ -452,6 +463,13 @@ def test( help="Number of parallel threads for running tests. Overrides test.threads.", ), ] = None, + k: Annotated[ + int | None, + Parameter( + name=["-k", "--k"], + help="Number of times to run each test case, used to compute pass@k and pass^k. Overrides test.k.", + ), + ] = None, select: Annotated[ str | None, Parameter( @@ -483,6 +501,7 @@ def test( nao test -m openai:gpt-4.1 nao test -m openai:gpt-4.1 -m anthropic:claude-sonnet-4-20250514 nao test --threads 4 + nao test --k 5 nao test -s test_name nao test -s 12,13,14 nao test -u user@example.com --password secret @@ -497,6 +516,11 @@ def test( test_config = config.test or TestConfig() thread_count = threads if threads is not None else test_config.threads + k_count = k if k is not None else test_config.k + + if k_count < 1: + UI.error(f"k must be >= 1, got {k_count}") + return try: model_configs = [ModelConfig.parse(m) for m in models or test_config.models] @@ -508,7 +532,10 @@ def test( tests_dir = project_path / TESTS_FOLDER UI.print(f"[dim]Project: {config.project_name}[/dim]") UI.print(f"[dim]Tests folder: {tests_dir}[/dim]") - UI.print(f"[dim]Models: {', '.join(str(m) for m in model_configs)}[/dim]\n") + UI.print(f"[dim]Models: {', '.join(str(m) for m in model_configs)}[/dim]") + if k_count > 1: + UI.print(f"[dim]k: {k_count}[/dim]") + UI.print("") test_cases = discover_tests(project_path) @@ -524,18 +551,27 @@ def test( ensure_verification_engine(test_cases) - total_runs = len(test_cases) * len(model_configs) - UI.print(f"[bold]Found {len(test_cases)} test(s) × {len(model_configs)} model(s) = {total_runs} run(s)[/bold]") + total_runs = len(test_cases) * len(model_configs) * k_count + UI.print( + f"[bold]Found {len(test_cases)} test(s) × {len(model_configs)} model(s) × {k_count} attempt(s) = {total_runs} run(s)[/bold]" + if k_count > 1 + else f"[bold]Found {len(test_cases)} test(s) × {len(model_configs)} model(s) = {total_runs} run(s)[/bold]" + ) if thread_count > 1: UI.print(f"[dim]Running with {thread_count} threads (output may be interleaved)[/dim]") UI.print("") - # Build list of (test_case, model) pairs - test_runs = [(test_case, model) for model in model_configs for test_case in test_cases] + # Build list of (test_case, model, attempt) triples — attempt is 1-indexed + test_runs = [ + (test_case, model, attempt) + for model in model_configs + for test_case in test_cases + for attempt in range(1, k_count + 1) + ] results: list[TestRunResult] = [] if thread_count == 1: - for test_case, model in test_runs: + for test_case, model, attempt in test_runs: result = run_test( test_case, model, @@ -544,6 +580,7 @@ def test( costs=resolve_model_costs(config, model), comparison=test_config.comparison, ) + result.attempt = attempt results.append(result) UI.print("") else: @@ -559,11 +596,14 @@ def test( password=pwd, costs=resolve_model_costs(config, m), comparison=test_config.comparison, - ): index - for index, (tc, m) in enumerate(test_runs) + ): (index, attempt) + for index, (tc, m, attempt) in enumerate(test_runs) } for future in as_completed(futures): - completed[futures[future]] = future.result() + index, attempt = futures[future] + result = future.result() + result.attempt = attempt + completed[index] = result UI.print("") results = [completed[index] for index in sorted(completed)] @@ -573,16 +613,17 @@ def test( print_run_table(results) - model_summaries = summarize_by_model([asdict(r) for r in results]) + run_dicts = [asdict(r) for r in results] + model_summaries = summarize_by_model(run_dicts) + + print_summary_table(results) if len(model_summaries) > 1: - print_model_table(model_summaries) print_model_matrix(results) passed = sum(1 for r in results if r.passed) failed = sum(1 for r in results if not r.passed) total = len(results) - unit = "run" if len(model_summaries) > 1 else "test" - + unit = "run" if len(model_summaries) > 1 or k_count > 1 else "test" UI.print("") if failed == 0: UI.success(f"All {total} {unit}(s) passed") diff --git a/cli/nao_core/commands/test/summary.py b/cli/nao_core/commands/test/summary.py index be36d6393..3dba153b2 100644 --- a/cli/nao_core/commands/test/summary.py +++ b/cli/nao_core/commands/test/summary.py @@ -1,5 +1,6 @@ """Aggregation of test run results, shared by the CLI output and the results server.""" +from collections import defaultdict from collections.abc import Mapping, Sequence from dataclasses import asdict, dataclass from typing import Any @@ -24,6 +25,10 @@ class ModelSummary: avg_duration_ms: float total_tool_calls: int avg_tool_calls: float + k: int = 1 + pass_at_1: float = 0.0 + pass_at_k: float = 0.0 + pass_hat_k: float = 0.0 def summarize(runs: Runs) -> dict[str, Any]: @@ -47,8 +52,91 @@ def summarize(runs: Runs) -> dict[str, Any]: } +def group_by_test_and_model(runs: Runs) -> dict[tuple[str, str], list[Mapping[str, Any]]]: + """Group runs by (test name, model).""" + groups: dict[tuple[str, str], list[Mapping[str, Any]]] = defaultdict(list) + for run in runs: + name = str(run.get("name") or "") + model = str(run.get("model") or UNKNOWN_MODEL) + groups[(name, model)].append(run) + return dict(groups) + + +def pass_metrics_for_group(runs: Runs) -> dict[str, float | int]: + """Compute empirical pass@1 / pass@k / pass^k for one (test, model) group. + + Because nao test --k N runs exactly N attempts per test case (not an + oversampled pool), these use the direct empirical definitions: + + - pass@1: mean of passed across the group's attempts + - pass@k: 1.0 if any attempt passed, else 0.0 + - pass^k: 1.0 if all attempts passed, else 0.0 + """ + outcomes = [bool(run.get("passed")) for run in runs] + k_count = len(outcomes) + if k_count == 0: + return {"k": 0, "pass_at_1": 0.0, "pass_at_k": 0.0, "pass_hat_k": 0.0} + + pass_at_1 = sum(1 for passed in outcomes if passed) / k_count + pass_at_k = 1.0 if any(outcomes) else 0.0 + pass_hat_k = 1.0 if all(outcomes) else 0.0 + return { + "k": k_count, + "pass_at_1": pass_at_1, + "pass_at_k": pass_at_k, + "pass_hat_k": pass_hat_k, + } + + +def summarize_pass_metrics(runs: Runs) -> dict[str, Any]: + """Summarize pass@k metrics per (test, model) and as suite aggregates. + + Suite-level pass_at_1 / pass_at_k / pass_hat_k are means across test-case + groups (not across raw attempts), so a case with k=5 counts once. A 75% + per-trial success rate over many independent cases therefore averages + pass^3 toward ~0.75^3 ≈ 42% at the suite level, even though each + individual test case's pass^k is a hard 0-or-1. + """ + groups = group_by_test_and_model(runs) + by_test: list[dict[str, Any]] = [] + for (name, model), group_runs in sorted(groups.items(), key=lambda item: (item[0][0], item[0][1])): + metrics = pass_metrics_for_group(group_runs) + by_test.append( + { + "name": name, + "model": model, + "k": metrics["k"], + "pass_at_1": metrics["pass_at_1"], + "pass_at_k": metrics["pass_at_k"], + "pass_hat_k": metrics["pass_hat_k"], + } + ) + + k_count = max((int(item["k"]) for item in by_test), default=0) + if by_test: + # Mean across test-case groups (not raw attempts) — see docstring. + aggregate = { + "pass_at_1": sum(float(item["pass_at_1"]) for item in by_test) / len(by_test), + "pass_at_k": sum(float(item["pass_at_k"]) for item in by_test) / len(by_test), + "pass_hat_k": sum(float(item["pass_hat_k"]) for item in by_test) / len(by_test), + } + else: + aggregate = {"pass_at_1": 0.0, "pass_at_k": 0.0, "pass_hat_k": 0.0} + + return { + "k": k_count, + "aggregate": aggregate, + "by_test": by_test, + } + + def summarize_by_model(runs: Runs) -> list[ModelSummary]: - """Aggregate runs per model, best pass rate first and cheapest as tie-breaker.""" + """Aggregate runs per model, best pass rate first and cheapest as tie-breaker. + + Token/cost/duration totals still sum every attempt. pass_at_1 / pass_at_k / + pass_hat_k average across test cases (each case counts once), matching + summarize_pass_metrics. + """ grouped: dict[str, list[Mapping[str, Any]]] = {} for run in runs: grouped.setdefault(str(run.get("model") or UNKNOWN_MODEL), []).append(run) @@ -70,6 +158,18 @@ def _summarize_model(model: str, runs: Runs) -> ModelSummary: total_duration_ms = _sum(runs, "duration_ms") total_tool_calls = _sum(runs, "tool_call_count") + # Pass@k metrics: group by test case first so k attempts count once. + by_test = group_by_test_and_model(runs) + per_test = [pass_metrics_for_group(group_runs) for group_runs in by_test.values()] + n_tests = len(per_test) + k_count = max((int(m["k"]) for m in per_test), default=0) + if n_tests: + pass_at_1 = sum(float(m["pass_at_1"]) for m in per_test) / n_tests + pass_at_k = sum(float(m["pass_at_k"]) for m in per_test) / n_tests + pass_hat_k = sum(float(m["pass_hat_k"]) for m in per_test) / n_tests + else: + pass_at_1 = pass_at_k = pass_hat_k = 0.0 + return ModelSummary( model=model, total=total, @@ -82,6 +182,10 @@ def _summarize_model(model: str, runs: Runs) -> ModelSummary: avg_duration_ms=round(total_duration_ms / total, 0) if total else 0.0, total_tool_calls=int(total_tool_calls), avg_tool_calls=round(total_tool_calls / total, 1) if total else 0.0, + k=k_count, + pass_at_1=pass_at_1, + pass_at_k=pass_at_k, + pass_hat_k=pass_hat_k, ) diff --git a/cli/nao_core/config/test/__init__.py b/cli/nao_core/config/test/__init__.py index 2ae06ac31..cb1a44043 100644 --- a/cli/nao_core/config/test/__init__.py +++ b/cli/nao_core/config/test/__init__.py @@ -23,6 +23,11 @@ class TestConfig(BaseModel): description="The models to run the tests against, in the format 'provider:model_id'", ) threads: int = Field(default=1, ge=1, description="Number of test runs to execute in parallel") + k: int = Field( + default=1, + ge=1, + description="Number of times to run each test case, used to compute pass@k and pass^k", + ) comparison: ComparisonConfig = Field( default_factory=ComparisonConfig, description="Tolerances used when comparing results to the expected data" ) diff --git a/cli/tests/nao_core/commands/test_runner.py b/cli/tests/nao_core/commands/test_runner.py index e62fb5954..15f8acd51 100644 --- a/cli/tests/nao_core/commands/test_runner.py +++ b/cli/tests/nao_core/commands/test_runner.py @@ -287,10 +287,13 @@ def test_filter_test_cases_by_name_without_tests_dir_unchanged(): assert filtered[0].name == "users" -def run_test_command(monkeypatch, tmp_path, config, tables: list | None = None, **flags) -> list[dict]: +def run_test_command( + monkeypatch, tmp_path, config, tables: list | None = None, saved_results: list | None = None, **flags +) -> list[dict]: """Run the `nao test` command against stubbed collaborators and report what it ran. Pass ``tables`` to also collect the (title, dataframe) pairs printed as summaries. + Pass ``saved_results`` to capture the results list handed to ``save_results``. """ cases = [ NaoTestCase(name="orders", prompt="p1", file_path=tmp_path / "orders.yml", sql="select 1"), @@ -306,11 +309,16 @@ def table(df, title=None, **kwargs): if tables is not None: tables.append((title, df)) + def save(results, output_dir): + if saved_results is not None: + saved_results.extend(results) + return output_dir / "results.json" + monkeypatch.chdir(tmp_path) monkeypatch.setattr(test_runner_module, "NaoConfig", Mock(try_load=Mock(return_value=config))) monkeypatch.setattr(test_runner_module, "discover_tests", lambda project_path: cases) monkeypatch.setattr(test_runner_module, "run_test", run) - monkeypatch.setattr(test_runner_module, "save_results", lambda results, output_dir: output_dir / "results.json") + monkeypatch.setattr(test_runner_module, "save_results", save) monkeypatch.setattr(test_runner_module.UI, "table", table) test_runner_module.test(**flags) @@ -331,10 +339,12 @@ def test_run_uses_the_test_block_defaults(tmp_path, monkeypatch): def test_run_falls_back_to_defaults_without_a_test_block(tmp_path, monkeypatch): - runs = run_test_command(monkeypatch, tmp_path, NaoConfig(project_name="test-project")) + saved: list[NaoTestRunResult] = [] + runs = run_test_command(monkeypatch, tmp_path, NaoConfig(project_name="test-project"), saved_results=saved) assert [str(run["model"]) for run in runs] == ["openai:gpt-4.1"] * 2 assert runs[0]["comparison"].decimals == 2 + assert [r.attempt for r in saved] == [1, 1] def test_model_flag_overrides_the_test_block(tmp_path, monkeypatch): @@ -346,28 +356,179 @@ def test_model_flag_overrides_the_test_block(tmp_path, monkeypatch): assert [str(run["model"]) for run in runs] == ["openai:gpt-4.1"] -def test_single_model_runs_only_print_the_run_table(tmp_path, monkeypatch): +def test_single_model_runs_print_results_and_summary_tables(tmp_path, monkeypatch): tables: list = [] run_test_command(monkeypatch, tmp_path, NaoConfig(project_name="test-project"), tables=tables) - assert [title for title, _ in tables] == ["Test Results"] + assert [title for title, _ in tables] == ["Test Results", "Summary"] + results_table = dict(tables)["Test Results"] + assert list(results_table.columns) == [ + "Test", + "Model", + "Status", + "Success %", + "Tokens", + "Cost", + "Time (s)", + "Tools", + ] + assert results_table["Success %"].tolist() == ["[green]100.0%[/green]"] * 2 -def test_multi_model_runs_print_per_model_summaries(tmp_path, monkeypatch): +def test_multi_model_runs_print_summary_and_matrix(tmp_path, monkeypatch): config = NaoConfig(project_name="test-project", test=TestConfig(models=["openai:gpt-4.1", "anthropic:claude-4-5"])) tables: list = [] run_test_command(monkeypatch, tmp_path, config, tables=tables) titles = [title for title, _ in tables] - assert titles == ["Test Results", "Performance by Model", "Pass / Fail by Test and Model"] + assert titles == [ + "Test Results", + "Summary", + "Pass / Fail by Test and Model", + ] matrix = dict(tables)["Pass / Fail by Test and Model"] assert list(matrix.columns) == ["Test", "openai\ngpt-4.1", "anthropic\nclaude-4-5"] assert matrix["Test"].tolist() == ["orders", "users"] +def test_results_table_status_requires_all_attempts_to_pass(monkeypatch): + results = [ + NaoTestRunResult(name="orders", model="m", passed=True, message="match"), + NaoTestRunResult(name="orders", model="m", passed=False, message="values differ"), + ] + tables: list = [] + + monkeypatch.setattr(test_runner_module.UI, "table", lambda df, title=None, **kwargs: tables.append((title, df))) + test_runner_module.print_run_table(results) + + table = dict(tables)["Test Results"] + assert table["Status"].tolist() == ["[red]✗[/red]"] + assert table["Success %"].tolist() == ["[yellow]50.0%[/yellow]"] + + +def test_model_matrix_status_requires_all_attempts_to_pass(monkeypatch): + results = [ + NaoTestRunResult(name="orders", model="m", passed=True, message="match"), + NaoTestRunResult(name="orders", model="m", passed=False, message="values differ"), + ] + tables: list = [] + + monkeypatch.setattr(test_runner_module.UI, "table", lambda df, title=None, **kwargs: tables.append((title, df))) + test_runner_module.print_model_matrix(results) + + table = dict(tables)["Pass / Fail by Test and Model"] + assert table["m"].tolist() == ["[red]✗[/red]"] + + +def test_summary_table_is_one_totals_row_with_always_pass_rate(monkeypatch): + results = [ + NaoTestRunResult( + name="orders", + model="m", + passed=True, + message="match", + tokens=100, + cost=0.1, + duration_ms=1000, + tool_call_count=2, + ), + NaoTestRunResult( + name="orders", + model="m", + passed=False, + message="values differ", + tokens=200, + cost=0.2, + duration_ms=2000, + tool_call_count=3, + ), + NaoTestRunResult( + name="users", + model="m", + passed=True, + message="match", + tokens=300, + cost=0.3, + duration_ms=3000, + tool_call_count=4, + ), + ] + tables: list = [] + + monkeypatch.setattr(test_runner_module.UI, "table", lambda df, title=None, **kwargs: tables.append((title, df))) + test_runner_module.print_summary_table(results) + + title, table = tables[0] + assert title == "Summary" + assert len(table) == 1 + assert table.iloc[0].to_dict() == { + "Tests": 2, + "Success %": "[yellow]66.7%[/yellow]", + "Always Pass %": "[yellow]50.0%[/yellow]", + "Tokens": 600, + "Cost": "$0.6000", + "Time (s)": 6.0, + "Tools": 9, + } + + +def test_k_flag_runs_each_case_k_times_with_attempt_index(tmp_path, monkeypatch): + config = NaoConfig(project_name="test-project", test=TestConfig(models=["openai:gpt-4.1"])) + saved: list[NaoTestRunResult] = [] + tables: list = [] + + runs = run_test_command(monkeypatch, tmp_path, config, tables=tables, saved_results=saved, k=3) + + # 2 test cases × 1 model × 3 attempts + assert len(runs) == 6 + assert [(r.name, r.attempt) for r in saved] == [ + ("orders", 1), + ("orders", 2), + ("orders", 3), + ("users", 1), + ("users", 2), + ("users", 3), + ] + results_table = dict(tables)["Test Results"] + assert len(results_table) == 2 + assert "Success %" in results_table.columns + assert dict(tables)["Summary"]["Always Pass %"].tolist() == ["[green]100.0%[/green]"] + + +def test_k_config_default_used_when_flag_omitted(tmp_path, monkeypatch): + config = NaoConfig(project_name="test-project", test=TestConfig(models=["openai:gpt-4.1"], k=2)) + saved: list[NaoTestRunResult] = [] + + runs = run_test_command(monkeypatch, tmp_path, config, saved_results=saved) + + assert len(runs) == 4 + assert sorted(r.attempt for r in saved if r.name == "orders") == [1, 2] + + +def test_k_flag_overrides_config(tmp_path, monkeypatch): + config = NaoConfig(project_name="test-project", test=TestConfig(models=["openai:gpt-4.1"], k=5)) + saved: list[NaoTestRunResult] = [] + + runs = run_test_command(monkeypatch, tmp_path, config, saved_results=saved, k=2, select="orders") + + assert len(runs) == 2 + assert [r.attempt for r in saved] == [1, 2] + + +def test_invalid_k_is_rejected(tmp_path, monkeypatch): + errors: list[str] = [] + monkeypatch.setattr(test_runner_module.UI, "error", lambda msg: errors.append(msg)) + config = NaoConfig(project_name="test-project", test=TestConfig(models=["openai:gpt-4.1"])) + + runs = run_test_command(monkeypatch, tmp_path, config, k=0) + + assert runs == [] + assert errors and "k must be >= 1" in errors[0] + + def test_threaded_runs_are_reported_grouped_by_model(tmp_path, monkeypatch): config = NaoConfig(project_name="test-project", test=TestConfig(models=["openai:gpt-4.1", "anthropic:claude-4-5"])) cases = [ @@ -406,6 +567,7 @@ def save_results(results, output_dir): ("anthropic:claude-4-5", "orders"), ("anthropic:claude-4-5", "users"), ] + assert [r.attempt for r in saved] == [1, 1, 1, 1] def test_save_results_records_per_model_summaries(tmp_path): @@ -419,6 +581,7 @@ def test_save_results_records_per_model_summaries(tmp_path): cost=0.2, duration_ms=1000, tool_call_count=2, + attempt=1, ), NaoTestRunResult( name="orders", @@ -429,6 +592,7 @@ def test_save_results_records_per_model_summaries(tmp_path): cost=0.1, duration_ms=3000, tool_call_count=4, + attempt=1, ), ] @@ -439,3 +603,7 @@ def test_save_results_records_per_model_summaries(tmp_path): assert [model["model"] for model in data["by_model"]] == ["openai:gpt-4.1", "anthropic:claude-4-5"] assert data["by_model"][0]["pass_rate"] == 100.0 assert data["by_model"][1]["avg_duration_ms"] == 3000 + assert "pass_metrics" in data + assert data["pass_metrics"]["k"] == 1 + assert set(data["pass_metrics"]["aggregate"].keys()) == {"pass_at_1", "pass_at_k", "pass_hat_k"} + assert data["results"][0]["attempt"] == 1 diff --git a/cli/tests/nao_core/commands/test_summary.py b/cli/tests/nao_core/commands/test_summary.py index ed24a3e4f..415e7fbf8 100644 --- a/cli/tests/nao_core/commands/test_summary.py +++ b/cli/tests/nao_core/commands/test_summary.py @@ -1,4 +1,11 @@ -from nao_core.commands.test.summary import summarize, summarize_by_model, with_model_summaries +from nao_core.commands.test.summary import ( + group_by_test_and_model, + pass_metrics_for_group, + summarize, + summarize_by_model, + summarize_pass_metrics, + with_model_summaries, +) def run(model: str, passed: bool, **overrides) -> dict: @@ -11,6 +18,11 @@ def run(model: str, passed: bool, **overrides) -> dict: "cost": overrides.get("cost", 0.01), "duration_ms": overrides.get("duration_ms", 2000), "tool_call_count": overrides.get("tool_call_count", 3), + **{ + key: value + for key, value in overrides.items() + if key not in {"name", "tokens", "cost", "duration_ms", "tool_call_count"} + }, } @@ -73,3 +85,137 @@ def test_with_model_summaries_keeps_existing_summaries(): data = {"results": [run("openai:gpt-4.1", True)], "by_model": [{"model": "kept"}]} assert with_model_summaries(data)["by_model"] == [{"model": "kept"}] + + +def test_pass_metrics_partial_group(): + runs = [ + run("openai:gpt-4.1", True), + run("openai:gpt-4.1", True), + run("openai:gpt-4.1", False), + ] + + metrics = pass_metrics_for_group(runs) + + assert metrics["k"] == 3 + assert metrics["pass_at_1"] == 2 / 3 + assert metrics["pass_at_k"] == 1.0 + assert metrics["pass_hat_k"] == 0.0 + + +def test_pass_metrics_all_passed(): + runs = [run("m", True), run("m", True), run("m", True)] + + metrics = pass_metrics_for_group(runs) + + assert metrics["pass_at_1"] == 1.0 + assert metrics["pass_at_k"] == 1.0 + assert metrics["pass_hat_k"] == 1.0 + + +def test_pass_metrics_all_failed(): + runs = [run("m", False), run("m", False)] + + metrics = pass_metrics_for_group(runs) + + assert metrics["pass_at_1"] == 0.0 + assert metrics["pass_at_k"] == 0.0 + assert metrics["pass_hat_k"] == 0.0 + + +def test_pass_metrics_k_equals_one(): + assert pass_metrics_for_group([run("m", True)]) == { + "k": 1, + "pass_at_1": 1.0, + "pass_at_k": 1.0, + "pass_hat_k": 1.0, + } + assert pass_metrics_for_group([run("m", False)]) == { + "k": 1, + "pass_at_1": 0.0, + "pass_at_k": 0.0, + "pass_hat_k": 0.0, + } + + +def test_suite_aggregation_weights_by_test_case_not_attempts(): + # One test with k=5 (all pass) and one with k=1 (fail) must average 0.5 each metric, + # not weight the k=5 case five times. + runs = [ + run("m", True, name="a"), + run("m", True, name="a"), + run("m", True, name="a"), + run("m", True, name="a"), + run("m", True, name="a"), + run("m", False, name="b"), + ] + + summary = summarize_pass_metrics(runs) + + assert summary["k"] == 5 + assert summary["aggregate"]["pass_at_1"] == 0.5 + assert summary["aggregate"]["pass_at_k"] == 0.5 + assert summary["aggregate"]["pass_hat_k"] == 0.5 + assert len(summary["by_test"]) == 2 + + +def test_summarize_pass_metrics_output_shape(): + runs = [ + run("openai:gpt-4.1", True, name="orders", attempt=1), + run("openai:gpt-4.1", False, name="orders", attempt=2), + run("anthropic:claude", True, name="users", attempt=1), + ] + + summary = summarize_pass_metrics(runs) + + assert set(summary.keys()) == {"k", "aggregate", "by_test"} + assert set(summary["aggregate"].keys()) == {"pass_at_1", "pass_at_k", "pass_hat_k"} + assert summary["k"] == 2 + for item in summary["by_test"]: + assert set(item.keys()) == {"name", "model", "k", "pass_at_1", "pass_at_k", "pass_hat_k"} + + +def test_group_by_test_and_model(): + runs = [ + run("m1", True, name="a"), + run("m1", False, name="a"), + run("m2", True, name="a"), + run("m1", True, name="b"), + ] + + groups = group_by_test_and_model(runs) + + assert set(groups.keys()) == {("a", "m1"), ("a", "m2"), ("b", "m1")} + assert len(groups[("a", "m1")]) == 2 + + +def test_summarize_by_model_includes_pass_metrics_averaged_across_tests(): + runs = [ + run("m1", True, name="a"), + run("m1", False, name="a"), # pass@1=0.5, pass@k=1, pass^k=0 + run("m1", True, name="b"), # pass@1=1, pass@k=1, pass^k=1 + run("m2", False, name="a"), + ] + + by_model = {s.model: s for s in summarize_by_model(runs)} + + assert by_model["m1"].pass_at_1 == 0.75 + assert by_model["m1"].pass_at_k == 1.0 + assert by_model["m1"].pass_hat_k == 0.5 + assert by_model["m1"].k == 2 + assert by_model["m2"].pass_at_1 == 0.0 + + +def test_with_model_summaries_handles_old_results_without_k_or_pass_metrics(): + data = { + "results": [ + run("openai:gpt-4.1", True, name="orders"), + run("openai:gpt-4.1", False, name="users"), + ], + "summary": {"total": 2, "passed": 1, "failed": 1}, + } + + backfilled = with_model_summaries(data) + + assert "pass_metrics" not in data # input unchanged shape is fine + assert backfilled["by_model"][0]["pass_rate"] == 50.0 + assert backfilled["by_model"][0]["k"] == 1 diff --git a/cli/tests/nao_core/config/test_test_config.py b/cli/tests/nao_core/config/test_test_config.py index 62d8a49db..1e989c9aa 100644 --- a/cli/tests/nao_core/config/test_test_config.py +++ b/cli/tests/nao_core/config/test_test_config.py @@ -41,6 +41,7 @@ def test_defaults(): assert test_config.models == ["openai:gpt-4.1"] assert test_config.threads == 1 + assert test_config.k == 1 assert test_config.comparison.rtol == 1e-5 assert test_config.comparison.atol == 1e-8 assert test_config.comparison.decimals == 2 @@ -55,3 +56,18 @@ def test_models_must_declare_a_provider_and_a_model_id(model): def test_threads_must_be_positive(): with pytest.raises(ValidationError): TestConfig(threads=0) + + +def test_k_must_be_positive(): + with pytest.raises(ValidationError): + TestConfig(k=0) + + +def test_k_is_loaded_from_config(tmp_path): + config_file = tmp_path / "nao_config.yaml" + config_file.write_text("project_name: test-project\ntest:\n k: 5\n") + + config = NaoConfig.load(tmp_path) + + assert config.test is not None + assert config.test.k == 5