Skip to content

feat(cli): Add pass@k and pass^k metrics to nao test KPIs #1368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
justin212407 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(cli): Add pass@k and pass^k metrics to nao test KPIs #1368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c7a048d
feat: Add pass@k and pass^k metrics to nao test KPIs
justin212407 Aug 8, 2026
43f6c8a
feat: reiterate on intended UI
justin212407 Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion cli/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ Options:

- `--model` / `-m`: Models to test against (default: `openai:gpt-4.1`). Can be specified multiple times.
- `--threads` / `-t`: Number of parallel threads (default: `1`)
- `--k` / `-k`: Number of times to run each test case, used to compute pass@k and pass^k (default: `1`)
- `--select` / `-s`: Run only selected tests by name, yaml stem, or subfolder. Comma-separated.
- `--username` / `-u`, `--password`: Credentials for the nao backend. Fall back to `NAO_USERNAME` / `NAO_PASSWORD`.

Expand All @@ -216,16 +217,18 @@ Examples:
nao test -m openai:gpt-4.1
nao test -m openai:gpt-4.1 -m anthropic:claude-sonnet-4-20250514
nao test --threads 4
nao test --k 5
```

Defaults for every run live in the `test` block of `nao_config.yaml`, and the `--model` / `--threads` flags override them:
Defaults for every run live in the `test` block of `nao_config.yaml`, and the `--model` / `--threads` / `--k` flags override them:

```yaml
test:
models:
- openai:gpt-4.1
- anthropic:claude-sonnet-4-5
threads: 4
# k: 5 # run each case 5 times to compute pass@k / pass^k
comparison:
rtol: 0.00001
atol: 0.00000001
Expand Down
151 changes: 96 additions & 55 deletions cli/nao_core/commands/test/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,13 @@
from .case import TESTS_FOLDER, TestCase, discover_tests
from .client import BACKEND_URL, AgentClientError, VerificationResult, get_client
from .compare import normalize_dataframe_numbers
from .summary import ModelSummary, summarize, summarize_by_model
from .summary import (
group_by_test_and_model,
pass_metrics_for_group,
summarize,
summarize_by_model,
summarize_pass_metrics,
)


@dataclass
Expand Down Expand Up @@ -70,6 +76,7 @@ class TestRunResult:
tool_call_count: int | None = None
error: str | None = None
details: TestRunDetails | None = None
attempt: int | None = None


def check_dataframe(
Expand Down Expand Up @@ -295,59 +302,62 @@ def save_results(results: list[TestRunResult], output_dir: Path) -> Path:
"results": runs,
"summary": summarize(runs),
"by_model": [asdict(s) for s in summarize_by_model(runs)],
"pass_metrics": summarize_pass_metrics(runs),
}

output_file.write_text(json.dumps(data, indent=2))
return output_file


def print_run_table(results: list[TestRunResult]) -> None:
"""Print one row per run, i.e. per (test, model) pair."""
df = pd.DataFrame(
[
"""Print one row per (test, model), aggregating all attempts for that pair."""
run_dicts = [asdict(r) for r in results]
rows = []
for (name, model), group in sorted(group_by_test_and_model(run_dicts).items()):
metrics = pass_metrics_for_group(group)
rows.append(
{
"Test": r.name,
"Model": r.model,
"Status": status_icon(r.passed),
"Message": r.message,
"Tokens": r.tokens or 0,
"Cost": r.cost or 0,
"Time (s)": round((r.duration_ms or 0) / 1000, 1),
"Tools": r.tool_call_count or 0,
"Test": name,
"Model": model,
"Status": status_icon(float(metrics["pass_hat_k"]) == 1.0),
"Success %": format_pass_fraction(float(metrics["pass_at_1"])),

@cubic-dev-ai cubic-dev-ai Bot Aug 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: The PR states the CLI table shows pass@k and pass^k columns when k>1, but the code never renders pass@k (any attempt passed). The run table exposes only pass@1 ("Success %") and pass^k ("Status"/"Always Pass %"), so pass@k is available only in the saved JSON. If pass@k should be visible in the table output as described, add a column driven by metrics["pass_at_k"] (and pass_hat_k) rather than only pass_at_1.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At cli/nao_core/commands/test/runner.py, line 323:

<comment>The PR states the CLI table shows pass@k and pass^k columns when k>1, but the code never renders pass@k (any attempt passed). The run table exposes only pass@1 ("Success %") and pass^k ("Status"/"Always Pass %"), so pass@k is available only in the saved JSON. If pass@k should be visible in the table output as described, add a column driven by metrics["pass_at_k"] (and pass_hat_k) rather than only pass_at_1.</comment>

<file context>
@@ -295,59 +302,62 @@ def save_results(results: list[TestRunResult], output_dir: Path) -> Path:
+                "Test": name,
+                "Model": model,
+                "Status": status_icon(float(metrics["pass_hat_k"]) == 1.0),
+                "Success %": format_pass_fraction(float(metrics["pass_at_1"])),
+                "Tokens": sum(run.get("tokens") or 0 for run in group),
+                "Cost": sum(run.get("cost") or 0 for run in group),
</file context>

"Tokens": sum(run.get("tokens") or 0 for run in group),
"Cost": sum(run.get("cost") or 0 for run in group),
"Time (s)": round(sum(run.get("duration_ms") or 0 for run in group) / 1000, 1),
"Tools": sum(run.get("tool_call_count") or 0 for run in group),
}
for r in results
]
)

UI.table(df, title="Test Results", sum_columns={"Tokens": "", "Cost": "$", "Time (s)": "", "Tools": ""})


def print_model_table(summaries: list[ModelSummary]) -> None:
"""Print one row per model, ranked from best to worst pass rate."""
df = pd.DataFrame(
[
{
"Model": column_label(s.model),
"Pass Rate": format_pass_rate(s.pass_rate),
"Passed": f"{s.passed}/{s.total}",
"Tokens": s.total_tokens,
"Cost": s.total_cost,
"Avg Time (s)": round(s.avg_duration_ms / 1000, 1),
"Avg Tools": s.avg_tool_calls,
}
for s in summaries
]
)
)

UI.table(df, title="Performance by Model", sum_columns={"Tokens": "", "Cost": "$"}, fixed_columns={"Model"})
UI.table(pd.DataFrame(rows), title="Test Results")


def print_summary_table(results: list[TestRunResult]) -> None:
"""Print one totals row across all test/model pairs and attempts."""
run_dicts = [asdict(r) for r in results]
groups = group_by_test_and_model(run_dicts)
total = len(run_dicts)
passed = sum(1 for run in run_dicts if run.get("passed"))
always_passed = sum(1 for group in groups.values() if all(run.get("passed") for run in group))
total_duration_ms = sum(run.get("duration_ms") or 0 for run in run_dicts)

row = {
"Tests": len(groups),
"Success %": format_pass_fraction(passed / total if total else 0.0),
"Always Pass %": format_pass_fraction(always_passed / len(groups) if groups else 0.0),
"Tokens": sum(run.get("tokens") or 0 for run in run_dicts),
"Cost": f"${sum(run.get('cost') or 0 for run in run_dicts):.4f}",
"Time (s)": round(total_duration_ms / 1000, 1),
"Tools": sum(run.get("tool_call_count") or 0 for run in run_dicts),
}
UI.table(pd.DataFrame([row]), title="Summary")


def print_model_matrix(results: list[TestRunResult]) -> None:
"""Print a test × model grid to show which model passes which test."""
models = list(dict.fromkeys(r.model for r in results))
statuses: dict[tuple[str, str], list[str]] = {}
for result in results:
statuses.setdefault((result.name, result.model), []).append(status_icon(result.passed))
for key, group in group_by_test_and_model([asdict(result) for result in results]).items():
statuses[key] = [status_icon(all(bool(run.get("passed")) for run in group))]

rows = [
{"Test": name}
Expand All @@ -368,10 +378,11 @@ def status_icon(passed: bool) -> str:
return "[green]✓[/green]" if passed else "[red]✗[/red]"


def format_pass_rate(pass_rate: float) -> str:
"""Colour a pass rate from green (all passing) to red (mostly failing)."""
color = "green" if pass_rate == 100 else "red" if pass_rate < 50 else "yellow"
return f"[{color}]{pass_rate}%[/{color}]"
def format_pass_fraction(rate: float) -> str:
"""Colour a 0-1 pass metric (pass@k / pass^k) for terminal tables."""
pct = round(rate * 100, 1)
color = "green" if pct == 100 else "red" if pct < 50 else "yellow"
return f"[{color}]{pct}%[/{color}]"


def filter_test_cases(
Expand Down Expand Up @@ -452,6 +463,13 @@ def test(
help="Number of parallel threads for running tests. Overrides test.threads.",
),
] = None,
k: Annotated[
int | None,
Parameter(
name=["-k", "--k"],
help="Number of times to run each test case, used to compute pass@k and pass^k. Overrides test.k.",
),
] = None,
select: Annotated[
str | None,
Parameter(
Expand Down Expand Up @@ -483,6 +501,7 @@ def test(
nao test -m openai:gpt-4.1
nao test -m openai:gpt-4.1 -m anthropic:claude-sonnet-4-20250514
nao test --threads 4
nao test --k 5
nao test -s test_name
nao test -s 12,13,14
nao test -u user@example.com --password secret
Expand All @@ -497,6 +516,11 @@ def test(

test_config = config.test or TestConfig()
thread_count = threads if threads is not None else test_config.threads
k_count = k if k is not None else test_config.k

if k_count < 1:
UI.error(f"k must be >= 1, got {k_count}")
return

try:
model_configs = [ModelConfig.parse(m) for m in models or test_config.models]
Expand All @@ -508,7 +532,10 @@ def test(
tests_dir = project_path / TESTS_FOLDER
UI.print(f"[dim]Project: {config.project_name}[/dim]")
UI.print(f"[dim]Tests folder: {tests_dir}[/dim]")
UI.print(f"[dim]Models: {', '.join(str(m) for m in model_configs)}[/dim]\n")
UI.print(f"[dim]Models: {', '.join(str(m) for m in model_configs)}[/dim]")
if k_count > 1:
UI.print(f"[dim]k: {k_count}[/dim]")
UI.print("")

test_cases = discover_tests(project_path)

Expand All @@ -524,18 +551,27 @@ def test(

ensure_verification_engine(test_cases)

total_runs = len(test_cases) * len(model_configs)
UI.print(f"[bold]Found {len(test_cases)} test(s) × {len(model_configs)} model(s) = {total_runs} run(s)[/bold]")
total_runs = len(test_cases) * len(model_configs) * k_count
UI.print(
f"[bold]Found {len(test_cases)} test(s) × {len(model_configs)} model(s) × {k_count} attempt(s) = {total_runs} run(s)[/bold]"
if k_count > 1
else f"[bold]Found {len(test_cases)} test(s) × {len(model_configs)} model(s) = {total_runs} run(s)[/bold]"
)
if thread_count > 1:
UI.print(f"[dim]Running with {thread_count} threads (output may be interleaved)[/dim]")
UI.print("")

# Build list of (test_case, model) pairs
test_runs = [(test_case, model) for model in model_configs for test_case in test_cases]
# Build list of (test_case, model, attempt) triples — attempt is 1-indexed
test_runs = [
(test_case, model, attempt)
for model in model_configs
for test_case in test_cases
for attempt in range(1, k_count + 1)
]

results: list[TestRunResult] = []
if thread_count == 1:
for test_case, model in test_runs:
for test_case, model, attempt in test_runs:
result = run_test(
test_case,
model,
Expand All @@ -544,6 +580,7 @@ def test(
costs=resolve_model_costs(config, model),
comparison=test_config.comparison,
)
result.attempt = attempt
results.append(result)
UI.print("")
else:
Expand All @@ -559,11 +596,14 @@ def test(
password=pwd,
costs=resolve_model_costs(config, m),
comparison=test_config.comparison,
): index
for index, (tc, m) in enumerate(test_runs)
): (index, attempt)
for index, (tc, m, attempt) in enumerate(test_runs)
}
for future in as_completed(futures):
completed[futures[future]] = future.result()
index, attempt = futures[future]
result = future.result()
result.attempt = attempt
completed[index] = result
UI.print("")
results = [completed[index] for index in sorted(completed)]

Expand All @@ -573,16 +613,17 @@ def test(

print_run_table(results)

model_summaries = summarize_by_model([asdict(r) for r in results])
run_dicts = [asdict(r) for r in results]
model_summaries = summarize_by_model(run_dicts)

print_summary_table(results)
if len(model_summaries) > 1:
print_model_table(model_summaries)
print_model_matrix(results)

passed = sum(1 for r in results if r.passed)
failed = sum(1 for r in results if not r.passed)
total = len(results)
unit = "run" if len(model_summaries) > 1 else "test"

unit = "run" if len(model_summaries) > 1 or k_count > 1 else "test"
UI.print("")
if failed == 0:
UI.success(f"All {total} {unit}(s) passed")
Expand Down
Loading
Loading