diff --git a/fern/versions/latest/pages/reference/cli-commands.mdx b/fern/versions/latest/pages/reference/cli-commands.mdx index 14e12b6d99..fd838ddcf7 100644 --- a/fern/versions/latest/pages/reference/cli-commands.mdx +++ b/fern/versions/latest/pages/reference/cli-commands.mdx @@ -53,6 +53,7 @@ gym eval health-check # verify rollout artifact quality for an existing gym eval profile # compute a reward profile from rollouts gym eval reverify # recompute rewards from existing rollouts without re-running inference gym eval compare # compare a baseline eval run against a candidate run +gym eval stat-test # paired statistical significance test of a baseline run vs a candidate run # Contributor helpers gym dev test # run NeMo Gym's unit tests @@ -777,7 +778,9 @@ See [Repeat-Level Metrics](/evaluation/aggregate-metrics#repeat-level-metrics) f Compare a baseline eval run against a candidate run and write a report. -The report contains a metric table (the change, plus each side's value and confidence interval), split into key metrics and all other metrics, and a per-task sample-flips table. Confidence intervals are read from what each run recorded; this command does not yet perform statistical tests or issue a pass/fail verdict. +The report contains a metric table (the change, plus each side's value and confidence interval), split into key metrics and all other metrics, and a per-task sample-flips table. Confidence intervals are read from what each run recorded. + +By default, `gym eval compare` also runs a paired statistical significance test on every key metric with per-task pairing data (the same test [`gym eval stat-test`](#gym-eval-stat-test) runs standalone) and writes it to a separate `statistical_tests/` subdirectory next to the report — `compare_report.md`/`compare_report.json` themselves are unaffected by this step; it's a side effect, not a schema change. | Option | Description | | --- | --- | @@ -790,6 +793,11 @@ The report contains a metric table (the change, plus each side's value and confi | `--candidate-agents NAME[,NAME...]` | Agent to read from each candidate, in `--candidates` order. Takes precedence over `--agent`. | | `--output-dir DIR`, `-o` | Where to write the report. Defaults to the directory containing the candidate rollouts JSONL. | | `--report-format` | `md`, `json`, or `both` (default). | +| `--metric METRIC[,METRIC...]` | Restrict the statistics step to specific metric(s). Defaults to every key metric with per-task pairing data. | +| `--margin DELTA` | Non-inferiority margin for the statistics step (e.g. `0.01` for 1pp). See [`gym eval stat-test`](#gym-eval-stat-test) for the two framings this switches between. | +| `--alpha` | Significance level for the statistics step (default `0.05`). | +| `--stats-output-dir DIR` | Where to write the statistics artifacts. Independent of `--output-dir`, which only controls `compare_report.*`. Defaults to `statistical_tests/` inside the candidate run's own directory. | +| `--no-stats` | Skip the statistics step entirely — no `statistical_tests/` directory is written. | ```bash gym eval compare \ @@ -803,9 +811,55 @@ Writes into the output directory: | --- | --- | | `compare_report.md` | Human-readable report. | | `compare_report.json` | Machine-readable result (`schema_version` `"1"`). | +| `statistical_tests/*` | The default statistics step's own artifacts (see `--no-stats` above). | If a run was collected with `--disable-aggregation` it has no `*_aggregate_metrics.json`; run [`gym eval aggregate`](#gym-eval-aggregate) first, or point at the file directly with `--baseline-agg-metrics` / `--candidates-agg-metrics`. +### `gym eval stat-test` + +Paired statistical significance test of a baseline eval run against a candidate run — the same test `gym eval compare` runs by default, but standalone: `gym eval stat-test` never depends on `compare` having been run, and never reads or writes `compare_report.*`. + +The method is a single paired test on per-task mean scores (Evan Miller, "Adding Error Bars to Evals", arXiv:2411.00640): each task's repeats are collapsed to one mean, baseline and candidate are paired per task, and the Central Limit Theorem licenses a t-test on the average of the paired differences. Two framings, selected by whether `--margin` is given: + +- **No `--margin`**: two-sided test of "did anything change at all" — for repeatability checks (same benchmark/model run twice). +- **`--margin DELTA`**: one-sided non-inferiority test of "is the candidate not meaningfully worse than `DELTA`" — for regression checks with a tolerance (e.g. FP4 quantization: is accuracy within 1pp of the baseline). + +Reports only the raw statistical output (n, mean difference, standard error, p-value, significant, minimum detectable effect at 80% power) — no PASS/WARN/FAIL verdict. + +| Option | Description | +| --- | --- | +| `--baseline PATH` | Baseline run's rollouts JSONL. Its `*_aggregate_metrics.json` sibling is what gets read. | +| `--candidates PATH[,PATH...]` | Candidate run's rollouts JSONL. One candidate is supported today. | +| `--baseline-agg-metrics PATH` | Baseline's aggregate-metrics JSON, when it is not the sibling of `--baseline`. | +| `--candidates-agg-metrics PATH[,PATH...]` | Candidate's aggregate-metrics JSON, when not the sibling of `--candidates`. | +| `--agent NAME`, `--baseline-agent NAME`, `--candidate-agents NAME[,NAME...]` | Same agent-selection semantics as `gym eval compare`. | +| `--test NAME` | Which statistical test to run. `paired` (the default) is the only one today; `--metric` and `--margin` below belong to it. | +| `--metric METRIC[,METRIC...]` | Metric(s) to test (bare field name, e.g. `reward`, not `mean/reward`). Defaults to every key metric with per-task pairing data. | +| `--margin DELTA` | Non-inferiority margin. See above. | +| `--alpha` | Significance level (default `0.05`). | +| `--output-dir DIR`, `-o` | Given explicitly, used literally. Left unset, defaults to `statistical_tests/` inside the candidate run's own directory (auto-created). | +| `--report-format` | `md`, `json`, or `both` (default). | + +```bash +# Repeatability: is the difference between two runs of the same config noise, or real? +gym eval stat-test \ + --baseline runs/run_a/rollouts.jsonl \ + --candidates runs/run_b/rollouts.jsonl + +# FP4 quantization: is the candidate not meaningfully worse than a 1pp margin? +gym eval stat-test \ + --baseline runs/bf16/rollouts.jsonl \ + --candidates runs/nvfp4/rollouts.jsonl \ + --metric reward --margin 0.01 +``` + +Writes into `statistical_tests/` (or `--output-dir` directly, with no nesting, when given explicitly), with the filename indexed by this invocation's parameters — a rerun with a different `--metric`/`--margin`/`--alpha` against the same location writes an additional file rather than overwriting the previous one: + +``` +statistical_tests/paired__two-sided__alpha-0.05.{md,json} +statistical_tests/paired__metric-reward__margin-0.01__alpha-0.05.{md,json} +``` + ### `gym eval reverify` Recompute rewards from existing rollouts by replaying them through a resources server's `/verify` endpoint — without re-running model inference. Starts the resources server automatically from the provided config. Requires the `*_materialized_inputs.jsonl` and `rollouts.jsonl` artifacts produced by `gym eval run`. diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index fbb0fc8958..b64fb830d1 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -20,7 +20,7 @@ from copy import deepcopy from multiprocessing import Pool from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple from omegaconf import DictConfig, OmegaConf, open_dict from pydantic import Field @@ -602,16 +602,67 @@ def reward_profile(): # pragma: no cover Repeat-level metrics: {repeat_level_metrics_fpath}""") +def _run_stats_step_for_compare(config: Any, overrides: Dict[str, Any]) -> None: + """Build the `stat-test` config for `gym eval compare`'s default stats step and run it. + + `overrides` carries the raw stats-flag values off `eval compare`'s own global config dict + (`metric`/`margin`/`alpha`/`stats_output_dirpath`) -- `stats_output_dirpath` is remapped to + `output_dirpath` here since `--output-dir` on `compare` must keep controlling only + `compare_report.*`, not this step's own output location. + """ + from nemo_gym.statistical_tests.registry import resolve_stat_test + from nemo_gym.statistical_tests.schema import DEFAULT_STAT_TEST + + stats_config_dict = config.model_dump(exclude={"output_dirpath"}) + stats_config_dict.update({k: v for k, v in overrides.items() if k != "stats_output_dirpath"}) + if overrides.get("stats_output_dirpath"): + stats_config_dict["output_dirpath"] = overrides["stats_output_dirpath"] + test = resolve_stat_test(stats_config_dict.get("test") or DEFAULT_STAT_TEST) + stat_test(test.config_type.model_validate(stats_config_dict)) + + @exit_cleanly_on_config_error def compare() -> None: # pragma: no cover from nemo_gym.comparison.report import render_key_metrics_tables, summary_lines from nemo_gym.comparison.runner import invoked_command, run_comparison from nemo_gym.comparison.schema import ComparisonConfig - config = ComparisonConfig.model_validate(get_global_config_dict()) + global_config_dict = get_global_config_dict() + config = ComparisonConfig.model_validate(global_config_dict) result, written = run_comparison(config, invoked_command()) for table in render_key_metrics_tables(result): print_rich_table(table) print("\n".join(summary_lines(result, written))) + + if not global_config_dict.get("no_stats", False): + _run_stats_step_for_compare( + config, + { + "metric": global_config_dict.get("metric"), + "margin": global_config_dict.get("margin"), + "alpha": global_config_dict.get("alpha", 0.05), + "stats_output_dirpath": global_config_dict.get("stats_output_dirpath"), + }, + ) + + +@exit_cleanly_on_config_error +def stat_test(config: Optional[Any] = None) -> None: # pragma: no cover + from nemo_gym.statistical_tests.common import invoked_command + from nemo_gym.statistical_tests.registry import resolve_stat_test, run_stat_test + from nemo_gym.statistical_tests.schema import DEFAULT_STAT_TEST + + standalone = config is None + if standalone: + global_config_dict = get_global_config_dict() + test = resolve_stat_test(global_config_dict.get("test") or DEFAULT_STAT_TEST) + config = test.config_type.model_validate(global_config_dict) + else: + test = resolve_stat_test(config.test) + + # Record whichever command actually ran: sys.argv holds *its* overrides, not stat-test's. + report, written = run_stat_test(test, config, invoked_command("stat-test" if standalone else "compare")) + + print("\n".join(test.summary(report, written))) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 2f9a98294e..9ca0f2c11f 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -631,6 +631,61 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: dispatch(targets[args.storage], overrides) +# Run-selection flags shared by `eval compare` and `eval stat-test` -- both read the same +# `_aggregate_metrics.json` pair and pick an agent the same way. +_EVAL_RUN_SELECTION_FLAGS = ( + _value_flag( + "baseline", + "baseline_rollouts_jsonl_fpath", + "Baseline run's rollouts JSONL (its *_aggregate_metrics.json sibling is what gets read today).", + quote=True, + ), + _comma_list_flag( + "candidates", + "candidate_rollouts_jsonl_fpaths", + "Candidate run's rollouts JSONL. Comma-separated list; one candidate is supported today.", + metavar="PATH[,PATH...]", + ), + _value_flag( + "baseline-agg-metrics", + "baseline_aggregate_metrics_fpath", + "Baseline's aggregate-metrics JSON, when it is not the sibling of --baseline.", + quote=True, + ), + _comma_list_flag( + "candidates-agg-metrics", + "candidate_aggregate_metrics_fpaths", + "Candidates' aggregate-metrics JSON, in --candidates order, when not siblings of --candidates.", + metavar="PATH[,PATH...]", + ), + _value_flag("agent", "agent_name", "Agent to compare on both sides (default: all shared agents)."), + _value_flag("baseline-agent", "baseline_agent_name", "Agent to read from the baseline's metrics."), + _comma_list_flag( + "candidate-agents", + "candidate_agent_names", + "Agent to read from each candidate's metrics, in --candidates order.", + metavar="NAME[,NAME...]", + ), +) + +# Statistical-test flags shared by `eval compare` (its default stats step) and `eval stat-test`. +_STATISTICAL_TEST_FLAGS = ( + _comma_list_flag( + "metric", + "metric", + "Metric(s) to test, e.g. `reward` (comma-separated). Default: every key metric with per-task pairing data.", + metavar="METRIC[,METRIC...]", + ), + _value_flag( + "margin", + "margin", + "Non-inferiority margin delta (e.g. 0.01 for 1pp). Given: one-sided test of 'candidate is not " + "meaningfully worse than delta'. Omitted: two-sided test of 'did anything change at all'.", + ), + _value_flag("alpha", "alpha", "Significance level (default: 0.05)."), +) + + # One-line help for each command group, shown in `gym --help`. GROUPS = { "list": "List available components (benchmarks, environments, agents, models, resources-servers).", @@ -1074,42 +1129,41 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: target="nemo_gym.cli.eval:compare", summary="Compare a baseline eval run against candidate runs.", flags=( + *_EVAL_RUN_SELECTION_FLAGS, _value_flag( - "baseline", - "baseline_rollouts_jsonl_fpath", - "Baseline run's rollouts JSONL (its *_aggregate_metrics.json sibling is what gets read today).", + "output-dir", + "output_dirpath", + "Where to write the report (default: the candidate run's own directory).", + aliases=("-o",), quote=True, ), - _comma_list_flag( - "candidates", - "candidate_rollouts_jsonl_fpaths", - "Candidate run's rollouts JSONL. Comma-separated list; one candidate is supported today.", - metavar="PATH[,PATH...]", + _value_flag( + "report-format", + "report_format", + "Report artifacts to write (default: both).", + choices=("md", "json", "both"), ), + *_STATISTICAL_TEST_FLAGS, _value_flag( - "baseline-agg-metrics", - "baseline_aggregate_metrics_fpath", - "Baseline's aggregate-metrics JSON, when it is not the sibling of --baseline.", + "stats-output-dir", + "stats_output_dirpath", + "Where to write the statistical-test step's own report (default: " + "`/statistical_tests/`). Independent of --output-dir, which " + "controls only compare_report.*.", quote=True, ), - _comma_list_flag( - "candidates-agg-metrics", - "candidate_aggregate_metrics_fpaths", - "Candidates' aggregate-metrics JSON, in --candidates order, when not siblings of --candidates.", - metavar="PATH[,PATH...]", - ), - _value_flag("agent", "agent_name", "Agent to compare on both sides (default: all shared agents)."), - _value_flag("baseline-agent", "baseline_agent_name", "Agent to read from the baseline's metrics."), - _comma_list_flag( - "candidate-agents", - "candidate_agent_names", - "Agent to read from each candidate's metrics, in --candidates order.", - metavar="NAME[,NAME...]", - ), + _bool_flag("no-stats", "no_stats", "Skip the default statistical-test step."), + ), + ), + "eval stat-test": Command( + target="nemo_gym.cli.eval:stat_test", + summary="Statistical significance test between a baseline and a candidate run (default: paired).", + flags=( + *_EVAL_RUN_SELECTION_FLAGS, _value_flag( "output-dir", "output_dirpath", - "Where to write the report (default: the candidate run's own directory).", + "Where to write the report (default: `/statistical_tests/`).", aliases=("-o",), quote=True, ), @@ -1119,6 +1173,11 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "Report artifacts to write (default: both).", choices=("md", "json", "both"), ), + # Choices are spelled out rather than derived from `statistical_tests.registry`: + # importing it here would put pydantic + the whole stats package on the path of every + # `gym` invocation. `test_cli_test_flag_choices_match_the_registry` pins them together. + _value_flag("test", "test", "Statistical test to run (default: paired).", choices=("paired",)), + *_STATISTICAL_TEST_FLAGS, ), ), "dev test": Command(target="nemo_gym.cli.dev:dev_test", summary="Run NeMo Gym's unit tests."), diff --git a/nemo_gym/comparison/schema.py b/nemo_gym/comparison/schema.py index 2eb424c36b..be0aedae77 100644 --- a/nemo_gym/comparison/schema.py +++ b/nemo_gym/comparison/schema.py @@ -20,11 +20,11 @@ """ from pathlib import Path -from typing import Any, Dict, List, Literal, Optional +from typing import Any, ClassVar, Dict, List, Literal, Optional -from pydantic import BaseModel, ConfigDict, Field, computed_field, model_validator +from pydantic import BaseModel, ConfigDict, Field, computed_field -from nemo_gym.config_types import BaseNeMoGymCLIConfig +from nemo_gym.config_types import RunSelectionConfig ReportFormat = Literal["md", "json", "both"] @@ -38,7 +38,7 @@ MAX_FLIPS_SHOWN = 10 -class ComparisonConfig(BaseNeMoGymCLIConfig): +class ComparisonConfig(RunSelectionConfig): """Compare a baseline eval run against a candidate run. Reads only each run's `_aggregate_metrics.json`, derived from the rollouts JSONL path you @@ -57,39 +57,7 @@ class ComparisonConfig(BaseNeMoGymCLIConfig): set `baseline_aggregate_metrics_fpath` and `candidate_aggregate_metrics_fpaths`. """ - baseline_rollouts_jsonl_fpath: str = Field( - description="Baseline run's rollouts JSONL, as passed to `gym eval run --output`. Used to derive " - "`_aggregate_metrics.json`; the JSONL itself is not read." - ) - candidate_rollouts_jsonl_fpaths: List[str] = Field( - min_length=1, - description="Candidate runs' rollouts JSONL paths (comma-separated via --candidates).", - ) - baseline_aggregate_metrics_fpath: Optional[str] = Field( - default=None, - description="Override for the baseline's aggregate-metrics JSON. Defaults to the " - "`_aggregate_metrics.json` sibling of baseline_rollouts_jsonl_fpath.", - ) - candidate_aggregate_metrics_fpaths: Optional[List[str]] = Field( - default=None, - description="Overrides for the candidates' aggregate-metrics JSON files. When set, must have the " - "same length as candidate_rollouts_jsonl_fpaths.", - ) - - agent_name: Optional[str] = Field( - default=None, - description="Agent to compare on every side. When unset, compares every agent name present in all " - "runs. Overridden per side by baseline_agent_name / candidate_agent_names.", - ) - baseline_agent_name: Optional[str] = Field( - default=None, - description="Agent to read from the baseline's metrics. Takes precedence over agent_name.", - ) - candidate_agent_names: Optional[List[str]] = Field( - default=None, - description="Agent to read from each candidate's metrics, by position. When set, must have the same " - "length as candidate_rollouts_jsonl_fpaths. Takes precedence over agent_name.", - ) + MAX_CANDIDATES: ClassVar[int] = MAX_CANDIDATES output_dirpath: Optional[str] = Field( default=None, @@ -101,25 +69,6 @@ class ComparisonConfig(BaseNeMoGymCLIConfig): description="Which report artifacts to write: `md`, `json`, or `both`.", ) - @model_validator(mode="after") - def _check_candidate_parallel_lists(self) -> "ComparisonConfig": - num_candidates = len(self.candidate_rollouts_jsonl_fpaths) - if num_candidates > MAX_CANDIDATES: - raise ValueError( - f"{num_candidates} candidates were given, but comparing more than {MAX_CANDIDATES} candidate " - "is not supported yet. Give a single candidate run." - ) - for field_name, value in ( - ("candidate_agent_names", self.candidate_agent_names), - ("candidate_aggregate_metrics_fpaths", self.candidate_aggregate_metrics_fpaths), - ): - if value is not None and len(value) != num_candidates: - raise ValueError( - f"{field_name} has {len(value)} entries but {num_candidates} candidate run(s) were given. " - "Give one entry per candidate, in the same order." - ) - return self - class MetricValue(BaseModel): """One side's reading of a metric, plus whatever uncertainty the run recorded for it.""" diff --git a/nemo_gym/config_types.py b/nemo_gym/config_types.py index 5074f86973..66b4d8e7bd 100644 --- a/nemo_gym/config_types.py +++ b/nemo_gym/config_types.py @@ -104,6 +104,67 @@ def pre_process(cls, data): exit() +class RunSelectionConfig(BaseNeMoGymCLIConfig): + """Which run pair to read and which agent to compare -- shared by `gym eval compare` and + `gym eval stat-test`, which both read a baseline/candidate run's `_aggregate_metrics.json` + and pick an agent the same way. + """ + + # Subclasses may raise this once >1 candidate is supported. + MAX_CANDIDATES: ClassVar[int] = 1 + + baseline_rollouts_jsonl_fpath: str = Field( + description="Baseline run's rollouts JSONL, as passed to `gym eval run --output`. Used to derive " + "`_aggregate_metrics.json`; the JSONL itself is not read." + ) + candidate_rollouts_jsonl_fpaths: List[str] = Field( + min_length=1, + description="Candidate run's rollouts JSONL paths (comma-separated); one candidate is supported today.", + ) + baseline_aggregate_metrics_fpath: Optional[str] = Field( + default=None, + description="Override for the baseline's aggregate-metrics JSON. Defaults to the " + "`_aggregate_metrics.json` sibling of baseline_rollouts_jsonl_fpath.", + ) + candidate_aggregate_metrics_fpaths: Optional[List[str]] = Field( + default=None, + description="Override for the candidate's aggregate-metrics JSON. When set, must have the same " + "length as candidate_rollouts_jsonl_fpaths.", + ) + + agent_name: Optional[str] = Field( + default=None, + description="Agent to compare on both sides. When unset, compares the agent present in both runs.", + ) + baseline_agent_name: Optional[str] = Field( + default=None, + description="Agent to read from the baseline's metrics. Takes precedence over agent_name.", + ) + candidate_agent_names: Optional[List[str]] = Field( + default=None, + description="Agent to read from the candidate's metrics. Takes precedence over agent_name.", + ) + + @model_validator(mode="after") + def _check_candidate_parallel_lists(self) -> "RunSelectionConfig": + num_candidates = len(self.candidate_rollouts_jsonl_fpaths) + if num_candidates > self.MAX_CANDIDATES: + raise ValueError( + f"{num_candidates} candidates were given, but more than {self.MAX_CANDIDATES} candidate " + "is not supported yet. Give a single candidate run." + ) + for field_name, value in ( + ("candidate_agent_names", self.candidate_agent_names), + ("candidate_aggregate_metrics_fpaths", self.candidate_aggregate_metrics_fpaths), + ): + if value is not None and len(value) != num_candidates: + raise ValueError( + f"{field_name} has {len(value)} entries but {num_candidates} candidate run(s) were given. " + "Give one entry per candidate, in the same order." + ) + return self + + ######################################## # Server references # diff --git a/nemo_gym/statistical_tests/__init__.py b/nemo_gym/statistical_tests/__init__.py new file mode 100644 index 0000000000..1fd3d363d7 --- /dev/null +++ b/nemo_gym/statistical_tests/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Statistical testing of eval runs (`gym eval stat-test`, and `gym eval compare`'s stats step).""" diff --git a/nemo_gym/statistical_tests/common.py b/nemo_gym/statistical_tests/common.py new file mode 100644 index 0000000000..50d6c7a6b9 --- /dev/null +++ b/nemo_gym/statistical_tests/common.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Everything a statistical test needs that isn't its own statistic: loading the run pair, and writing its report.""" + +import re +import shlex +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +import orjson + +from nemo_gym import _resolve_under_cwd_or_install +from nemo_gym.comparison.loading import LoadedRun, build_loaded_run, load_agg_metrics_file, resolve_agent_selections +from nemo_gym.comparison.schema import RunFile +from nemo_gym.config_types import ConfigError +from nemo_gym.package_info import __version__ +from nemo_gym.secret_utils import hide_secrets_in_overrides +from nemo_gym.statistical_tests.schema import STATS_SUBDIR_NAME, StatTestConfig + + +MISSING = "—" + + +def fmt(value: Optional[float]) -> str: + return MISSING if value is None else (f"{value:.4f}" if abs(value) < 10 else f"{value:.2f}") + + +def fmt_p(value: Optional[float]) -> str: + return MISSING if value is None else (f"{value:.4f}" if value >= 0.0001 else f"{value:.2e}") + + +def fmt_bool(value: Optional[bool]) -> str: + return MISSING if value is None else ("yes" if value else "no") + + +@dataclass(frozen=True) +class RunPair: + baseline_file: RunFile + candidate_file: RunFile + baseline: LoadedRun + candidate: LoadedRun + baseline_agent: str + candidate_agent: str + warnings: List[str] + + def report_identity(self, config: StatTestConfig, command: str) -> Dict[str, Any]: + return { + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "nemo_gym_version": __version__, + "command": command, + "test": config.test, + "baseline_rollouts_jsonl_fpath": str(self.baseline_file.rollouts_jsonl_fpath), + "baseline_aggregate_metrics_fpath": str(self.baseline_file.aggregate_metrics_fpath), + "candidate_rollouts_jsonl_fpath": str(self.candidate_file.rollouts_jsonl_fpath), + "candidate_aggregate_metrics_fpath": str(self.candidate_file.aggregate_metrics_fpath), + "baseline_agent": self.baseline_agent, + "candidate_agent": self.candidate_agent, + "baseline_task_count": self.baseline.num_tasks, + "candidate_task_count": self.candidate.num_tasks, + "warnings": self.warnings, + } + + +def load_run_pair(config: StatTestConfig) -> RunPair: + baseline_file = load_agg_metrics_file( + config.baseline_rollouts_jsonl_fpath, + role="baseline", + aggregate_metrics_fpath_override=config.baseline_aggregate_metrics_fpath, + ) + (candidate_fpath,) = config.candidate_rollouts_jsonl_fpaths + candidate_agg = config.candidate_aggregate_metrics_fpaths[0] if config.candidate_aggregate_metrics_fpaths else None + candidate_file = load_agg_metrics_file( + candidate_fpath, role="candidate", index=0, aggregate_metrics_fpath_override=candidate_agg + ) + + selections, warnings, _skipped = resolve_agent_selections( + baseline_file, + [candidate_file], + agent_name=config.agent_name, + baseline_agent_name=config.baseline_agent_name, + candidate_agent_names=config.candidate_agent_names, + ) + if len(selections) != 1: + raise ConfigError( + "gym eval stat-test compares exactly one agent pair; narrow the selection with --agent, " + "--baseline-agent, or --candidate-agents." + ) + s = selections[0] + return RunPair( + baseline_file, + candidate_file, + build_loaded_run(baseline_file, s.baseline_agent), + build_loaded_run(candidate_file, s.candidate_agents[0]), + s.baseline_agent, + s.candidate_agents[0], + warnings, + ) + + +def invoked_command(subcommand: str = "stat-test") -> str: + return shlex.join(["gym", "eval", subcommand, *hide_secrets_in_overrides(sys.argv[1:])]) + + +def sanitize_filename_part(text: str) -> str: + return re.sub(r"[^A-Za-z0-9_.+-]+", "-", text).strip("-") + + +def report_stem(config: StatTestConfig) -> str: + return "__".join([sanitize_filename_part(config.test), *config.filename_parts(), f"alpha-{config.alpha:g}"]) + + +def resolve_output_dir(config: StatTestConfig) -> Path: + if config.output_dirpath: + p = Path(config.output_dirpath) + return p if p.is_absolute() else Path.cwd() / p + return _resolve_under_cwd_or_install(config.candidate_rollouts_jsonl_fpaths[-1]).parent / STATS_SUBDIR_NAME + + +def write_reports(output_dir: Path, stem: str, *, report_format: str, markdown: str, payload: dict) -> List[Path]: + if output_dir.exists() and not output_dir.is_dir(): + raise ConfigError(f"--output-dir '{output_dir}' exists and is not a directory.") + try: + output_dir.mkdir(parents=True, exist_ok=True) + written = [] + if report_format in ("md", "both"): + (output_dir / f"{stem}.md").write_text(markdown, encoding="utf-8") + written.append(output_dir / f"{stem}.md") + if report_format in ("json", "both"): + (output_dir / f"{stem}.json").write_bytes(orjson.dumps(payload, option=orjson.OPT_INDENT_2)) + written.append(output_dir / f"{stem}.json") + return written + except OSError as e: + raise ConfigError(f"Cannot write the report into '{output_dir}': {e}") from e diff --git a/nemo_gym/statistical_tests/registry.py b/nemo_gym/statistical_tests/registry.py new file mode 100644 index 0000000000..30e222a27b --- /dev/null +++ b/nemo_gym/statistical_tests/registry.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Which statistical test `gym eval stat-test --test ` runs.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, List, Sequence, Tuple, Type + +from nemo_gym.cli.utils import did_you_mean +from nemo_gym.config_types import ConfigError +from nemo_gym.statistical_tests import paired +from nemo_gym.statistical_tests.common import report_stem, resolve_output_dir, write_reports +from nemo_gym.statistical_tests.schema import StatTestConfig, StatTestReport + + +@dataclass(frozen=True) +class StatTest: + config_type: Type[StatTestConfig] + build_report: Callable[[StatTestConfig, str], StatTestReport] + render_markdown: Callable[[StatTestReport], str] + summary: Callable[[StatTestReport, Sequence[Path]], Sequence[str]] + + +STAT_TESTS: Dict[str, StatTest] = { + "paired": StatTest( + config_type=paired.PairedTestConfig, + build_report=paired.build_report, + render_markdown=paired.render_markdown, + summary=paired.summary, + ), +} + + +def resolve_stat_test(name: str) -> StatTest: + if name not in STAT_TESTS: + raise ConfigError( + f"Unknown statistical test '{name}'. Available: {', '.join(sorted(STAT_TESTS))}." + + did_you_mean(name, STAT_TESTS) + ) + return STAT_TESTS[name] + + +def run_stat_test(test: StatTest, config: StatTestConfig, command: str) -> Tuple[StatTestReport, List[Path]]: + report = test.build_report(config, command) + return report, write_reports( + resolve_output_dir(config), + report_stem(config), + report_format=config.report_format, + markdown=test.render_markdown(report), + payload=report.model_dump(mode="json"), + ) diff --git a/nemo_gym/statistical_tests/schema.py b/nemo_gym/statistical_tests/schema.py new file mode 100644 index 0000000000..8fadd69ca2 --- /dev/null +++ b/nemo_gym/statistical_tests/schema.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The config and report envelope every statistical test builds on.""" + +from typing import ClassVar, List, Literal, Optional + +from pydantic import BaseModel, Field, model_validator + +from nemo_gym.config_types import RunSelectionConfig + + +ReportFormat = Literal["md", "json", "both"] +MAX_CANDIDATES = 1 +STATS_SUBDIR_NAME = "statistical_tests" +DEFAULT_STAT_TEST = "paired" + + +class StatTestConfig(RunSelectionConfig): + MAX_CANDIDATES: ClassVar[int] = MAX_CANDIDATES + + test: str = Field(default=DEFAULT_STAT_TEST, description="Which statistical test to run.") + output_dirpath: Optional[str] = Field( + default=None, description=f"Report directory. Defaults to `/{STATS_SUBDIR_NAME}/`." + ) + report_format: ReportFormat = Field(default="both", description="Artifacts to write: `md`, `json`, or `both`.") + alpha: float = Field(default=0.05, description="Significance level.") + + @model_validator(mode="after") + def _check_alpha(self) -> "StatTestConfig": + if not (0 < self.alpha < 1): + raise ValueError(f"--alpha must be between 0 and 1, exclusive (got {self.alpha}).") + return self + + def filename_parts(self) -> List[str]: + return [] + + +class StatTestReport(BaseModel): + schema_version: Literal["1"] = "1" + generated_at: str + nemo_gym_version: str + command: str + test: str + baseline_rollouts_jsonl_fpath: str + baseline_aggregate_metrics_fpath: str + candidate_rollouts_jsonl_fpath: str + candidate_aggregate_metrics_fpath: str + baseline_agent: str + candidate_agent: str + baseline_task_count: int + candidate_task_count: int + notes: List[str] = Field(default_factory=list) + warnings: List[str] = Field(default_factory=list) diff --git a/tests/unit_tests/statistical_tests/__init__.py b/tests/unit_tests/statistical_tests/__init__.py new file mode 100644 index 0000000000..467079831e --- /dev/null +++ b/tests/unit_tests/statistical_tests/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit_tests/statistical_tests/test_cli_flags.py b/tests/unit_tests/statistical_tests/test_cli_flags.py new file mode 100644 index 0000000000..4bbb5b9e44 --- /dev/null +++ b/tests/unit_tests/statistical_tests/test_cli_flags.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import List + +from nemo_gym.statistical_tests.paired import PairedTestConfig + + +class TestCliFlagTranslation: + """`gym eval stat-test`'s flags must survive the argparse -> Hydra override -> pydantic round trip.""" + + def _overrides(self, argv: List[str]) -> List[str]: + from nemo_gym.cli.main import build_parser + + args, unknown = build_parser().parse_known_args(argv) + assert unknown == [], f"flags left unparsed: {unknown}" + return [token for flag in args._command.flags for token in flag.translate_to_hydra(args)] + + def _config(self, argv: List[str]) -> PairedTestConfig: + from hydra.core.override_parser.overrides_parser import OverridesParser + + parsed = OverridesParser.create().parse_overrides(self._overrides(argv)) + return PairedTestConfig.model_validate({o.key_or_group: o.value() for o in parsed}) + + def test_paths_round_trip_through_hydra(self): + config = self._config( + ["eval", "stat-test", "--baseline", "runs/a/rollouts.jsonl", "--candidates", "runs/b/rollouts.jsonl"] + ) + assert config.baseline_rollouts_jsonl_fpath == "runs/a/rollouts.jsonl" + assert config.candidate_rollouts_jsonl_fpaths == ["runs/b/rollouts.jsonl"] + assert config.metric is None + assert config.margin is None + assert config.alpha == 0.05 + + def test_metric_margin_alpha_translate(self): + config = self._config( + [ + "eval", + "stat-test", + "--baseline", + "a.jsonl", + "--candidates", + "b.jsonl", + "--metric", + "reward,output_tokens", + "--margin", + "0.01", + "--alpha", + "0.1", + ] + ) + assert config.metric == ["reward", "output_tokens"] + assert config.margin == 0.01 + assert config.alpha == 0.1 + + def test_unset_flags_contribute_no_overrides(self): + overrides = self._overrides(["eval", "stat-test", "--baseline", "a.jsonl", "--candidates", "b.jsonl"]) + assert not [token for token in overrides if "metric" in token or "margin" in token] + + def test_test_selector_round_trips_and_defaults_to_paired(self): + argv = ["eval", "stat-test", "--baseline", "a.jsonl", "--candidates", "b.jsonl"] + assert self._config([*argv, "--test", "paired"]).test == "paired" + # Unset, the flag contributes no override and the pydantic default supplies the same value. + assert not [token for token in self._overrides(argv) if token.startswith("+test=")] + assert self._config(argv).test == "paired" + + def test_eval_compare_gets_the_same_statistical_flags(self): + overrides = self._overrides( + [ + "eval", + "compare", + "--baseline", + "a.jsonl", + "--candidates", + "b.jsonl", + "--metric", + "reward", + "--margin", + "0.01", + "--stats-output-dir", + "elsewhere", + "--no-stats", + ] + ) + assert '+metric=["reward"]' in overrides + assert "+margin=0.01" in overrides + assert '+stats_output_dirpath="elsewhere"' in overrides + assert "+no_stats=true" in overrides diff --git a/tests/unit_tests/statistical_tests/test_common.py b/tests/unit_tests/statistical_tests/test_common.py new file mode 100644 index 0000000000..a7ea76b1ce --- /dev/null +++ b/tests/unit_tests/statistical_tests/test_common.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import pytest + +from nemo_gym.config_types import ConfigError +from nemo_gym.statistical_tests.common import ( + invoked_command, + load_run_pair, + report_stem, + resolve_output_dir, + sanitize_filename_part, + write_reports, +) +from nemo_gym.statistical_tests.schema import STATS_SUBDIR_NAME, StatTestConfig +from tests.unit_tests.test_compare import _entry, _group, _write_run + + +AGENT = "bird_sql_simple_agent" +BASE = { + "baseline_rollouts_jsonl_fpath": "runs/a/rollouts.jsonl", + "candidate_rollouts_jsonl_fpaths": ["runs/b/r.jsonl"], +} + + +def _config(baseline, candidate, **overrides) -> StatTestConfig: + return StatTestConfig.model_validate( + { + "baseline_rollouts_jsonl_fpath": str(baseline), + "candidate_rollouts_jsonl_fpaths": [str(candidate)], + **overrides, + } + ) + + +class TestLoadRunPair: + def test_loads_both_sides_and_narrows_to_the_sole_shared_agent(self, tmp_path): + baseline = _write_run(tmp_path, "run_a", [_entry(groups=[_group(0, [1.0]), _group(1, [1.0])])]) + candidate = _write_run(tmp_path, "run_b", [_entry(groups=[_group(0, [0.0]), _group(1, [1.0])])]) + + pair = load_run_pair(_config(baseline, candidate)) + + assert pair.baseline_agent == AGENT and pair.candidate_agent == AGENT + assert pair.baseline.num_tasks == 2 and pair.candidate.num_tasks == 2 + + def test_an_ambiguous_agent_selection_is_an_error_not_a_loop(self, tmp_path): + baseline = _write_run( + tmp_path, + "run_a", + [_entry(agent="a1", groups=[_group(0, [1.0])]), _entry(agent="a2", groups=[_group(0, [1.0])])], + ) + candidate = _write_run( + tmp_path, + "run_b", + [_entry(agent="a1", groups=[_group(0, [0.0])]), _entry(agent="a2", groups=[_group(0, [0.0])])], + ) + with pytest.raises(ConfigError, match="exactly one agent pair"): + load_run_pair(_config(baseline, candidate)) + + def test_an_explicit_agent_narrows_an_otherwise_ambiguous_pair(self, tmp_path): + baseline = _write_run( + tmp_path, + "run_a", + [_entry(agent="a1", groups=[_group(0, [1.0])]), _entry(agent="a2", groups=[_group(0, [1.0])])], + ) + candidate = _write_run( + tmp_path, + "run_b", + [_entry(agent="a1", groups=[_group(0, [0.0])]), _entry(agent="a2", groups=[_group(0, [0.0])])], + ) + pair = load_run_pair(_config(baseline, candidate, agent_name="a2")) + assert pair.baseline_agent == "a2" and pair.candidate_agent == "a2" + + def test_report_identity_describes_both_runs_and_the_selected_test(self, tmp_path): + baseline = _write_run(tmp_path, "run_a", [_entry(groups=[_group(0, [1.0])])]) + candidate = _write_run(tmp_path, "run_b", [_entry(groups=[_group(0, [0.0])])]) + config = _config(baseline, candidate) + + identity = load_run_pair(config).report_identity(config, "gym eval stat-test ...") + + assert identity["test"] == "paired" + assert identity["command"] == "gym eval stat-test ..." + assert identity["baseline_task_count"] == 1 and identity["candidate_task_count"] == 1 + assert identity["generated_at"] and identity["nemo_gym_version"] + + +class TestReportStem: + def test_leads_with_the_test_name_so_two_tests_cannot_overwrite_each_other(self): + config = StatTestConfig.model_validate(BASE) + assert report_stem(config) == "paired__alpha-0.05" + + @pytest.mark.parametrize( + ("raw", "expected"), + [("reward", "reward"), ("a/b", "a-b"), ("pass@1[avg-of-2]", "pass-1-avg-of-2"), ("--x--", "x")], + ) + def test_sanitize_makes_a_filename_safe_token(self, raw, expected): + assert sanitize_filename_part(raw) == expected + + +class TestResolveOutputDir: + def test_unset_nests_under_the_candidate_runs_own_directory(self, tmp_path): + config = StatTestConfig.model_validate( + {**BASE, "candidate_rollouts_jsonl_fpaths": [str(tmp_path / "run_b" / "rollouts.jsonl")]} + ) + assert resolve_output_dir(config) == tmp_path / "run_b" / STATS_SUBDIR_NAME + + def test_explicit_path_is_used_literally_with_no_nesting(self, tmp_path): + config = StatTestConfig.model_validate({**BASE, "output_dirpath": str(tmp_path / "elsewhere")}) + assert resolve_output_dir(config) == tmp_path / "elsewhere" + + +class TestWriteReports: + def _write(self, output_dir, report_format="both"): + return write_reports( + output_dir, "stem", report_format=report_format, markdown="plain text", payload={"schema_version": "1"} + ) + + @pytest.mark.parametrize( + ("report_format", "expected"), + [("both", ["stem.md", "stem.json"]), ("md", ["stem.md"]), ("json", ["stem.json"])], + ) + def test_report_format_selects_the_artifacts(self, tmp_path, report_format, expected): + written = self._write(tmp_path, report_format) + assert [path.name for path in written] == expected + assert all(path.exists() for path in written) + + def test_an_output_dir_that_is_a_file_is_rejected_cleanly(self, tmp_path): + not_a_dir = tmp_path / "file" + not_a_dir.write_text("") + with pytest.raises(ConfigError, match="exists and is not a directory"): + self._write(not_a_dir) + + +class TestInvokedCommand: + def test_records_the_subcommand_that_actually_ran(self, monkeypatch): + monkeypatch.setattr("sys.argv", ["gym", "+no_stats=false"]) + assert invoked_command() == "gym eval stat-test +no_stats=false" + assert invoked_command("compare") == "gym eval compare +no_stats=false" + + def test_quotes_awkward_overrides_and_redacts_secrets(self, monkeypatch): + monkeypatch.setattr("sys.argv", ["gym", "+baseline_rollouts_jsonl_fpath=a b.jsonl"]) + assert invoked_command() == "gym eval stat-test '+baseline_rollouts_jsonl_fpath=a b.jsonl'" diff --git a/tests/unit_tests/statistical_tests/test_paired.py b/tests/unit_tests/statistical_tests/test_paired.py new file mode 100644 index 0000000000..ab09bce621 --- /dev/null +++ b/tests/unit_tests/statistical_tests/test_paired.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import pytest +from pydantic import ValidationError +from scipy import stats + +from nemo_gym.comparison.loading import LoadedRun +from nemo_gym.config_types import ConfigError +from nemo_gym.statistical_tests.paired import ( + PairedTestConfig, + build_report, + paired_task_deltas, + render_markdown, + resolve_metrics, + run_metric, + summary, +) +from tests.unit_tests.test_compare import _entry, _group, _write_run + + +BASE = {"baseline_rollouts_jsonl_fpath": "a.jsonl", "candidate_rollouts_jsonl_fpaths": ["b.jsonl"]} + + +def _run(groups, key_metrics=None) -> LoadedRun: + return LoadedRun( + agent_name="agent", + agent_metrics={}, + key_metrics=key_metrics or {}, + group_level_metrics=groups, + num_tasks=len(groups), + ) + + +def _g(task_index, **fields): + """A bare `group_level_metrics` entry (distinct from test_compare's `_group`, which computes mean/min/max).""" + return {"_ng_task_index": task_index, **fields} + + +def two_runs(tmp_path): + baseline = _write_run( + tmp_path, + "run_a", + [ + _entry( + key_metrics={"mean/reward": 0.75, "pass@1/accuracy": 75.0}, + groups=[_group(i, [1.0, 1.0]) for i in range(6)], + ) + ], + ) + candidate = _write_run( + tmp_path, + "run_b", + [ + _entry( + key_metrics={"mean/reward": 0.5, "pass@1/accuracy": 50.0}, + groups=[_group(i, [0.0, 1.0]) for i in range(6)], + ) + ], + ) + return baseline, candidate + + +def config_for(baseline, candidate, **overrides) -> PairedTestConfig: + return PairedTestConfig.model_validate( + { + "baseline_rollouts_jsonl_fpath": str(baseline), + "candidate_rollouts_jsonl_fpaths": [str(candidate)], + **overrides, + } + ) + + +class TestPairedTestConfig: + def test_valid_minimal_config(self): + config = PairedTestConfig.model_validate(BASE) + assert config.test == "paired" and config.metric is None and config.margin is None + + @pytest.mark.parametrize("margin", [0, -0.01, -5]) + def test_non_positive_margin_is_rejected(self, margin): + with pytest.raises(ValidationError, match="--margin must be a positive number"): + PairedTestConfig.model_validate({**BASE, "margin": margin}) + + def test_filename_parts_reflect_the_framing_and_the_metric_subset(self): + assert PairedTestConfig.model_validate(BASE).filename_parts() == ["two-sided"] + assert PairedTestConfig.model_validate({**BASE, "margin": 0.01}).filename_parts() == ["margin-0.01"] + assert PairedTestConfig.model_validate({**BASE, "metric": ["reward", "a/b"]}).filename_parts() == [ + "metric-reward+a-b", + "two-sided", + ] + + +class TestPairedTaskDeltas: + def test_ties_are_included_not_filtered(self): + baseline = _run([_g(0, **{"mean/reward": 0.5}), _g(1, **{"mean/reward": 0.3})]) + candidate = _run([_g(0, **{"mean/reward": 0.5}), _g(1, **{"mean/reward": 0.1})]) + assert paired_task_deltas(baseline, candidate, "reward") == pytest.approx([0.0, -0.2]) + + def test_missing_metric_on_one_side_drops_that_task_only(self): + baseline = _run([_g(0, **{"mean/reward": 0.5}), _g(1, **{"mean/reward": 0.5})]) + candidate = _run([_g(0, **{"mean/reward": 0.4}), _g(1)]) + assert paired_task_deltas(baseline, candidate, "reward") == pytest.approx([-0.1]) + + def test_no_data_returns_none_not_empty_list(self): + assert paired_task_deltas(_run([_g(0, **{"mean/reward": 0.5})]), _run([_g(0)]), "reward") is None + + +class TestResolveMetrics: + def test_explicit_request_is_returned_verbatim_deduped(self): + resolved, skipped = resolve_metrics(_run([]), _run([]), ["reward", "reward", "output_tokens"]) + assert resolved == ["reward", "output_tokens"] and skipped == [] + + def test_default_skips_non_mean_and_no_data_metrics(self): + baseline = _run([_g(0, **{"mean/reward": 1.0})], key_metrics={"mean/reward": 1.0, "pass@1/acc": 1.0}) + candidate = _run([_g(0, **{"mean/reward": 0.5})], key_metrics={"mean/reward": 0.5, "pass@1/acc": 0.5}) + resolved, skipped = resolve_metrics(baseline, candidate, None) + assert resolved == ["reward"] and skipped == ["pass@1/acc"] + + +class TestRunMetric: + def test_no_common_task_returns_a_note_rather_than_raising(self): + baseline, candidate = _run([_g(0, **{"mean/reward": 1.0})]), _run([_g(1, **{"mean/reward": 0.0})]) + result = run_metric(baseline, candidate, metric="reward", margin=None, alpha=0.05) + assert result.n_pairs == 0 and result.p_value is None and "no per-task" in result.note + + def test_single_paired_task_cannot_estimate_se(self): + baseline, candidate = _run([_g(0, **{"mean/reward": 1.0})]), _run([_g(0, **{"mean/reward": 0.7})]) + result = run_metric(baseline, candidate, metric="reward", margin=None, alpha=0.05) + assert result.n_pairs == 1 and result.se is None and "cannot estimate" in result.note + + def test_zero_variance_nonzero_mean_is_significant(self): + baseline = _run([_g(i, **{"mean/reward": 0.5}) for i in range(3)]) + candidate = _run([_g(i, **{"mean/reward": 0.0}) for i in range(3)]) + result = run_metric(baseline, candidate, metric="reward", margin=None, alpha=0.05) + assert result.se == 0.0 and result.p_value == 0.0 and result.significant is True + + def test_p_value_matches_scipy_ttest_1samp_directly(self): + deltas = [0.2, -0.1, 0.3, 0.05, -0.05, 0.15] + baseline = _run([_g(i, **{"mean/reward": 0.0}) for i in range(len(deltas))]) + candidate = _run([_g(i, **{"mean/reward": d}) for i, d in enumerate(deltas)]) + result = run_metric(baseline, candidate, metric="reward", margin=None, alpha=0.05) + _, expected_p = stats.ttest_1samp(deltas, popmean=0.0) + assert result.p_value == pytest.approx(expected_p) + + def test_regression_within_margin_is_not_meaningfully_worse(self): + deltas = [-0.05, -0.06, -0.04, -0.05, -0.05, -0.06] + baseline = _run([_g(i, **{"mean/reward": 0.0}) for i in range(len(deltas))]) + candidate = _run([_g(i, **{"mean/reward": d}) for i, d in enumerate(deltas)]) + result = run_metric(baseline, candidate, metric="reward", margin=0.2, alpha=0.05) + assert result.significant is True and result.p_value < 0.05 + + +class TestBuildReport: + def test_default_tests_every_key_metric_with_pairing_data_and_notes_the_rest(self, tmp_path): + report = build_report(config_for(*two_runs(tmp_path)), "gym eval stat-test ...") + assert [result.metric for result in report.results] == ["reward"] + assert report.notes == ["Skipped 1 key metric(s) with no per-task pairing data: pass@1/accuracy."] + + def test_an_explicitly_named_metric_with_no_pairing_data_raises(self, tmp_path): + config = config_for(*two_runs(tmp_path), metric=["does_not_exist"]) + with pytest.raises(ConfigError, match="does_not_exist"): + build_report(config, "gym eval stat-test ...") + + def test_no_key_metric_has_pairing_data_raises(self, tmp_path): + baseline = _write_run(tmp_path, "run_a", [_entry(key_metrics={}, groups=[_group(0, [1.0])])]) + candidate = _write_run(tmp_path, "run_b", [_entry(key_metrics={}, groups=[_group(0, [0.0])])]) + with pytest.raises(ConfigError, match="No key metric has per-task pairing data"): + build_report(config_for(baseline, candidate), "gym eval stat-test ...") + + +class TestReportRendering: + def test_markdown_is_a_plain_line_per_metric(self, tmp_path): + report = build_report(config_for(*two_runs(tmp_path), metric=["reward"]), "gym eval stat-test ...") + markdown = render_markdown(report) + assert "gym eval stat-test: paired" in markdown + assert "reward: n=6" in markdown + + def test_a_report_with_no_results_says_so(self, tmp_path): + report = build_report(config_for(*two_runs(tmp_path), metric=["reward"]), "gym eval stat-test ...") + report.results = [] + assert "No metrics were tested." in render_markdown(report) + + def test_summary_reports_each_metric_and_every_path_written(self, tmp_path): + report = build_report(config_for(*two_runs(tmp_path), metric=["reward"]), "gym eval stat-test ...") + text = "\n".join(summary(report, [tmp_path / "out.json"])) + assert "reward: n=6" in text and str(tmp_path / "out.json") in text diff --git a/tests/unit_tests/statistical_tests/test_registry.py b/tests/unit_tests/statistical_tests/test_registry.py new file mode 100644 index 0000000000..bd1b17bc5b --- /dev/null +++ b/tests/unit_tests/statistical_tests/test_registry.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import argparse + +import pytest + +from nemo_gym.config_types import ConfigError +from nemo_gym.statistical_tests import paired +from nemo_gym.statistical_tests.paired import PairedTestConfig +from nemo_gym.statistical_tests.registry import STAT_TESTS, StatTest, resolve_stat_test +from nemo_gym.statistical_tests.schema import DEFAULT_STAT_TEST, StatTestConfig, StatTestReport + + +BASE = {"baseline_rollouts_jsonl_fpath": "a.jsonl", "candidate_rollouts_jsonl_fpaths": ["b.jsonl"]} + + +class TestStatTestRegistry: + @pytest.mark.parametrize("name", sorted(STAT_TESTS)) + def test_every_registered_test_is_fully_wired(self, name): + test = STAT_TESTS[name] + assert issubclass(test.config_type, StatTestConfig) + assert all(callable(fn) for fn in (test.build_report, test.render_markdown, test.summary)) + assert test.config_type.model_fields["test"].default == name + + def test_paired_is_the_default_and_resolves_to_the_paired_implementation(self): + assert DEFAULT_STAT_TEST == "paired" + assert StatTestConfig.model_fields["test"].default == DEFAULT_STAT_TEST + + test = resolve_stat_test(DEFAULT_STAT_TEST) + assert test.config_type is PairedTestConfig + assert test.build_report is paired.build_report + assert test.render_markdown is paired.render_markdown + assert test.summary is paired.summary + + def test_unknown_test_name_lists_what_exists_and_suggests_the_close_one(self): + with pytest.raises(ConfigError) as excinfo: + resolve_stat_test("paried") + message = str(excinfo.value) + assert "Unknown statistical test 'paried'" in message + assert "Did you mean `paired`?" in message + + def test_stat_test_runs_the_test_the_name_selected(self, monkeypatch, capsys, tmp_path): + """A stub entry must be dispatched to instead of the paired implementation.""" + from nemo_gym.cli.eval import stat_test + from nemo_gym.statistical_tests import registry + + stub_report = StatTestReport( + generated_at="2026-01-01T00:00:00+00:00", + nemo_gym_version="0.0.0", + command="gym eval stat-test ...", + test="paired", + baseline_rollouts_jsonl_fpath="a.jsonl", + baseline_aggregate_metrics_fpath="a_aggregate_metrics.json", + candidate_rollouts_jsonl_fpath="b.jsonl", + candidate_aggregate_metrics_fpath="b_aggregate_metrics.json", + baseline_agent="agent", + candidate_agent="agent", + baseline_task_count=1, + candidate_task_count=1, + ) + calls = [] + monkeypatch.setitem( + registry.STAT_TESTS, + "paired", + StatTest( + config_type=PairedTestConfig, + build_report=lambda config, command: (calls.append(command), stub_report)[1], + render_markdown=lambda report: "stub markdown", + summary=lambda report, written: ("stub ran",), + ), + ) + + stat_test(PairedTestConfig.model_validate({**BASE, "output_dirpath": str(tmp_path)})) + + assert calls, "the registered build_report was never called -- dispatch is still hardcoded" + assert calls[0].startswith("gym eval compare") + assert "stub ran" in capsys.readouterr().out + assert (tmp_path / "paired__two-sided__alpha-0.05.md").read_text() == "stub markdown" + + def test_cli_test_flag_choices_match_the_registry(self): + from nemo_gym.cli.main import COMMANDS + + parser = argparse.ArgumentParser() + for flag in COMMANDS["eval stat-test"].flags: + flag.register(parser) + action = next(a for a in parser._actions if "--test" in a.option_strings) + assert set(action.choices) == set(STAT_TESTS) diff --git a/tests/unit_tests/statistical_tests/test_schema.py b/tests/unit_tests/statistical_tests/test_schema.py new file mode 100644 index 0000000000..a17a3bd8f5 --- /dev/null +++ b/tests/unit_tests/statistical_tests/test_schema.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import pytest +from pydantic import ValidationError + +from nemo_gym.statistical_tests.schema import DEFAULT_STAT_TEST, StatTestConfig + + +BASE = {"baseline_rollouts_jsonl_fpath": "a.jsonl", "candidate_rollouts_jsonl_fpaths": ["b.jsonl"]} + + +class TestStatTestConfigValidation: + """The rules every test inherits, asserted on the base directly rather than through a subclass.""" + + def test_valid_minimal_config(self): + config = StatTestConfig.model_validate(BASE) + assert config.test == DEFAULT_STAT_TEST + assert config.alpha == 0.05 + assert config.report_format == "both" + assert config.output_dirpath is None + + def test_more_than_one_candidate_is_rejected(self): + with pytest.raises(ValidationError, match="more than 1 candidate"): + StatTestConfig.model_validate({**BASE, "candidate_rollouts_jsonl_fpaths": ["b.jsonl", "c.jsonl"]}) + + def test_mismatched_candidate_agent_names_length_is_rejected(self): + with pytest.raises(ValidationError, match="candidate_agent_names has 2 entries"): + StatTestConfig.model_validate({**BASE, "candidate_agent_names": ["a", "b"]}) + + def test_mismatched_candidate_agg_metrics_length_is_rejected(self): + with pytest.raises(ValidationError, match="candidate_aggregate_metrics_fpaths has 2 entries"): + StatTestConfig.model_validate({**BASE, "candidate_aggregate_metrics_fpaths": ["x.json", "y.json"]}) + + @pytest.mark.parametrize("alpha", [0, 1, -0.1, 1.5]) + def test_alpha_out_of_range_is_rejected(self, alpha): + with pytest.raises(ValidationError, match="--alpha must be between 0 and 1"): + StatTestConfig.model_validate({**BASE, "alpha": alpha}) + + def test_base_filename_parts_is_empty_so_the_stem_is_just_the_test_and_alpha(self): + # A test that adds no filename_parts() still gets a distinct, non-colliding stem. + assert StatTestConfig.model_validate(BASE).filename_parts() == [] diff --git a/tests/unit_tests/test_compare.py b/tests/unit_tests/test_compare.py index d8685dc83c..980ccbec17 100644 --- a/tests/unit_tests/test_compare.py +++ b/tests/unit_tests/test_compare.py @@ -27,7 +27,7 @@ resolve_agent_selections, ) from nemo_gym.comparison.report import render_markdown, summary_lines, write_reports -from nemo_gym.comparison.runner import build_comparison_result, resolve_output_dir +from nemo_gym.comparison.runner import build_comparison_result, resolve_output_dir, run_comparison from nemo_gym.comparison.schema import ComparisonConfig from nemo_gym.config_types import ConfigError, ConfigPathNotFoundError from nemo_gym.path_utils import aggregate_metrics_path_for @@ -1027,3 +1027,112 @@ def test_warnings_section_lists_skipped_agents(self, tmp_path): markdown = render_markdown(result) assert "## Warnings" in markdown assert "extra" in markdown + + +class TestStatsWiring: + """`gym eval compare`'s default statistics step (`cli.eval._run_stats_step_for_compare`). + + This is a side effect layered on top of `compare`, not a change to it: `compare_report.*` -- + same schema, same bytes -- is asserted unaffected. The statistics themselves are + `nemo_gym.statistical_tests`'s own responsibility and are tested there; this only checks the + wiring (where the extra artifacts land, and that nothing about `compare`'s own output moved). + """ + + def _config(self, tmp_path: Path) -> ComparisonConfig: + baseline = _write_run( + tmp_path, + "run_a", + [ + _entry( + agent_metrics={"mean/reward": 0.75}, + key_metrics={"mean/reward": 0.75}, + groups=[_group(i, [1.0, 1.0] if i % 2 == 0 else [0.0, 1.0]) for i in range(6)], + ) + ], + ) + candidate = _write_run( + tmp_path, + "run_b", + [ + _entry( + agent_metrics={"mean/reward": 0.5}, + key_metrics={"mean/reward": 0.5}, + groups=[_group(i, [1.0, 0.0] if i % 2 == 0 else [0.0, 0.0]) for i in range(6)], + ) + ], + ) + return ComparisonConfig.model_validate( + { + "baseline_rollouts_jsonl_fpath": str(baseline), + "candidate_rollouts_jsonl_fpaths": [str(candidate)], + } + ) + + def test_compare_report_is_byte_identical_with_or_without_the_stats_step(self, tmp_path): + from nemo_gym.cli.eval import _run_stats_step_for_compare + + config = self._config(tmp_path) + result, written = run_comparison(config, "gym eval compare ...") + (compare_json,) = [p for p in written if p.name == "compare_report.json"] + before = compare_json.read_bytes() + + _run_stats_step_for_compare(config, {}) + + after = compare_json.read_bytes() + assert before == after + # And the schema itself never gained a statistics field. + payload = orjson.loads(before) + assert set(payload.keys()) == { + "schema_version", + "generated_at", + "nemo_gym_version", + "command", + "baseline", + "candidates", + "comparisons", + "skipped_agents", + "warnings", + } + assert "statistical_tests" not in payload["comparisons"][0] + + def test_stats_step_writes_its_own_subdirectory_next_to_compare_report(self, tmp_path): + from nemo_gym.cli.eval import _run_stats_step_for_compare + from nemo_gym.statistical_tests.schema import STATS_SUBDIR_NAME + + config = self._config(tmp_path) + run_comparison(config, "gym eval compare ...") + _run_stats_step_for_compare(config, {}) + + run_b_dir = tmp_path / "run_b" + assert {"compare_report.md", "compare_report.json", STATS_SUBDIR_NAME}.issubset( + {p.name for p in run_b_dir.iterdir()} + ) + stats_files = list((run_b_dir / STATS_SUBDIR_NAME).iterdir()) + assert stats_files, "expected at least one statistical_tests/ artifact" + + def test_stats_output_dir_flag_redirects_without_nesting(self, tmp_path): + from nemo_gym.cli.eval import _run_stats_step_for_compare + from nemo_gym.statistical_tests.schema import STATS_SUBDIR_NAME + + config = self._config(tmp_path) + elsewhere = tmp_path / "elsewhere" + run_comparison(config, "gym eval compare ...") + _run_stats_step_for_compare(config, {"stats_output_dirpath": str(elsewhere)}) + + assert elsewhere.is_dir() + assert not (elsewhere / STATS_SUBDIR_NAME).exists() + assert list(elsewhere.iterdir()) + + def test_metric_and_margin_overrides_flow_through(self, tmp_path): + from nemo_gym.cli.eval import _run_stats_step_for_compare + from nemo_gym.statistical_tests.schema import STATS_SUBDIR_NAME + + config = self._config(tmp_path) + run_comparison(config, "gym eval compare ...") + _run_stats_step_for_compare(config, {"metric": ["reward"], "margin": 0.5, "alpha": 0.2}) + + (stats_json,) = (tmp_path / "run_b" / STATS_SUBDIR_NAME).glob("*.json") + payload = orjson.loads(stats_json.read_bytes()) + assert payload["results"][0]["metric"] == "reward" + assert payload["results"][0]["margin"] == 0.5 + assert payload["results"][0]["alpha"] == 0.2