From b803c08b685ca874d900cbefab828a23d16c90d9 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:07:51 +0100 Subject: [PATCH 01/15] Honor explicit HealthBench debug sample counts --- gpt_oss/evals/__main__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/gpt_oss/evals/__main__.py b/gpt_oss/evals/__main__.py index 40d56c12..bcd9d1a8 100644 --- a/gpt_oss/evals/__main__.py +++ b/gpt_oss/evals/__main__.py @@ -14,6 +14,14 @@ from .responses_sampler import ResponsesSampler +def _resolve_num_examples( + explicit_examples: int | None, debug_mode: bool, debug_default: int +) -> int | None: + if explicit_examples is not None: + return explicit_examples + return debug_default if debug_mode else None + + def main(): parser = argparse.ArgumentParser( description="Evaluate the models.", @@ -95,9 +103,8 @@ def main(): ) def get_evals(eval_name, debug_mode): - num_examples = ( - args.examples if args.examples is not None else (5 if debug_mode else None) - ) + num_examples = _resolve_num_examples(args.examples, debug_mode, 5) + healthbench_num_examples = _resolve_num_examples(args.examples, debug_mode, 10) # Set num_examples = None to reproduce full evals match eval_name: case "basic": @@ -112,7 +119,7 @@ def get_evals(eval_name, debug_mode): case "healthbench": return HealthBenchEval( grader_model=grading_sampler, - num_examples=10 if debug_mode else num_examples, + num_examples=healthbench_num_examples, n_repeats=1, n_threads=args.n_threads or 1, subset_name=None, @@ -120,7 +127,7 @@ def get_evals(eval_name, debug_mode): case "healthbench_hard": return HealthBenchEval( grader_model=grading_sampler, - num_examples=10 if debug_mode else num_examples, + num_examples=healthbench_num_examples, n_repeats=1, n_threads=args.n_threads or 1, subset_name="hard", @@ -128,7 +135,7 @@ def get_evals(eval_name, debug_mode): case "healthbench_consensus": return HealthBenchEval( grader_model=grading_sampler, - num_examples=10 if debug_mode else num_examples, + num_examples=healthbench_num_examples, n_repeats=1, n_threads=args.n_threads or 1, subset_name="consensus", From 3f86f329434a95bc4bba59586347a31b08e8185d Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:07:58 +0100 Subject: [PATCH 02/15] Add eval sample count regression tests --- tests/gpt_oss/evals/test_cli_examples.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/gpt_oss/evals/test_cli_examples.py diff --git a/tests/gpt_oss/evals/test_cli_examples.py b/tests/gpt_oss/evals/test_cli_examples.py new file mode 100644 index 00000000..29060bd2 --- /dev/null +++ b/tests/gpt_oss/evals/test_cli_examples.py @@ -0,0 +1,13 @@ +from gpt_oss.evals.__main__ import _resolve_num_examples + + +def test_explicit_examples_override_debug_default() -> None: + assert _resolve_num_examples(2, debug_mode=True, debug_default=10) == 2 + + +def test_debug_default_is_used_without_explicit_examples() -> None: + assert _resolve_num_examples(None, debug_mode=True, debug_default=10) == 10 + + +def test_full_eval_keeps_unlimited_examples() -> None: + assert _resolve_num_examples(None, debug_mode=False, debug_default=10) is None From c2e1341b70a985ab32f9bb72275fccccdb2b09f5 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:22:14 +0100 Subject: [PATCH 03/15] Move eval sample-count helper to lightweight module --- gpt_oss/evals/cli_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 gpt_oss/evals/cli_utils.py diff --git a/gpt_oss/evals/cli_utils.py b/gpt_oss/evals/cli_utils.py new file mode 100644 index 00000000..c32d85e4 --- /dev/null +++ b/gpt_oss/evals/cli_utils.py @@ -0,0 +1,6 @@ +def resolve_num_examples( + explicit_examples: int | None, debug_mode: bool, debug_default: int +) -> int | None: + if explicit_examples is not None: + return explicit_examples + return debug_default if debug_mode else None From 97118aff1851f34553daf81d63e4c98876b786db Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:22:33 +0100 Subject: [PATCH 04/15] Use lightweight eval sample-count helper --- gpt_oss/evals/__main__.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/gpt_oss/evals/__main__.py b/gpt_oss/evals/__main__.py index bcd9d1a8..49fe2a40 100644 --- a/gpt_oss/evals/__main__.py +++ b/gpt_oss/evals/__main__.py @@ -3,25 +3,18 @@ from datetime import datetime from . import report -from .basic_eval import BasicEval -from .gpqa_eval import GPQAEval from .aime_eval import AIME25Eval -from .healthbench_eval import HealthBenchEval +from .basic_eval import BasicEval from .chat_completions_sampler import ( OPENAI_SYSTEM_MESSAGE_API, ChatCompletionsSampler, ) +from .cli_utils import resolve_num_examples +from .gpqa_eval import GPQAEval +from .healthbench_eval import HealthBenchEval from .responses_sampler import ResponsesSampler -def _resolve_num_examples( - explicit_examples: int | None, debug_mode: bool, debug_default: int -) -> int | None: - if explicit_examples is not None: - return explicit_examples - return debug_default if debug_mode else None - - def main(): parser = argparse.ArgumentParser( description="Evaluate the models.", @@ -103,8 +96,8 @@ def main(): ) def get_evals(eval_name, debug_mode): - num_examples = _resolve_num_examples(args.examples, debug_mode, 5) - healthbench_num_examples = _resolve_num_examples(args.examples, debug_mode, 10) + num_examples = resolve_num_examples(args.examples, debug_mode, 5) + healthbench_num_examples = resolve_num_examples(args.examples, debug_mode, 10) # Set num_examples = None to reproduce full evals match eval_name: case "basic": From b1946d69d2683de73c6d740101ea7740b90eb8cb Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:22:46 +0100 Subject: [PATCH 05/15] Keep CLI helper tests dependency-light --- tests/gpt_oss/evals/test_cli_examples.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gpt_oss/evals/test_cli_examples.py b/tests/gpt_oss/evals/test_cli_examples.py index 29060bd2..3b825476 100644 --- a/tests/gpt_oss/evals/test_cli_examples.py +++ b/tests/gpt_oss/evals/test_cli_examples.py @@ -1,13 +1,13 @@ -from gpt_oss.evals.__main__ import _resolve_num_examples +from gpt_oss.evals.cli_utils import resolve_num_examples def test_explicit_examples_override_debug_default() -> None: - assert _resolve_num_examples(2, debug_mode=True, debug_default=10) == 2 + assert resolve_num_examples(2, debug_mode=True, debug_default=10) == 2 def test_debug_default_is_used_without_explicit_examples() -> None: - assert _resolve_num_examples(None, debug_mode=True, debug_default=10) == 10 + assert resolve_num_examples(None, debug_mode=True, debug_default=10) == 10 def test_full_eval_keeps_unlimited_examples() -> None: - assert _resolve_num_examples(None, debug_mode=False, debug_default=10) is None + assert resolve_num_examples(None, debug_mode=False, debug_default=10) is None From 16883ebb4db1ddaa9edfc35f83b766bdfa75c853 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:26:28 +0100 Subject: [PATCH 06/15] Resolve eval repeat counts for explicit subsets --- gpt_oss/evals/cli_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gpt_oss/evals/cli_utils.py b/gpt_oss/evals/cli_utils.py index c32d85e4..640046a9 100644 --- a/gpt_oss/evals/cli_utils.py +++ b/gpt_oss/evals/cli_utils.py @@ -4,3 +4,7 @@ def resolve_num_examples( if explicit_examples is not None: return explicit_examples return debug_default if debug_mode else None + + +def resolve_n_repeats(explicit_examples: int | None, debug_mode: bool) -> int: + return 1 if explicit_examples is not None or debug_mode else 8 From a26c782e16e683a79510b005aa16e9300edcb686 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:26:52 +0100 Subject: [PATCH 07/15] Use single repeat for explicit eval subsets --- gpt_oss/evals/__main__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gpt_oss/evals/__main__.py b/gpt_oss/evals/__main__.py index 49fe2a40..5ce4c7df 100644 --- a/gpt_oss/evals/__main__.py +++ b/gpt_oss/evals/__main__.py @@ -9,7 +9,7 @@ OPENAI_SYSTEM_MESSAGE_API, ChatCompletionsSampler, ) -from .cli_utils import resolve_num_examples +from .cli_utils import resolve_n_repeats, resolve_num_examples from .gpqa_eval import GPQAEval from .healthbench_eval import HealthBenchEval from .responses_sampler import ResponsesSampler @@ -98,13 +98,14 @@ def main(): def get_evals(eval_name, debug_mode): num_examples = resolve_num_examples(args.examples, debug_mode, 5) healthbench_num_examples = resolve_num_examples(args.examples, debug_mode, 10) + n_repeats = resolve_n_repeats(args.examples, debug_mode) # Set num_examples = None to reproduce full evals match eval_name: case "basic": return BasicEval() case "gpqa": return GPQAEval( - n_repeats=1 if args.debug else 8, + n_repeats=n_repeats, num_examples=num_examples, debug=debug_mode, n_threads=args.n_threads or 1, @@ -135,7 +136,7 @@ def get_evals(eval_name, debug_mode): ) case "aime25": return AIME25Eval( - n_repeats=1 if args.debug else 8, + n_repeats=n_repeats, num_examples=num_examples, n_threads=args.n_threads or 1, ) From 7c02feaae821c0600f131629d5a3b60026cd5460 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:27:06 +0100 Subject: [PATCH 08/15] Cover eval repeat selection for explicit subsets --- tests/gpt_oss/evals/test_cli_examples.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/gpt_oss/evals/test_cli_examples.py b/tests/gpt_oss/evals/test_cli_examples.py index 3b825476..7dcb887d 100644 --- a/tests/gpt_oss/evals/test_cli_examples.py +++ b/tests/gpt_oss/evals/test_cli_examples.py @@ -1,4 +1,4 @@ -from gpt_oss.evals.cli_utils import resolve_num_examples +from gpt_oss.evals.cli_utils import resolve_n_repeats, resolve_num_examples def test_explicit_examples_override_debug_default() -> None: @@ -11,3 +11,15 @@ def test_debug_default_is_used_without_explicit_examples() -> None: def test_full_eval_keeps_unlimited_examples() -> None: assert resolve_num_examples(None, debug_mode=False, debug_default=10) is None + + +def test_explicit_subset_uses_single_repeat() -> None: + assert resolve_n_repeats(2, debug_mode=False) == 1 + + +def test_debug_run_uses_single_repeat() -> None: + assert resolve_n_repeats(None, debug_mode=True) == 1 + + +def test_full_run_keeps_eight_repeats() -> None: + assert resolve_n_repeats(None, debug_mode=False) == 8 From 00f2857161dc6bdea4f768c7aaaef324fc4a6aeb Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:31:20 +0100 Subject: [PATCH 09/15] Respect explicit GPQA subset selection --- gpt_oss/evals/cli_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gpt_oss/evals/cli_utils.py b/gpt_oss/evals/cli_utils.py index 640046a9..25eaeeb9 100644 --- a/gpt_oss/evals/cli_utils.py +++ b/gpt_oss/evals/cli_utils.py @@ -8,3 +8,7 @@ def resolve_num_examples( def resolve_n_repeats(explicit_examples: int | None, debug_mode: bool) -> int: return 1 if explicit_examples is not None or debug_mode else 8 + + +def resolve_gpqa_debug_mode(explicit_examples: int | None, debug_mode: bool) -> bool: + return debug_mode and explicit_examples is None From 5ce32a112e10c002f5c47eb1c86a4d463c3fb052 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:31:46 +0100 Subject: [PATCH 10/15] Let explicit GPQA sample counts override fixed debug example --- gpt_oss/evals/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpt_oss/evals/__main__.py b/gpt_oss/evals/__main__.py index 5ce4c7df..83ff06f0 100644 --- a/gpt_oss/evals/__main__.py +++ b/gpt_oss/evals/__main__.py @@ -9,7 +9,7 @@ OPENAI_SYSTEM_MESSAGE_API, ChatCompletionsSampler, ) -from .cli_utils import resolve_n_repeats, resolve_num_examples +from .cli_utils import resolve_gpqa_debug_mode, resolve_n_repeats, resolve_num_examples from .gpqa_eval import GPQAEval from .healthbench_eval import HealthBenchEval from .responses_sampler import ResponsesSampler @@ -107,7 +107,7 @@ def get_evals(eval_name, debug_mode): return GPQAEval( n_repeats=n_repeats, num_examples=num_examples, - debug=debug_mode, + debug=resolve_gpqa_debug_mode(args.examples, debug_mode), n_threads=args.n_threads or 1, ) case "healthbench": From c66f8ab73f95a4bc05aca4decf93774cfe83716f Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:31:59 +0100 Subject: [PATCH 11/15] Cover explicit GPQA debug subset selection --- tests/gpt_oss/evals/test_cli_examples.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/gpt_oss/evals/test_cli_examples.py b/tests/gpt_oss/evals/test_cli_examples.py index 7dcb887d..31cc0a6e 100644 --- a/tests/gpt_oss/evals/test_cli_examples.py +++ b/tests/gpt_oss/evals/test_cli_examples.py @@ -1,4 +1,8 @@ -from gpt_oss.evals.cli_utils import resolve_n_repeats, resolve_num_examples +from gpt_oss.evals.cli_utils import ( + resolve_gpqa_debug_mode, + resolve_n_repeats, + resolve_num_examples, +) def test_explicit_examples_override_debug_default() -> None: @@ -23,3 +27,9 @@ def test_debug_run_uses_single_repeat() -> None: def test_full_run_keeps_eight_repeats() -> None: assert resolve_n_repeats(None, debug_mode=False) == 8 + + +def test_gpqa_fixed_debug_example_is_default_only() -> None: + assert resolve_gpqa_debug_mode(None, debug_mode=True) is True + assert resolve_gpqa_debug_mode(2, debug_mode=True) is False + assert resolve_gpqa_debug_mode(None, debug_mode=False) is False From 93c23d154765a99bfd8f8c78d07328629c336438 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Mon, 17 Aug 2026 08:19:20 +0100 Subject: [PATCH 12/15] fix: preserve full-run repeats for zero examples Signed-off-by: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> --- gpt_oss/evals/cli_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gpt_oss/evals/cli_utils.py b/gpt_oss/evals/cli_utils.py index 25eaeeb9..769fa75c 100644 --- a/gpt_oss/evals/cli_utils.py +++ b/gpt_oss/evals/cli_utils.py @@ -7,7 +7,8 @@ def resolve_num_examples( def resolve_n_repeats(explicit_examples: int | None, debug_mode: bool) -> int: - return 1 if explicit_examples is not None or debug_mode else 8 + explicit_subset = explicit_examples is not None and explicit_examples != 0 + return 1 if explicit_subset or debug_mode else 8 def resolve_gpqa_debug_mode(explicit_examples: int | None, debug_mode: bool) -> bool: From e7fb5b0dd9b79ab7432eb335df993fab0546cff2 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Mon, 17 Aug 2026 08:19:30 +0100 Subject: [PATCH 13/15] test: cover zero-example repeat selection Signed-off-by: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> --- tests/gpt_oss/evals/test_cli_examples.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/gpt_oss/evals/test_cli_examples.py b/tests/gpt_oss/evals/test_cli_examples.py index 31cc0a6e..c28572cb 100644 --- a/tests/gpt_oss/evals/test_cli_examples.py +++ b/tests/gpt_oss/evals/test_cli_examples.py @@ -29,6 +29,10 @@ def test_full_run_keeps_eight_repeats() -> None: assert resolve_n_repeats(None, debug_mode=False) == 8 +def test_zero_examples_keeps_full_run_repeats() -> None: + assert resolve_n_repeats(0, debug_mode=False) == 8 + + def test_gpqa_fixed_debug_example_is_default_only() -> None: assert resolve_gpqa_debug_mode(None, debug_mode=True) is True assert resolve_gpqa_debug_mode(2, debug_mode=True) is False From 61ca8eeb0fe443e335c381602f5edb1096cdd404 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:15:20 +0100 Subject: [PATCH 14/15] fix: normalize zero-example CLI semantics Signed-off-by: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> --- gpt_oss/evals/cli_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gpt_oss/evals/cli_utils.py b/gpt_oss/evals/cli_utils.py index 769fa75c..911b8fa0 100644 --- a/gpt_oss/evals/cli_utils.py +++ b/gpt_oss/evals/cli_utils.py @@ -2,6 +2,8 @@ def resolve_num_examples( explicit_examples: int | None, debug_mode: bool, debug_default: int ) -> int | None: if explicit_examples is not None: + if explicit_examples == 0: + return debug_default if debug_mode else None return explicit_examples return debug_default if debug_mode else None From 5a2c5afbab34a060452464cdd5f9c972bab1cd0f Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:15:31 +0100 Subject: [PATCH 15/15] test: cover zero-example debug defaults Signed-off-by: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> --- tests/gpt_oss/evals/test_cli_examples.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/gpt_oss/evals/test_cli_examples.py b/tests/gpt_oss/evals/test_cli_examples.py index c28572cb..f141d2c8 100644 --- a/tests/gpt_oss/evals/test_cli_examples.py +++ b/tests/gpt_oss/evals/test_cli_examples.py @@ -17,6 +17,14 @@ def test_full_eval_keeps_unlimited_examples() -> None: assert resolve_num_examples(None, debug_mode=False, debug_default=10) is None +def test_zero_examples_keeps_full_run_outside_debug() -> None: + assert resolve_num_examples(0, debug_mode=False, debug_default=10) is None + + +def test_zero_examples_uses_debug_default_in_debug_mode() -> None: + assert resolve_num_examples(0, debug_mode=True, debug_default=10) == 10 + + def test_explicit_subset_uses_single_repeat() -> None: assert resolve_n_repeats(2, debug_mode=False) == 1