From 6d53aa45d9245cf2620551bdd42097028dd79cfe Mon Sep 17 00:00:00 2001 From: mshimkus-positron Date: Fri, 7 Aug 2026 09:12:24 -0700 Subject: [PATCH] Expose inspect-ai's sample_shuffle in the eval command --max-samples keeps the first N samples in dataset storage order, which skews subsampling on category-grouped datasets: mmlu_pro stores all business questions first, so an unshuffled --max-samples 1200 evaluates almost nothing but business. inspect-ai already applies sample_shuffle before the limit; this exposes it as --sample-shuffle for deterministic, order-independent subsets. Default None keeps current behavior. --- docs/source/inspect-ai.mdx | 8 ++++++++ src/lighteval/main_inspect.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/docs/source/inspect-ai.mdx b/docs/source/inspect-ai.mdx index ff413dd84..1956353a9 100644 --- a/docs/source/inspect-ai.mdx +++ b/docs/source/inspect-ai.mdx @@ -70,6 +70,14 @@ lighteval eval hf-inference-providers/openai/gpt-oss-20b gsm8k \ --max-samples 10 ``` +When subsampling with `--max-samples`, add `--sample-shuffle ` to evaluate a deterministic random subset instead of the first N samples in dataset storage order. This matters for category-grouped datasets: for example, `mmlu_pro` stores all business questions first, so an unshuffled `--max-samples 1200` would evaluate almost nothing else. + +```bash +lighteval eval hf-inference-providers/openai/gpt-oss-20b mmlu_pro \ + --max-samples 1200 \ + --sample-shuffle 42 +``` + 7. Use multiple epochs for more reliable results. ```bash diff --git a/src/lighteval/main_inspect.py b/src/lighteval/main_inspect.py index 4aa5cbf87..9ab69a96e 100644 --- a/src/lighteval/main_inspect.py +++ b/src/lighteval/main_inspect.py @@ -367,6 +367,13 @@ def eval( # noqa C901 rich_help_panel=HELP_PANEL_NAME_2, ), ] = None, + sample_shuffle: Annotated[ + int | None, + Option( + help="Seed for shuffling sample order before max-samples applies. Without it, max-samples keeps the first N samples in dataset storage order, which skews subsampling on category-grouped datasets (e.g. MMLU-Pro stores all business questions first).", + rich_help_panel=HELP_PANEL_NAME_2, + ), + ] = None, # Metric parameters epochs: Annotated[ int, @@ -464,6 +471,7 @@ def eval( # noqa C901 retry_on_error=retry_on_error, max_retries=max_retries, limit=max_samples, + sample_shuffle=sample_shuffle, max_tasks=max_tasks, log_dir=log_dir, log_dir_allow_dirty=log_dir_allow_dirty,