From 150bbd3e9e6455a0ad4b16549632024f54dbbd35 Mon Sep 17 00:00:00 2001 From: "Eugenio \"Jay\" Zuccarelli" <11176606+jayzuccarelli@users.noreply.github.com> Date: Mon, 20 Jul 2026 01:25:07 +0000 Subject: [PATCH] fix metric_class names in avg_at_n / avg_at_n_math / maj_at_n test cases The fixtures used avg_at_k / avg_at_k_math / maj_at_k, but the metrics are registered as avg_at_n / avg_at_n_math / maj_at_n. Unknown names fall into the "not available in METRIC_CLASSES" branch, which returns success: True, so 10 test cases have been reported as passing without ever calling the metric. The names were not the only thing off: metric_params used {"k": N} where the constructors take n, and expected_output keyed on avg@k:k=N instead of avg@n:n=N. Renaming metric_class alone just gets you "You did not set the value of n", so all three are fixed here, and the files are renamed to match their metric like the rest of test_cases/. An unresolvable metric_class now raises instead of silently skipping, unless it is listed in SKIPPED_METRICS. That is what let this go unnoticed. maj_at_n is added to SKIPPED_METRICS for now: once enabled it fails with IndexError in Doc.get_golds, because its first case has gold_index [1] against a single choice. That is the bug PR #1274 fixes, so it can be unskipped once that lands. --- .../{avg_at_k.json => avg_at_n.json} | 34 +++++++------- ...{avg_at_k_math.json => avg_at_n_math.json} | 34 +++++++------- .../{maj_at_k.json => maj_at_n.json} | 44 +++++++++---------- tests/unit/metrics/test_metrics_automated.py | 6 +++ 4 files changed, 62 insertions(+), 56 deletions(-) rename tests/unit/metrics/test_cases/{avg_at_k.json => avg_at_n.json} (60%) rename tests/unit/metrics/test_cases/{avg_at_k_math.json => avg_at_n_math.json} (54%) rename tests/unit/metrics/test_cases/{maj_at_k.json => maj_at_n.json} (62%) diff --git a/tests/unit/metrics/test_cases/avg_at_k.json b/tests/unit/metrics/test_cases/avg_at_n.json similarity index 60% rename from tests/unit/metrics/test_cases/avg_at_k.json rename to tests/unit/metrics/test_cases/avg_at_n.json index 907345b56..e908e6479 100644 --- a/tests/unit/metrics/test_cases/avg_at_k.json +++ b/tests/unit/metrics/test_cases/avg_at_n.json @@ -1,11 +1,11 @@ { - "name": "Avg At K Test Suite", - "description": "Test cases for avg_at_k metric", + "name": "Avg At N Test Suite", + "description": "Test cases for avg_at_n metric", "test_cases": [ { - "name": "Avg at K - Correct in Top K", - "metric_class": "avg_at_k", - "metric_params": {"k": 2}, + "name": "Avg at N - Correct in Top N", + "metric_class": "avg_at_n", + "metric_params": {"n": 2}, "doc": { "query": "What is the capital of France?", "choices": ["London", "Paris", "Berlin"], @@ -16,15 +16,15 @@ "text": ["Paris", "London", "Berlin"] }, "expected_output": { - "avg@k:k=2": 0.5 + "avg@n:n=2": 0.5 }, "tolerance": 0.01, - "description": "Test avg at k with correct answer in top k" + "description": "Test avg at n with correct answer in top n" }, { - "name": "Avg at K - Not in Top K", - "metric_class": "avg_at_k", - "metric_params": {"k": 1}, + "name": "Avg at N - Not in Top N", + "metric_class": "avg_at_n", + "metric_params": {"n": 1}, "doc": { "query": "What is the capital of France?", "choices": ["London", "Paris", "Berlin"], @@ -35,15 +35,15 @@ "text": ["London", "Berlin", "Paris"] }, "expected_output": { - "avg@k:k=1": 0.0 + "avg@n:n=1": 0.0 }, "tolerance": 0.01, - "description": "Test avg at k with correct answer not in top k" + "description": "Test avg at n with correct answer not in top n" }, { - "name": "Avg at K - Multiple Correct", - "metric_class": "avg_at_k", - "metric_params": {"k": 3}, + "name": "Avg at N - Multiple Correct", + "metric_class": "avg_at_n", + "metric_params": {"n": 3}, "doc": { "query": "Which are European capitals?", "choices": ["London", "Paris", "Tokyo", "Berlin"], @@ -54,10 +54,10 @@ "text": ["Paris", "London", "Berlin", "Tokyo"] }, "expected_output": { - "avg@k:k=3": 0.33 + "avg@n:n=3": 0.33 }, "tolerance": 0.01, - "description": "Test avg at k with multiple correct answers" + "description": "Test avg at n with multiple correct answers" } ] } diff --git a/tests/unit/metrics/test_cases/avg_at_k_math.json b/tests/unit/metrics/test_cases/avg_at_n_math.json similarity index 54% rename from tests/unit/metrics/test_cases/avg_at_k_math.json rename to tests/unit/metrics/test_cases/avg_at_n_math.json index 60a712ea0..ae040c3d7 100644 --- a/tests/unit/metrics/test_cases/avg_at_k_math.json +++ b/tests/unit/metrics/test_cases/avg_at_n_math.json @@ -1,11 +1,11 @@ { - "name": "Avg At K Math Test Suite", - "description": "Test cases for avg_at_k_math metric", + "name": "Avg At N Math Test Suite", + "description": "Test cases for avg_at_n_math metric", "test_cases": [ { - "name": "Avg at K Math - Correct Math", - "metric_class": "avg_at_k_math", - "metric_params": {"k": 1}, + "name": "Avg at N Math - Correct Math", + "metric_class": "avg_at_n_math", + "metric_params": {"n": 1}, "doc": { "query": "What is 2 + 2?", "choices": ["4"], @@ -16,15 +16,15 @@ "text": ["4"] }, "expected_output": { - "avg@k:k=1": 1.0 + "avg@n:n=1": 1.0 }, "tolerance": 0.01, - "description": "Test avg at k math with correct math answer" + "description": "Test avg at n math with correct math answer" }, { - "name": "Avg at K Math - Wrong Math", - "metric_class": "avg_at_k_math", - "metric_params": {"k": 1}, + "name": "Avg at N Math - Wrong Math", + "metric_class": "avg_at_n_math", + "metric_params": {"n": 1}, "doc": { "query": "What is 2 + 2?", "choices": ["4"], @@ -35,15 +35,15 @@ "text": ["5"] }, "expected_output": { - "avg@k:k=1": 0.0 + "avg@n:n=1": 0.0 }, "tolerance": 0.01, - "description": "Test avg at k math with wrong math answer" + "description": "Test avg at n math with wrong math answer" }, { - "name": "Avg at K Math - Multiple Attempts", - "metric_class": "avg_at_k_math", - "metric_params": {"k": 2}, + "name": "Avg at N Math - Multiple Attempts", + "metric_class": "avg_at_n_math", + "metric_params": {"n": 2}, "doc": { "query": "What is 3 * 4?", "choices": ["12"], @@ -54,10 +54,10 @@ "text": ["12", "15"] }, "expected_output": { - "avg@k:k=2": 0.5 + "avg@n:n=2": 0.5 }, "tolerance": 0.01, - "description": "Test avg at k math with multiple attempts" + "description": "Test avg at n math with multiple attempts" } ] } diff --git a/tests/unit/metrics/test_cases/maj_at_k.json b/tests/unit/metrics/test_cases/maj_at_n.json similarity index 62% rename from tests/unit/metrics/test_cases/maj_at_k.json rename to tests/unit/metrics/test_cases/maj_at_n.json index 031735938..7db0f8c06 100644 --- a/tests/unit/metrics/test_cases/maj_at_k.json +++ b/tests/unit/metrics/test_cases/maj_at_n.json @@ -1,11 +1,11 @@ { - "name": "Maj At K Test Suite", - "description": "Test cases for maj_at_k metric", + "name": "Maj At N Test Suite", + "description": "Test cases for maj_at_n metric", "test_cases": [ { - "name": "Maj at K - Majority Correct", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, + "name": "Maj at N - Majority Correct", + "metric_class": "maj_at_n", + "metric_params": {"n": 3}, "doc": { "query": "What is the capital of France?", "choices": ["London", "Paris", "Berlin"], @@ -16,15 +16,15 @@ "text": ["Paris", "Paris", "London"] }, "expected_output": { - "maj@k:k=3": 1 + "maj@n:n=3": 1 }, "tolerance": 0.01, - "description": "Test maj at k with majority correct" + "description": "Test maj at n with majority correct" }, { - "name": "Maj at K - No Majority", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, + "name": "Maj at N - No Majority", + "metric_class": "maj_at_n", + "metric_params": {"n": 3}, "doc": { "query": "What is the capital of France?", "choices": ["London", "Paris", "Berlin"], @@ -35,15 +35,15 @@ "text": ["Paris", "London", "Berlin"] }, "expected_output": { - "maj@k:k=3": 1 + "maj@n:n=3": 1 }, "tolerance": 0.01, - "description": "Test maj at k with no majority" + "description": "Test maj at n with no majority" }, { - "name": "Maj at K - All Correct", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, + "name": "Maj at N - All Correct", + "metric_class": "maj_at_n", + "metric_params": {"n": 3}, "doc": { "query": "What is the capital of France?", "choices": ["London", "Paris", "Berlin"], @@ -54,15 +54,15 @@ "text": ["Paris", "Paris", "Paris"] }, "expected_output": { - "maj@k:k=3": 1 + "maj@n:n=3": 1 }, "tolerance": 0.01, - "description": "Test maj at k with all correct" + "description": "Test maj at n with all correct" }, { - "name": "Maj at K - Wrong Answer", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, + "name": "Maj at N - Wrong Answer", + "metric_class": "maj_at_n", + "metric_params": {"n": 3}, "doc": { "query": "What is the capital of France?", "choices": ["London", "Paris", "Berlin"], @@ -73,10 +73,10 @@ "text": ["London", "London", "London"] }, "expected_output": { - "maj@k:k=3": 0 + "maj@n:n=3": 0 }, "tolerance": 0.01, - "description": "Test maj at k with wrong answer" + "description": "Test maj at n with wrong answer" } ] } diff --git a/tests/unit/metrics/test_metrics_automated.py b/tests/unit/metrics/test_metrics_automated.py index 2f5136cc9..464db7e1e 100644 --- a/tests/unit/metrics/test_metrics_automated.py +++ b/tests/unit/metrics/test_metrics_automated.py @@ -86,6 +86,7 @@ class MetricTestSuite(BaseModel): "faithfulness", # Need GPU to run "bert_score", # Issue with the scoring function, int too big to convert "simpleqa_judge", # Need to setup for compute costs + "maj_at_n", # Blocked by the gold_index bug in Doc.get_golds, see PR #1274 ] @@ -136,6 +137,11 @@ def run_test_case(self, test_case: MetricTestCase | CorpusLevelMetricTestCase) - """Run a single test case and return the result.""" # Check if metric is available in METRIC_CLASSES if test_case.metric_class not in self.METRIC_CLASSES: + if test_case.metric_class not in SKIPPED_METRICS: + raise ValueError( + f"Unknown metric class: {test_case.metric_class}. " + "It is neither a Metrics member nor listed in SKIPPED_METRICS." + ) return { "test_case": test_case.name, "success": True, # Mark as success to skip