Skip to content

fix metric_class names in avg_at_n / avg_at_n_math / maj_at_n test cases #1305

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jayzuccarelli wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix metric_class names in avg_at_n / avg_at_n_math / maj_at_n test cases #1305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions tests/unit/metrics/test_cases/avg_at_k.json → tests/unit/metrics/test_cases/avg_at_n.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "Avg At K Test Suite",
"description": "Test cases for avg_at_k metric",
"name": "Avg At N Test Suite",
"description": "Test cases for avg_at_n metric",
"test_cases": [
{
"name": "Avg at K - Correct in Top K",
"metric_class": "avg_at_k",
"metric_params": {"k": 2},
"name": "Avg at N - Correct in Top N",
"metric_class": "avg_at_n",
"metric_params": {"n": 2},
"doc": {
"query": "What is the capital of France?",
"choices": ["London", "Paris", "Berlin"],
Expand All @@ -16,15 +16,15 @@
"text": ["Paris", "London", "Berlin"]
},
"expected_output": {
"avg@k:k=2": 0.5
"avg@n:n=2": 0.5
},
"tolerance": 0.01,
"description": "Test avg at k with correct answer in top k"
"description": "Test avg at n with correct answer in top n"
},
{
"name": "Avg at K - Not in Top K",
"metric_class": "avg_at_k",
"metric_params": {"k": 1},
"name": "Avg at N - Not in Top N",
"metric_class": "avg_at_n",
"metric_params": {"n": 1},
"doc": {
"query": "What is the capital of France?",
"choices": ["London", "Paris", "Berlin"],
Expand All @@ -35,15 +35,15 @@
"text": ["London", "Berlin", "Paris"]
},
"expected_output": {
"avg@k:k=1": 0.0
"avg@n:n=1": 0.0
},
"tolerance": 0.01,
"description": "Test avg at k with correct answer not in top k"
"description": "Test avg at n with correct answer not in top n"
},
{
"name": "Avg at K - Multiple Correct",
"metric_class": "avg_at_k",
"metric_params": {"k": 3},
"name": "Avg at N - Multiple Correct",
"metric_class": "avg_at_n",
"metric_params": {"n": 3},
"doc": {
"query": "Which are European capitals?",
"choices": ["London", "Paris", "Tokyo", "Berlin"],
Expand All @@ -54,10 +54,10 @@
"text": ["Paris", "London", "Berlin", "Tokyo"]
},
"expected_output": {
"avg@k:k=3": 0.33
"avg@n:n=3": 0.33
},
"tolerance": 0.01,
"description": "Test avg at k with multiple correct answers"
"description": "Test avg at n with multiple correct answers"
}
]
}
34 changes: 17 additions & 17 deletions ...nit/metrics/test_cases/avg_at_k_math.json → ...nit/metrics/test_cases/avg_at_n_math.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "Avg At K Math Test Suite",
"description": "Test cases for avg_at_k_math metric",
"name": "Avg At N Math Test Suite",
"description": "Test cases for avg_at_n_math metric",
"test_cases": [
{
"name": "Avg at K Math - Correct Math",
"metric_class": "avg_at_k_math",
"metric_params": {"k": 1},
"name": "Avg at N Math - Correct Math",
"metric_class": "avg_at_n_math",
"metric_params": {"n": 1},
"doc": {
"query": "What is 2 + 2?",
"choices": ["4"],
Expand All @@ -16,15 +16,15 @@
"text": ["4"]
},
"expected_output": {
"avg@k:k=1": 1.0
"avg@n:n=1": 1.0
},
"tolerance": 0.01,
"description": "Test avg at k math with correct math answer"
"description": "Test avg at n math with correct math answer"
},
{
"name": "Avg at K Math - Wrong Math",
"metric_class": "avg_at_k_math",
"metric_params": {"k": 1},
"name": "Avg at N Math - Wrong Math",
"metric_class": "avg_at_n_math",
"metric_params": {"n": 1},
"doc": {
"query": "What is 2 + 2?",
"choices": ["4"],
Expand All @@ -35,15 +35,15 @@
"text": ["5"]
},
"expected_output": {
"avg@k:k=1": 0.0
"avg@n:n=1": 0.0
},
"tolerance": 0.01,
"description": "Test avg at k math with wrong math answer"
"description": "Test avg at n math with wrong math answer"
},
{
"name": "Avg at K Math - Multiple Attempts",
"metric_class": "avg_at_k_math",
"metric_params": {"k": 2},
"name": "Avg at N Math - Multiple Attempts",
"metric_class": "avg_at_n_math",
"metric_params": {"n": 2},
"doc": {
"query": "What is 3 * 4?",
"choices": ["12"],
Expand All @@ -54,10 +54,10 @@
"text": ["12", "15"]
},
"expected_output": {
"avg@k:k=2": 0.5
"avg@n:n=2": 0.5
},
"tolerance": 0.01,
"description": "Test avg at k math with multiple attempts"
"description": "Test avg at n math with multiple attempts"
}
]
}
44 changes: 22 additions & 22 deletions tests/unit/metrics/test_cases/maj_at_k.json → tests/unit/metrics/test_cases/maj_at_n.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "Maj At K Test Suite",
"description": "Test cases for maj_at_k metric",
"name": "Maj At N Test Suite",
"description": "Test cases for maj_at_n metric",
"test_cases": [
{
"name": "Maj at K - Majority Correct",
"metric_class": "maj_at_k",
"metric_params": {"k": 3},
"name": "Maj at N - Majority Correct",
"metric_class": "maj_at_n",
"metric_params": {"n": 3},
"doc": {
"query": "What is the capital of France?",
"choices": ["London", "Paris", "Berlin"],
Expand All @@ -16,15 +16,15 @@
"text": ["Paris", "Paris", "London"]
},
"expected_output": {
"maj@k:k=3": 1
"maj@n:n=3": 1
},
"tolerance": 0.01,
"description": "Test maj at k with majority correct"
"description": "Test maj at n with majority correct"
},
{
"name": "Maj at K - No Majority",
"metric_class": "maj_at_k",
"metric_params": {"k": 3},
"name": "Maj at N - No Majority",
"metric_class": "maj_at_n",
"metric_params": {"n": 3},
"doc": {
"query": "What is the capital of France?",
"choices": ["London", "Paris", "Berlin"],
Expand All @@ -35,15 +35,15 @@
"text": ["Paris", "London", "Berlin"]
},
"expected_output": {
"maj@k:k=3": 1
"maj@n:n=3": 1
},
"tolerance": 0.01,
"description": "Test maj at k with no majority"
"description": "Test maj at n with no majority"
},
{
"name": "Maj at K - All Correct",
"metric_class": "maj_at_k",
"metric_params": {"k": 3},
"name": "Maj at N - All Correct",
"metric_class": "maj_at_n",
"metric_params": {"n": 3},
"doc": {
"query": "What is the capital of France?",
"choices": ["London", "Paris", "Berlin"],
Expand All @@ -54,15 +54,15 @@
"text": ["Paris", "Paris", "Paris"]
},
"expected_output": {
"maj@k:k=3": 1
"maj@n:n=3": 1
},
"tolerance": 0.01,
"description": "Test maj at k with all correct"
"description": "Test maj at n with all correct"
},
{
"name": "Maj at K - Wrong Answer",
"metric_class": "maj_at_k",
"metric_params": {"k": 3},
"name": "Maj at N - Wrong Answer",
"metric_class": "maj_at_n",
"metric_params": {"n": 3},
"doc": {
"query": "What is the capital of France?",
"choices": ["London", "Paris", "Berlin"],
Expand All @@ -73,10 +73,10 @@
"text": ["London", "London", "London"]
},
"expected_output": {
"maj@k:k=3": 0
"maj@n:n=3": 0
},
"tolerance": 0.01,
"description": "Test maj at k with wrong answer"
"description": "Test maj at n with wrong answer"
}
]
}
6 changes: 6 additions & 0 deletions tests/unit/metrics/test_metrics_automated.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ class MetricTestSuite(BaseModel):
"faithfulness", # Need GPU to run
"bert_score", # Issue with the scoring function, int too big to convert
"simpleqa_judge", # Need to setup for compute costs
"maj_at_n", # Blocked by the gold_index bug in Doc.get_golds, see PR #1274
]


Expand Down Expand Up @@ -136,6 +137,11 @@ def run_test_case(self, test_case: MetricTestCase | CorpusLevelMetricTestCase) -
"""Run a single test case and return the result."""
# Check if metric is available in METRIC_CLASSES
if test_case.metric_class not in self.METRIC_CLASSES:
if test_case.metric_class not in SKIPPED_METRICS:
raise ValueError(
f"Unknown metric class: {test_case.metric_class}. "
"It is neither a Metrics member nor listed in SKIPPED_METRICS."
)
return {
"test_case": test_case.name,
"success": True, # Mark as success to skip
Expand Down