test simpleqa judge

Author: NathanHB

src/lighteval/metrics/metrics_sample.py

@@ -1167,7 +1167,7 @@ def __init__(self, k: int | None = None, **kwargs):
         self.k = k
         self.attribute_must_be_set = ["k"]
 
-    def compute(self, model_response: ModelResponse, doc: Doc):
+    def compute(self, model_response: ModelResponse, doc: Doc, **kwargs):
         """Computes the metric over a list of golds and predictions for one single sample.
         It applies normalisation (if needed) to model prediction and gold, and takes the most frequent answer of all the available ones,
         then compares it to the gold.

tests/unit/metrics/test_cases/acc_golds_likelihood.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f486ec84db5c556b13368da3317bd91629eb93f6a25f869c4972cfed61977656
-size 2012
+oid sha256:5fcce7ab58aed69f3f6bbcab853d40ab7867edc75297ce960a0bed80047d1589
+size 1251

tests/unit/metrics/test_cases/simpleqa_judge.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd3867c275c1afc6a76bdd7aa1cfc4835d4379f5e1b105167c6738a146854d48
-size 953
+oid sha256:4a64b4778c6c7f8b4a69aaf7eb269b156292eb24fff1a737266dadfb4e04a33a
+size 730

tests/unit/metrics/test_metrics_automated.py

@@ -118,8 +118,8 @@ class AutomatedMetricTester:
         "truthfulqa_mc_metrics": Metrics.truthfulqa_mc_metrics,
         # "faithfulness": Metrics.faithfulness,  # need GPU to run
         # "bert_score": Metrics.bert_score, issue with the scoring function, int too big to convert
+        # "simpleqa_judge": Metrics.simpleqa_judge, # Need to setup for compute costs
         "prediction_perplexity": Metrics.prediction_perplexity,
-        # "simpleqa_judge": Metrics.simpleqa_judge, Batched metrics not supported yet
         "bleu": Metrics.bleu,
         "bleu_1": Metrics.bleu_1,
         "bleu_4": Metrics.bleu_4,
@@ -219,15 +219,27 @@ def run_test_case(self, test_case: MetricTestCase | CorpusLevelMetricTestCase) -
         doc = self.create_doc_from_dict(test_case.doc)
         model_response = self.create_model_response_from_dict(test_case.model_response)
 
-        # Create sample_params for the metric
-        sample_params = {
-            "doc": doc,
-            "model_response": model_response,
-        }
+        # Check if this is a batched metric
+        if hasattr(metric, "batched_compute") and metric.batched_compute:
+            # For batched metrics, we need to pass lists of docs and responses
+            sample_params = {
+                "docs": [doc],
+                "responses": [model_response],
+            }
+        else:
+            # For non-batched metrics, use individual doc and model_response
+            sample_params = {
+                "doc": doc,
+                "model_response": model_response,
+            }
 
         # Run the metric using the Metrics enum value
         actual_output = metric.compute_sample(**sample_params)
 
+        # For batched metrics, extract the first result since we're only testing with one sample
+        if hasattr(metric, "batched_compute") and metric.batched_compute and isinstance(actual_output, list):
+            actual_output = actual_output[0]
+
         # Compare with expected output
         success = self._compare_dict_outputs(actual_output, test_case.expected_output, test_case.tolerance)
         return {