Fixe Sampling Metrics and Evals (#938)

* Fixe Sampling Metrics and Evals

* remove breakpoint

* Apply suggestion from @NathanHB

* Apply suggestion from @NathanHB

Author: NathanHB

src/lighteval/metrics/dynamic_metrics.py

@@ -220,7 +220,7 @@ def __init__(
 
     @timeout(2)
     def add_to_specifics_with_timeout(
-        formatted_doc: Doc, extracted_predictions: list[list[str]], extracted_golds: list[list[str]]
+        self, formatted_doc: Doc, extracted_predictions: list[list[str]], extracted_golds: list[list[str]]
     ) -> None:
         if formatted_doc.specific is None:
             formatted_doc.specific = {}
@@ -263,7 +263,7 @@ def compute(self, doc: Doc, model_response: ModelResponse) -> float:
         # We have to use timeout because the sypmy to str conversion can be very slow
         try:
             self.add_to_specifics_with_timeout(doc, extracted_predictions, extracted_golds)
-        except Exception:  # noqa: E722
+        except TimeoutError:  # noqa: E722
             logger.warning("Timeout when adding extracted predictions and golds to specific")
 
         return self.aggregation_function(

src/lighteval/metrics/metrics_sample.py

@@ -63,7 +63,7 @@
 
 class SampleLevelComputation(ABC):
     @abstractmethod
-    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
+    def compute(self, model_response: ModelResponse, doc: Doc, **kwargs):
         raise NotImplementedError
 
 
@@ -1112,13 +1112,16 @@ def __init__(
         if callable(sample_scoring_function):
             self.score_sample = sample_scoring_function
             self.type_exact_match = None
+        elif isinstance(sample_scoring_function, SampleLevelComputation):
+            self.score_sample = sample_scoring_function.compute
         else:
             if isinstance(sample_scoring_function, str):
                 if sample_scoring_function not in ["prefix", "suffix", "full"]:
                     raise ValueError(
                         f"type_exact_match (used in parametrized_exact_match) must be one of prefix, suffix, or full. Was {sample_scoring_function} instead."
                     )
                 self.type_exact_match = sample_scoring_function
+                self.score_sample = self.default_sample_scoring
             else:
                 self.type_exact_match = "full"
             self.compute_score = self.default_sample_scoring
@@ -1130,7 +1133,7 @@ def preprocess(self, text: str) -> str:
         if self.strip_strings:
             text = text.strip()
 
-        if self.normalize:
+        if self.normalize is not None:
             text = self.normalize(text)
 
         return text
@@ -1161,7 +1164,7 @@ def __init__(self, k: int | None = None, **kwargs):
             sample_scoring_function (callable | str, optional): Function to use to compute the score for each sample.
                 If None, uses the default scoring function which is a simple exact match.
         """
-        super().__init__(kwargs)
+        super().__init__(**kwargs)
         self.k = k
         self.attribute_must_be_set = ["k"]
 
@@ -1189,9 +1192,9 @@ def num_samples(self):
 
 
 class MajAtK(SamplingMetric, SampleLevelComputation):
-    def __init__(self, k: int = None, **kwargs):
+    def __init__(self, k: int | None = None, **kwargs):
         """An exact match class."""
-        super().__init__(kwargs)
+        super().__init__(**kwargs)
 
         self.k = k
         self.attribute_must_be_set = ["k"]
@@ -1214,15 +1217,15 @@ def compute(self, model_response: ModelResponse, docs: Doc, **kwargs):
         if len(golds) > 1:
             raise Exception("Cannot compute maj@k with several golds")
 
-        processed_choices = [self.preprocess(gold=g) for g in docs.get_golds()]
+        processed_choices = [self.preprocess(text=g) for g in docs.get_golds()]
         new_doc = Doc(
             choices=processed_choices,
             query=docs.query,
             gold_index=docs.gold_index,
         )
         all_answers = []
         for pred in model_response.final_text[: self.k]:
-            all_answers.append(self.preprocess(pred=pred))
+            all_answers.append(self.preprocess(text=pred))
         majority_prediction = max(all_answers, key=all_answers.count)
         new_model_response = ModelResponse(
             text=[majority_prediction],
@@ -1241,7 +1244,7 @@ def __init__(self, k: int | None = None, n: int | None = None, **kwargs):
             k (int): Threshold for the number of successful attempts.
             n (int): Number of samples to generate
         """
-        super().__init__(kwargs)
+        super().__init__(**kwargs)
         self.k = k
         self.n = n
         self.attribute_must_be_set = ["k"]
@@ -1269,7 +1272,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
         elif len(predictions) < self.n:
             logger.warning(f"Number of predictions is less than {self.n} for pass@k.")
 
-        processed_choices = [self.preprocess(gold=g) for g in doc.choices]
+        processed_choices = [self.preprocess(text=g) for g in doc.choices]
         new_doc = Doc(
             choices=processed_choices,
             query=doc.query,
@@ -1278,11 +1281,11 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
 
         all_scores = []
         for pred in predictions[: self.n]:
-            cur_pred = self.preprocess(pred=pred)
+            cur_pred = self.preprocess(text=pred)
             new_model_response = ModelResponse(
                 text=[cur_pred],
             )
-            all_scores.append(self.score_sample(new_doc, new_model_response))
+            all_scores.append(self.score_sample(doc=new_doc, model_response=new_model_response))
 
         return self.pass_at_k(all_scores)
 
@@ -1314,7 +1317,7 @@ def __init__(
             n (int): Number of samples to generate.
             thresholds (list): Thresholds to control successful attempts in k generate.
         """
-        super().__init__(kwargs)
+        super().__init__(**kwargs)
         self._k = k
         self.n = n
         self.attribute_must_be_set = ["k"]
@@ -1356,7 +1359,7 @@ def compute(self, model_response: ModelResponse, doc: Doc, **kwargs) -> float:
         elif len(predictions) < self.n:
             logger.warning(f"Number of predictions is less than {self.n} for G-Pass@k.")
 
-        processed_choices = [self.preprocess(gold=g) for g in doc.choices]
+        processed_choices = [self.preprocess(text=g) for g in doc.choices]
         new_doc = Doc(
             choices=processed_choices,
             query=doc.query,
@@ -1365,7 +1368,7 @@ def compute(self, model_response: ModelResponse, doc: Doc, **kwargs) -> float:
 
         all_scores = []
         for pred in predictions[: self.n]:
-            cur_pred = self.preprocess(pred=pred)
+            cur_pred = self.preprocess(text=pred)
             new_model_response = ModelResponse(
                 text=[cur_pred],
             )