Parametrizing the sampling evals from the CLI (#926)

This PR does several things.

    Contrain the Metric object creation.
    We go from

    f1_score_macro = CorpusLevelMetric(
        metric_name="f1",
        sample_level_fn=GenerativePreparator().prepare,
        category=SamplingMethod.GENERATIVE,
        corpus_level_fn=CorpusLevelF1Score(average="macro").compute,
        higher_is_better=True,
    )

to

    f1_score_macro = CorpusLevelMetric(
        metric_name="f1",
        sample_level_fn=GenerativePreparator(),
        category=SamplingMethod.GENERATIVE,
        corpus_level_fn=CorpusLevelF1Score(average="macro"),
        higher_is_better=True,
    )

    sample_level_fn must derive from either a SampleLevelComputation or Preparator class. The former must implement a compute method, the latter a prepare one.
    corpus_level_fn either is a function (np.mean and so on), or a CorpusLevelComputation which must implement a .compute_corpus class.

    All metrics with parametrizable sample_level_fn can now be parametrized at CLI call, example: "lighteval|math_500@k=1|0|0" (user can also use normalization function names if correctly defined in the normalization file). Corpus level parametrization is not supported but could probably be if we choose another symbol. This parametrization of Metrics.MyMetric relies on a trick in making the enum callable.

    The metric list has been simplified to remove duplicate metrics which were only different by some params

    All tasks have therefore been changed to use the new metrics names.

    The test suite has been updated

Author: clefourrier

community_tasks/aimo_evals.py

@@ -26,6 +26,7 @@
 """
 
 from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import math_normalizer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
@@ -49,7 +50,9 @@ def aimo_prompt(line, task_name: str = None):
     evaluation_splits=["train"],
     few_shots_split="train",
     few_shots_select="sequential",
-    metrics=[Metrics.quasi_exact_match_math],
+    metrics=[
+        Metrics.exact_match(sample_params={"normalize_gold": math_normalizer, "normalize_pred": math_normalizer})
+    ],
     generation_size=2048,
     stop_sequence=None,
 )

community_tasks/arabic_evals.py

@@ -31,8 +31,10 @@
 import re
 from typing import Any, Dict, List, Optional, Union
 
-from lighteval.metrics.llm_as_judge import JudgeLM
-from lighteval.metrics.metrics import Metric, Metrics
+from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import LogProbCharNorm
+from lighteval.metrics.utils.llm_as_judge import JudgeLM
+from lighteval.metrics.utils.metric_utils import Metric
 from lighteval.tasks.default_prompts import LETTER_INDICES
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc, SamplingMethod
@@ -103,7 +105,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=arabic_mmlu_pfn,
             hf_repo="MBZUAI/ArabicMMLU",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test"],
             evaluation_splits=["test"],
             few_shots_split=["dev"],
@@ -164,7 +166,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=arabic_mmlu_ht_pfn,
             hf_repo="MBZUAI/human_translated_arabic_mmlu",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test"],
             evaluation_splits=["test"],
             few_shots_split=None,
@@ -228,7 +230,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=arabic_mmlu_mt_pfn,
             hf_repo="OALL/Arabic_MMLU",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test", "dev"],
             evaluation_splits=["test"],
             few_shots_split="dev",
@@ -283,7 +285,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=acva_pfn,
             hf_repo="OALL/ACVA",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test", "validation"],
             evaluation_splits=["test"],
             few_shots_split="validation",
@@ -339,7 +341,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=aratrust_pfn,
             hf_repo="asas-ai/AraTrust-categorized",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["train"],
             evaluation_splits=["train"],
             few_shots_split=None,
@@ -387,7 +389,7 @@ def arabic_exams_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -437,7 +439,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=alghafa_pfn,
             hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test", "validation"],
             evaluation_splits=["test"],
             few_shots_split="validation",
@@ -463,7 +465,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -479,7 +481,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -495,7 +497,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -511,7 +513,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -527,7 +529,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -543,7 +545,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -580,7 +582,7 @@ def boolq_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -614,7 +616,7 @@ def copa_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -657,7 +659,7 @@ def hellaswag_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -693,7 +695,7 @@ def toxigen_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -743,7 +745,7 @@ def sciq_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -800,7 +802,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=madinah_qa_pfn,
             hf_repo="MBZUAI/MadinahQA",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test"],
             evaluation_splits=["test"],
             few_shots_split=["dev"],

community_tasks/french_evals.py

@@ -33,6 +33,7 @@
 import random
 
 from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import math_normalizer
 from lighteval.tasks.default_prompts import LETTER_INDICES
 from lighteval.tasks.extended.ifeval.main import ifeval_metrics
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
@@ -136,7 +137,10 @@ def prompt_bac_fr(line, task_name: str = None):
     few_shots_split=None,
     few_shots_select="random_sampling",
     generation_size=1,
-    metrics=[Metrics.quasi_exact_match_math, Metrics.exact_match],
+    metrics=[
+        Metrics.exact_match(sample_params={"normalize_gold": math_normalizer, "normalize_pred": math_normalizer}),
+        Metrics.exact_match,
+    ],
     stop_sequence=["\n"],
     version=0,
 )