use inspect-ai to evaluate aime25, gsm8k and IFEval

src/lighteval/main_inspect.py

@@ -0,0 +1,71 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+from inspect_ai import Epochs, Task, eval, task
+from inspect_ai.dataset import hf_dataset
+from inspect_ai.solver import generate, system_message
+
+from lighteval.tasks import default_tasks
+from lighteval.tasks.lighteval_task import LightevalTaskConfig_inspect as LightevalTaskConfig
+
+
+@task
+def get_task(lighteval_task_config: LightevalTaskConfig):
+    name = lighteval_task_config.name
+    sample_fields = lighteval_task_config.prompt_function
+
+    dataset_repo = lighteval_task_config.dataset_repo
+    dataset_subset = lighteval_task_config.dataset_subset
+    dataset_split = lighteval_task_config.dataset_split
+
+    system_prompt = lighteval_task_config.system_prompt
+    metrics = lighteval_task_config.metrics
+
+    dataset = hf_dataset(dataset_repo, name=dataset_subset, split=dataset_split, sample_fields=sample_fields)
+    solver = [
+        system_message(system_prompt),
+        generate(cache=True),
+    ]
+    scorer = metrics
+    epochs = lighteval_task_config.epochs
+    epochs_reducer = lighteval_task_config.epochs_reducer
+
+    return Task(dataset=dataset, solver=solver, scorer=scorer, name=name, epochs=Epochs(epochs, epochs_reducer))
+
+
+def main():
+    MODEL = ["openai/gpt-4o"]
+    all_tasks = [
+        default_tasks.gsm8k_lighteval,
+        default_tasks.aime25,
+        default_tasks.gpqa_diamond,
+    ]  # default_tasksifeval]
+    all_tasks = [get_task(task) for task in all_tasks]
+
+    # eval_set(all_tasks, model=MODEL, display="rich", limit=10, max_tasks=3, bundle_dir="./log_static", log_dir="./log_dynamic-1")
+
+    eval(all_tasks[-1], model=MODEL, display="rich", limit=10, max_tasks=3)
+
+
+if __name__ == "__main__":
+    main()

src/lighteval/metrics/metrics.py

@@ -25,6 +25,8 @@
 
 import numpy as np
 from aenum import Enum
+from inspect_ai.scorer import Score, Target, accuracy, scorer, stderr
+from inspect_ai.solver import TaskState
 
 from lighteval.metrics.dynamic_metrics import MultilingualExtractiveMatchMetric
 from lighteval.metrics.harness_compatibility.drop import DropMetrics
@@ -66,6 +68,8 @@
     ExprExtractionConfig,
     IndicesExtractionConfig,
     LatexExtractionConfig,
+    extract_target_from_pred,
+    get_extraction_regexes,
 )
 from lighteval.metrics.utils.metric_utils import (
     CorpusLevelMetric,
@@ -77,6 +81,66 @@
 from lighteval.utils.language import Language
 
 
+@scorer(metrics=[accuracy(), stderr()])
+def extractive_math_scorer():
+    gold_extraction_target = (ExprExtractionConfig(),)
+    pred_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0))
+    language = Language.ENGLISH
+    fallback_mode = "first_match"
+    extraction_mode = "first_match"
+    timeout_seconds = 5
+
+    gold_extraction_regexes = get_extraction_regexes(gold_extraction_target, language)
+    pred_extraction_regexes = get_extraction_regexes(pred_extraction_target, language)
+
+    async def score(state: TaskState, target: Target):
+        extracted_predictions = extract_target_from_pred(
+            state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+        )
+        extracted_gold = extract_target_from_pred(
+            target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+        )
+        return Score(
+            value="C" if extracted_predictions == extracted_gold else "I",
+            explanation=state.output.completion,
+            answer=str(extracted_predictions),
+        )
+
+    return score
+
+
+@scorer(metrics=[accuracy(), stderr()])
+def multichoice_scorer():
+    language = Language.ENGLISH
+    gold_extraction_target = (
+        IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
+    )
+    pred_extraction_target = (
+        IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
+    )
+    fallback_mode = "first_match"
+    extraction_mode = "first_match"
+    timeout_seconds = 5
+
+    gold_extraction_regexes = get_extraction_regexes(gold_extraction_target, language)
+    pred_extraction_regexes = get_extraction_regexes(pred_extraction_target, language)
+
+    async def score(state: TaskState, target: Target):
+        extracted_predictions = extract_target_from_pred(
+            state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+        )
+        extracted_gold = extract_target_from_pred(
+            target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+        )
+        return Score(
+            value="C" if extracted_predictions == extracted_gold else "I",
+            explanation=state.output.completion,
+            answer=str(extracted_predictions),
+        )
+
+    return score
+
+
 class Metrics(Enum):
     acc_golds_likelihood = SampleLevelMetric(  # todo: we need a better name for this!
         metric_name="acc",
@@ -85,14 +149,14 @@ class Metrics(Enum):
         corpus_level_fn=np.mean,
         higher_is_better=True,
     )
-    avg_at_k = SampleLevelMetric(
+    avg_at_k = SampleLevelMetric(  #
         metric_name="avg@k",
         sample_level_fn=AvgAtK(strip_strings=True),
         category=SamplingMethod.GENERATIVE,
         corpus_level_fn=np.mean,
         higher_is_better=True,
     )
-    avg_at_k_math = SampleLevelMetric(
+    avg_at_k_math = SampleLevelMetric(  #
         metric_name="avg@k",
         sample_level_fn=AvgAtK(
             sample_scoring_function=MultilingualExtractiveMatchMetric(

src/lighteval/metrics/utils/extractive_match_utils.py

@@ -31,7 +31,6 @@
 from sympy.parsing import parse_expr
 
 from lighteval.metrics.utils.math_comparison import should_treat_as_complex
-from lighteval.tasks.requests import Doc
 from lighteval.tasks.templates.utils.formulation import ChoicePrefix, get_prefix
 from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
 from lighteval.utils.imports import requires
@@ -345,14 +344,14 @@ def lazy_indices_regex(
 
 
 def get_extraction_regexes(
-    formatted_doc: Doc, target_types: Sequence[ExtractionTarget], language: Language
+    target_types: Sequence[ExtractionTarget], language: Language, len_choices: int = 1
 ) -> list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]:
     extraction_regexes: list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]] = [
         (lazy_latex_regex(target_type, language), target_type)
         if isinstance(target_type, LatexExtractionConfig)
         else (lazy_expr_regex(target_type, language), target_type)
         if isinstance(target_type, ExprExtractionConfig)
-        else (lazy_indices_regex(target_type, len(formatted_doc.choices), language), target_type)
+        else (lazy_indices_regex(target_type, len_choices, language), target_type)
         for target_type in target_types
     ]
 

src/lighteval/tasks/default_prompts.py

@@ -30,6 +30,7 @@
 
 import numpy as np
 import pycountry
+from inspect_ai.dataset import Sample
 
 from lighteval.tasks.requests import Doc
 from lighteval.utils.utils import as_list
@@ -130,21 +131,14 @@ def simpleqa(line, task_name: str = None):
     )
 
 
-def aime_prompt_fn(line, task_name: str = None):
+def aime_prompt_fn(record):
     # Prompt template adapted from
     # - simple-evals: https://github.com/openai/simple-evals/blob/6e84f4e2aed6b60f6a0c7b8f06bbbf4bfde72e58/math_eval.py#L17
     # - Llama 3: https://huggingface.co/datasets/meta-llama/Llama-3.2-1B-Instruct-evals/viewer/Llama-3.2-1B-Instruct-evals__math__details?views%5B%5D=llama_32_1b_instruct_evals__math__details
     # Note that it is important to have the final answer in a box for math-verify to work correctly
-    MATH_QUERY_TEMPLATE = """
-Solve the following math problem efficiently and clearly.  The last line of your response should be of the following format: 'Therefore, the final answer is: $\\boxed{{ANSWER}}$. I hope it is correct' (without quotes) where ANSWER is just the final number or expression that solves the problem. Think step by step before answering.
-
-{Question}
-""".strip()
-    return Doc(
-        task_name=task_name,
-        query=MATH_QUERY_TEMPLATE.format(Question=line["problem"]),
-        choices=[line["answer"]],
-        gold_index=0,
+    return Sample(
+        input=record["problem"],
+        target=record["answer"],
     )
 
 
@@ -894,29 +888,16 @@ def gpqa(line, task_name: str = None):
     )
 
 
-def gpqa_instruct(line, task_name: str = None):
+def gpqa_instruct(record):
     """Prompt template adapted from simple-evals: https://github.com/openai/simple-evals/blob/83ed7640a7d9cd26849bcb3340125002ef14abbe/common.py#L14"""
     gold_index = random.randint(0, 3)
-    choices = [line["Incorrect Answer 1"], line["Incorrect Answer 2"], line["Incorrect Answer 3"]]
-    choices.insert(gold_index, line["Correct Answer"])
-    instruction = "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering."
-    query_template = "{Instruction}\n\n{Question}\n\nA) {A}\nB) {B}\nC) {C}\nD) {D}"
-    query = query_template.format(
-        # Stripping to avoid accidental extra whitespaces, present in GPQA
-        A=choices[0].strip(),
-        B=choices[1].strip(),
-        C=choices[2].strip(),
-        D=choices[3].strip(),
-        Question=line["Question"].strip(),
-        Instruction=instruction,
-    )
+    choices = [record["Incorrect Answer 1"], record["Incorrect Answer 2"], record["Incorrect Answer 3"]]
+    choices.insert(gold_index, record["Correct Answer"])
 
-    return Doc(
-        task_name=task_name,
-        query=query,
-        choices=LETTER_INDICES[: len(choices)],
-        gold_index=gold_index,
-        instruction=instruction,
+    return Sample(
+        input=record["Question"].strip(),
+        choices=choices,
+        target=LETTER_INDICES[gold_index],
     )
 
 
@@ -936,13 +917,10 @@ def gsm_plus(line, task_name: str = None):
     )
 
 
-def gsm8k(line, task_name: str = None):
-    # Has special analysis in metric for number decomposition
-    return Doc(
-        task_name=task_name,
-        query=f"Question: {line['question']}\nAnswer:",
-        choices=[f" {line['answer']}"],
-        gold_index=0,
+def gsm8k(record):
+    return Sample(
+        input=record["question"],
+        target=record["answer"],
     )