revert uneeded changes

Author: NathanHB

src/lighteval/metrics/utils/extractive_match_utils.py

@@ -345,17 +345,13 @@ def lazy_indices_regex(
 
 
 def get_extraction_regexes(
-    # target_types: Sequence[ExtractionTarget], language: Language, len_choices: int = 1
-    formatted_doc: Doc,
-    target_types: Sequence[ExtractionTarget],
-    language: Language,
+    formatted_doc: Doc, target_types: Sequence[ExtractionTarget], language: Language
 ) -> list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]:
     extraction_regexes: list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]] = [
         (lazy_latex_regex(target_type, language), target_type)
         if isinstance(target_type, LatexExtractionConfig)
         else (lazy_expr_regex(target_type, language), target_type)
         if isinstance(target_type, ExprExtractionConfig)
-        # else (lazy_indices_regex(target_type, len_choices, language), target_type)
         else (lazy_indices_regex(target_type, len(formatted_doc.choices), language), target_type)
         for target_type in target_types
     ]

src/lighteval/tasks/default_prompts.py

@@ -30,7 +30,6 @@
 
 import numpy as np
 import pycountry
-from inspect_ai.dataset import Sample
 
 from lighteval.tasks.requests import Doc
 from lighteval.utils.utils import as_list
@@ -131,14 +130,21 @@ def simpleqa(line, task_name: str = None):
     )
 
 
-def aime_prompt_fn(record):
+def aime_prompt_fn(line, task_name: str = None):
     # Prompt template adapted from
     # - simple-evals: https://github.com/openai/simple-evals/blob/6e84f4e2aed6b60f6a0c7b8f06bbbf4bfde72e58/math_eval.py#L17
     # - Llama 3: https://huggingface.co/datasets/meta-llama/Llama-3.2-1B-Instruct-evals/viewer/Llama-3.2-1B-Instruct-evals__math__details?views%5B%5D=llama_32_1b_instruct_evals__math__details
     # Note that it is important to have the final answer in a box for math-verify to work correctly
-    return Sample(
-        input=record["problem"],
-        target=record["answer"],
+    MATH_QUERY_TEMPLATE = """
+Solve the following math problem efficiently and clearly.  The last line of your response should be of the following format: 'Therefore, the final answer is: $\\boxed{{ANSWER}}$. I hope it is correct' (without quotes) where ANSWER is just the final number or expression that solves the problem. Think step by step before answering.
+
+{Question}
+""".strip()
+    return Doc(
+        task_name=task_name,
+        query=MATH_QUERY_TEMPLATE.format(Question=line["problem"]),
+        choices=[line["answer"]],
+        gold_index=0,
     )
 
 
@@ -330,7 +336,7 @@ def bbh_harness(line, task_name: str = None):
     )
 
 
-def bbh(line, task_name: str = None):
+def bbh_lighteval(line, task_name: str = None):
     line = {k: v for k, v in line.items() if v is not None}
 
     query = line.get("task_prefix", "")
@@ -349,6 +355,16 @@ def bbh(line, task_name: str = None):
     )
 
 
+def bbh(line, instruction, choices, task_name: str = None):
+    return Doc(
+        task_name=task_name,
+        query=f"{instruction}Q: {line['input']}\nA:",
+        choices=choices,
+        gold_index=choices.index(line["target"]),
+        instruction=instruction,
+    )
+
+
 def bbh_boolean_expressions(line, task_name: str = None):
     instruction = "Evaluate the result of a random Boolean expression.\n\n"
     choices = ["False", "True"]
@@ -878,19 +894,6 @@ def gpqa(line, task_name: str = None):
     )
 
 
-# def gpqa_instruct(record):
-#     """Prompt template adapted from simple-evals: https://github.com/openai/simple-evals/blob/83ed7640a7d9cd26849bcb3340125002ef14abbe/common.py#L14"""
-#     gold_index = random.randint(0, 3)
-#     choices = [record["Incorrect Answer 1"], record["Incorrect Answer 2"], record["Incorrect Answer 3"]]
-#     choices.insert(gold_index, record["Correct Answer"])
-
-#     return Sample(
-#         input=record["Question"].strip(),
-#         choices=choices,
-#         target=LETTER_INDICES[gold_index],
-#     )
-
-
 def gpqa_instruct(line, task_name: str = None):
     """Prompt template adapted from simple-evals: https://github.com/openai/simple-evals/blob/83ed7640a7d9cd26849bcb3340125002ef14abbe/common.py#L14"""
     gold_index = random.randint(0, 3)
@@ -917,22 +920,29 @@ def gpqa_instruct(line, task_name: str = None):
     )
 
 
-def gsm_plus(record):
+def gsm_plus(line, task_name: str = None):
     # GSM8K with 8 prompt variations per sample
 
     # Some prompts require critical thinking (around 1k/10k), we skip them as
     # they are a bit trickier to eval with regular text extraction.
+    if line["perturbation_type"] == "critical thinking":
+        return None
 
-    return Sample(
-        input=record["question"],
-        target=record["answer"],
+    return Doc(
+        task_name=task_name,
+        query=f"Question: {line['question']}\n\nAnswer:",
+        choices=[line["answer"]],
+        gold_index=0,
     )
 
 
-def gsm8k(record):
-    return Sample(
-        input=record["question"],
-        target=record["answer"],
+def gsm8k(line, task_name: str = None):
+    # Has special analysis in metric for number decomposition
+    return Doc(
+        task_name=task_name,
+        query=f"Question: {line['question']}\nAnswer:",
+        choices=[f" {line['answer']}"],
+        gold_index=0,
     )
 
 

src/lighteval/tasks/lighteval_task.py

@@ -27,7 +27,6 @@
 
 from datasets import DatasetDict, load_dataset
 from huggingface_hub import TextGenerationInputGrammarType
-from inspect_ai.dataset import Sample
 from multiprocess import Pool
 from pytablewriter import MarkdownTableWriter
 
@@ -44,28 +43,6 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclass
-class LightevalTaskConfig_inspect:
-    """Configuration dataclass for a LightevalTask.
-
-    This class stores all the configuration parameters needed to define and run
-    an evaluation task, including dataset information, prompt formatting,
-    evaluation metrics, and generation parameters.
-    """
-
-    name: str
-    prompt_function: Callable[[dict], Sample]
-    dataset_repo: str
-    dataset_subset: str
-    dataset_split: str
-    scorers: list
-    solvers: list | None = None
-    system_prompt: str | None = None
-    dataset_revision: str | None = None
-    epochs: int = 1
-    epochs_reducer: str | None = None
-
-
 @dataclass
 class LightevalTaskConfig:
     """Configuration dataclass for a LightevalTask.