move tasks to individual files

Author: NathanHB

src/lighteval/metrics/metrics.py

@@ -81,64 +81,64 @@
 from lighteval.utils.language import Language
 
 
-@scorer(metrics=[accuracy(), stderr()])
-def extractive_math_scorer():
-    gold_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0))
-    pred_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0))
-    language = Language.ENGLISH
-    fallback_mode = "first_match"
-    extraction_mode = "first_match"
-    timeout_seconds = 5
+# @scorer(metrics=[accuracy(), stderr()])
+# def extractive_math_scorer():
+#     gold_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0))
+#     pred_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0))
+#     language = Language.ENGLISH
+#     fallback_mode = "first_match"
+#     extraction_mode = "first_match"
+#     timeout_seconds = 5
 
-    gold_extraction_regexes = get_extraction_regexes(gold_extraction_target, language)
-    pred_extraction_regexes = get_extraction_regexes(pred_extraction_target, language)
+#     gold_extraction_regexes = get_extraction_regexes(gold_extraction_target, language)
+#     pred_extraction_regexes = get_extraction_regexes(pred_extraction_target, language)
 
-    async def score(state: TaskState, target: Target):
-        extracted_predictions = extract_target_from_pred(
-            state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
-        )
-        extracted_gold = extract_target_from_pred(
-            target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
-        )
-        return Score(
-            value="C" if extracted_predictions == extracted_gold else "I",
-            explanation=state.output.completion,
-            answer=str(extracted_predictions),
-        )
+#     async def score(state: TaskState, target: Target):
+#         extracted_predictions = extract_target_from_pred(
+#             state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+#         )
+#         extracted_gold = extract_target_from_pred(
+#             target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+#         )
+#         return Score(
+#             value="C" if extracted_predictions == extracted_gold else "I",
+#             explanation=state.output.completion,
+#             answer=str(extracted_predictions),
+#         )
 
-    return score
+#     return score
 
 
-@scorer(metrics=[accuracy(), stderr()])
-def multichoice_scorer():
-    language = Language.ENGLISH
-    gold_extraction_target = (
-        IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
-    )
-    pred_extraction_target = (
-        IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
-    )
-    fallback_mode = "first_match"
-    extraction_mode = "first_match"
-    timeout_seconds = 5
+# @scorer(metrics=[accuracy(), stderr()])
+# def multichoice_scorer():
+#     language = Language.ENGLISH
+#     gold_extraction_target = (
+#         IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
+#     )
+#     pred_extraction_target = (
+#         IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
+#     )
+#     fallback_mode = "first_match"
+#     extraction_mode = "first_match"
+#     timeout_seconds = 5
 
-    gold_extraction_regexes = get_extraction_regexes(gold_extraction_target, language, len_choices=4)
-    pred_extraction_regexes = get_extraction_regexes(pred_extraction_target, language, len_choices=4)
+#     gold_extraction_regexes = get_extraction_regexes(gold_extraction_target, language, len_choices=4)
+#     pred_extraction_regexes = get_extraction_regexes(pred_extraction_target, language, len_choices=4)
 
-    async def score(state: TaskState, target: Target):
-        extracted_predictions = extract_target_from_pred(
-            state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
-        )
-        extracted_gold = extract_target_from_pred(
-            target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
-        )
-        return Score(
-            value="C" if extracted_predictions == extracted_gold else "I",
-            explanation=state.output.completion,
-            answer=str(extracted_predictions),
-        )
+#     async def score(state: TaskState, target: Target):
+#         extracted_predictions = extract_target_from_pred(
+#             state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+#         )
+#         extracted_gold = extract_target_from_pred(
+#             target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
+#         )
+#         return Score(
+#             value="C" if extracted_predictions == extracted_gold else "I",
+#             explanation=state.output.completion,
+#             answer=str(extracted_predictions),
+#         )
 
-    return score
+#     return score
 
 
 class Metrics(Enum):

src/lighteval/metrics/utils/extractive_match_utils.py

@@ -31,6 +31,7 @@
 from sympy.parsing import parse_expr
 
 from lighteval.metrics.utils.math_comparison import should_treat_as_complex
+from lighteval.tasks.requests import Doc
 from lighteval.tasks.templates.utils.formulation import ChoicePrefix, get_prefix
 from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
 from lighteval.utils.imports import requires
@@ -344,14 +345,16 @@ def lazy_indices_regex(
 
 
 def get_extraction_regexes(
-    target_types: Sequence[ExtractionTarget], language: Language, len_choices: int = 1
+    # target_types: Sequence[ExtractionTarget], language: Language, len_choices: int = 1
+    formatted_doc: Doc, target_types: Sequence[ExtractionTarget], language: Language
 ) -> list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]:
     extraction_regexes: list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]] = [
         (lazy_latex_regex(target_type, language), target_type)
         if isinstance(target_type, LatexExtractionConfig)
         else (lazy_expr_regex(target_type, language), target_type)
         if isinstance(target_type, ExprExtractionConfig)
-        else (lazy_indices_regex(target_type, len_choices, language), target_type)
+        # else (lazy_indices_regex(target_type, len_choices, language), target_type)
+        else (lazy_indices_regex(target_type, len(formatted_doc.choices), language), target_type)
         for target_type in target_types
     ]
 

src/lighteval/tasks/__init__.py

@@ -19,3 +19,50 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+
+"""
+Automatically imports all task configs from the tasks/ directory.
+This module dynamically loads all Python files in tasks/ and exposes their LightevalTaskConfig objects.
+"""
+
+import importlib
+import logging
+from pathlib import Path
+
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+
+
+logger = logging.getLogger(__name__)
+
+
+# Get the tasks directory
+TASKS_DIR = Path(__file__).parent / "tasks"
+
+
+def _load_all_task_configs():
+    """Load all LightevalTaskConfig objects from all Python files in the tasks/ directory."""
+    loaded_configs = {}
+
+    # Get all Python files in the tasks directory (excluding __init__.py and subdirectories)
+    task_files = [f for f in TASKS_DIR.glob("*.py") if f.name != "__init__.py"]
+
+    for task_file in task_files:
+        module_name = task_file.stem
+        # Import the module
+        module = importlib.import_module(f"lighteval.tasks.tasks.{module_name}")
+
+        # Find all LightevalTaskConfig objects in the module
+        for attr_name in dir(module):
+            attr = getattr(module, attr_name)
+            if isinstance(attr, LightevalTaskConfig):
+                loaded_configs[attr_name] = attr
+
+    return loaded_configs
+
+
+# Load all configs and add them to module namespace
+_configs = _load_all_task_configs()
+globals().update(_configs)
+
+# Clean up
+del _configs

src/lighteval/tasks/default_prompts.py

@@ -888,16 +888,41 @@ def gpqa(line, task_name: str = None):
     )
 
 
-def gpqa_instruct(record):
+# def gpqa_instruct(record):
+#     """Prompt template adapted from simple-evals: https://github.com/openai/simple-evals/blob/83ed7640a7d9cd26849bcb3340125002ef14abbe/common.py#L14"""
+#     gold_index = random.randint(0, 3)
+#     choices = [record["Incorrect Answer 1"], record["Incorrect Answer 2"], record["Incorrect Answer 3"]]
+#     choices.insert(gold_index, record["Correct Answer"])
+
+#     return Sample(
+#         input=record["Question"].strip(),
+#         choices=choices,
+#         target=LETTER_INDICES[gold_index],
+#     )
+
+def gpqa_instruct(line, task_name: str = None):
     """Prompt template adapted from simple-evals: https://github.com/openai/simple-evals/blob/83ed7640a7d9cd26849bcb3340125002ef14abbe/common.py#L14"""
     gold_index = random.randint(0, 3)
-    choices = [record["Incorrect Answer 1"], record["Incorrect Answer 2"], record["Incorrect Answer 3"]]
-    choices.insert(gold_index, record["Correct Answer"])
+    choices = [line["Incorrect Answer 1"], line["Incorrect Answer 2"], line["Incorrect Answer 3"]]
+    choices.insert(gold_index, line["Correct Answer"])
+    instruction = "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering."
+    query_template = "{Instruction}\n\n{Question}\n\nA) {A}\nB) {B}\nC) {C}\nD) {D}"
+    query = query_template.format(
+        # Stripping to avoid accidental extra whitespaces, present in GPQA
+        A=choices[0].strip(),
+        B=choices[1].strip(),
+        C=choices[2].strip(),
+        D=choices[3].strip(),
+        Question=line["Question"].strip(),
+        Instruction=instruction,
+    )
 
-    return Sample(
-        input=record["Question"].strip(),
-        choices=choices,
-        target=LETTER_INDICES[gold_index],
+    return Doc(
+        task_name=task_name,
+        query=query,
+        choices=LETTER_INDICES[: len(choices)],
+        gold_index=gold_index,
+        instruction=instruction,
     )
 
 

src/lighteval/tasks/registry.py

@@ -33,8 +33,7 @@
 from pathlib import Path
 from types import ModuleType
 
-import lighteval.tasks.default_tasks as default_tasks
-from lighteval.tasks.extended import AVAILABLE_EXTENDED_TASKS_MODULES
+import lighteval.tasks as default_tasks
 from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig
 
 
@@ -227,51 +226,18 @@ def _load_full_registry(self) -> dict[str, LightevalTaskConfig]:
 
         Example:
             {
-                "lighteval|arc_easy": LightevalTaskConfig(name="arc_easy", suite="lighteval", ...),
+                "arc_easy": LightevalTaskConfig(name="arc_easy", suite="lighteval", ...),
             }
         """
-        custom_tasks_registry = {}
-        custom_tasks_module = []
-        custom_task_configs = []
 
-        if self._custom_tasks is not None:
-            custom_tasks_module.append(Registry.create_custom_tasks_module(custom_tasks=self._custom_tasks))
-
-        # Need to load extended tasks
-        if self._load_extended:
-            for extended_task_module in AVAILABLE_EXTENDED_TASKS_MODULES:
-                custom_tasks_module.append(extended_task_module)
-
-        # Need to load community tasks
-        if self._load_community:
-            community_modules = load_community_tasks()
-            for community_task_module in community_modules:
-                custom_tasks_module.append(community_task_module)
+        return Registry.create_task_config_dict()
 
         # Need to load multilingual tasks
         if self._load_multilingual:
-            import lighteval.tasks.multilingual.tasks as multilingual_tasks
-
-            custom_tasks_module.append(multilingual_tasks)
-
-        # We load all
-        for module in custom_tasks_module:
-            custom_task_configs.extend(module.TASKS_TABLE)
-            logger.info(f"Found {len(module.TASKS_TABLE)} custom tasks in {module.__file__}")
-
-        if len(custom_task_configs) > 0:
-            custom_tasks_registry = Registry.create_task_config_dict(meta_table=custom_task_configs)
-
-        default_tasks_registry = Registry.create_task_config_dict()
-
-        # Check the overlap between default_tasks_registry and custom_tasks_registry
-        intersection = set(default_tasks_registry.keys()).intersection(set(custom_tasks_registry.keys()))
-        if len(intersection) > 0:
-            logger.warning(
-                f"Following tasks ({intersection}) exists both in the default and custom tasks. Will use the custom ones on conflict."
-            )
+            pass
+        tasks_registry = {}
 
-        return {**default_tasks_registry, **custom_tasks_registry}
+        return tasks_registry
 
     def _update_task_configs(self) -> dict[str, LightevalTaskConfig]:  # noqa: C901
         """