enable extended tasks as well

Author: NathanHB

src/lighteval/tasks/__init__.py

@@ -43,9 +43,12 @@ def _load_all_task_configs():
     """Load all LightevalTaskConfig objects from all Python files in the tasks/ directory."""
     loaded_configs = {}
 
-    # Get all Python files in the tasks directory (excluding __init__.py and subdirectories)
+    # Get all Python files in the tasks directory (excluding __init__.py)
     task_files = [f for f in TASKS_DIR.glob("*.py") if f.name != "__init__.py"]
 
+    # Also get all subdirectories with main.py files
+    task_subdirs = [d for d in TASKS_DIR.iterdir() if d.is_dir() and (d / "main.py").exists()]
+
     for task_file in task_files:
         module_name = task_file.stem
         # Import the module
@@ -57,6 +60,18 @@ def _load_all_task_configs():
             if isinstance(attr, LightevalTaskConfig):
                 loaded_configs[attr_name] = attr
 
+    # Load from subdirectories' main.py files
+    for task_dir in task_subdirs:
+        module_name = task_dir.name
+        # Import the main.py from the subdirectory
+        module = importlib.import_module(f"lighteval.tasks.tasks.{module_name}.main")
+
+        # Find all LightevalTaskConfig objects in the module
+        for attr_name in dir(module):
+            attr = getattr(module, attr_name)
+            if isinstance(attr, LightevalTaskConfig):
+                loaded_configs[attr_name] = attr
+
     return loaded_configs
 
 

src/lighteval/tasks/tasks/ifbench/evaluation_lib.py

@@ -20,7 +20,7 @@
 import json
 from typing import Dict, Optional, Union
 
-import lighteval.tasks.extended.ifbench.instructions_registry as instructions_registry
+import lighteval.tasks.tasks.ifbench.instructions_registry as instructions_registry
 
 
 @dataclasses.dataclass

src/lighteval/tasks/tasks/ifbench/instructions.py

@@ -35,7 +35,7 @@
 if is_package_available("spacy"):
     import spacy
 
-import lighteval.tasks.extended.ifeval.instructions_utils as instructions_util
+import lighteval.tasks.tasks.ifeval.instructions_utils as instructions_util
 
 
 logger = logging.getLogger(__name__)

src/lighteval/tasks/tasks/ifbench/instructions_registry.py

@@ -14,7 +14,7 @@
 
 """Registry of all instructions."""
 
-import lighteval.tasks.extended.ifbench.instructions as instructions
+import lighteval.tasks.tasks.ifbench.instructions as instructions
 
 
 INSTRUCTION_DICT = {

src/lighteval/tasks/tasks/ifbench/main.py

@@ -30,9 +30,9 @@
     SampleLevelMetricGrouping,
 )
 from lighteval.models.model_output import ModelResponse
-from lighteval.tasks.extended.ifbench import evaluation_lib
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc, SamplingMethod
+from lighteval.tasks.tasks.ifbench import evaluation_lib
 
 
 def ifbench_prompt(line, task_name: str = ""):

src/lighteval/tasks/tasks/ifeval/instructions.py

@@ -27,7 +27,7 @@
 if is_package_available("langdetect"):
     import langdetect
 
-import lighteval.tasks.extended.ifeval.instructions_utils as instructions_util
+import lighteval.tasks.tasks.ifeval.instructions_utils as instructions_util
 
 
 logger = logging.getLogger(__name__)

src/lighteval/tasks/tasks/ifeval/instructions_registry.py

@@ -14,7 +14,7 @@
 
 """Registry of all instructions."""
 
-import lighteval.tasks.extended.ifeval.instructions as instructions
+import lighteval.tasks.tasks.ifeval.instructions as instructions
 
 
 _KEYWORD = "keywords:"

src/lighteval/tasks/tasks/ifeval/main.py

@@ -22,13 +22,12 @@
 
 
 import numpy as np
-from inspect_ai.dataset import Sample
-from inspect_ai.scorer import Score, Target, accuracy, scorer, stderr
-from inspect_ai.solver import TaskState
 
-import lighteval.tasks.extended.ifeval.instructions_registry as instructions_registry
+import lighteval.tasks.tasks.ifeval.instructions_registry as instructions_registry
 from lighteval.metrics.metrics_sample import SampleLevelComputation
-from lighteval.metrics.utils.metric_utils import SampleLevelMetricGrouping
+from lighteval.metrics.utils.metric_utils import (
+    SampleLevelMetricGrouping,
+)
 from lighteval.models.model_output import ModelResponse
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc, SamplingMethod
@@ -37,35 +36,37 @@
 
 # Very specific task where there are no precise outputs but instead we test if the format obeys rules
 @requires("langdetect")
-def ifeval_prompt(record):
-    metadata = {"instruction_id_list": record["instruction_id_list"], "kwargs": record["kwargs"]}
-
-    return Sample(
-        input=record["prompt"],
-        metadata=metadata,
+def ifeval_prompt(line, task_name: str = ""):
+    return Doc(
+        task_name=task_name,
+        query=line["prompt"],
+        choices=[""],
+        gold_index=0,
+        instruction="",
+        specific={"instructions_id_list": line["instruction_id_list"], "kwargs": line["kwargs"]},
     )
 
 
 submetric_names = [
     "prompt_level_strict_acc",
+    "inst_level_strict_acc",
     "prompt_level_loose_acc",
+    "inst_level_loose_acc",
+]
+
+REASONING_TAG_PAIRS = [
+    ("<think>", "</think>"),
 ]
 
 
-@scorer(
-    metrics={
-        "prompt_level_strict_acc": [accuracy(), stderr()],
-        "prompt_level_loose_acc": [accuracy(), stderr()],
-    }
-)
-def ifeval_scorer():
-    async def score(state: TaskState, target: Target):
-        response = state.output.completion
+class IFEvalMetrics(SampleLevelComputation):
+    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict:
+        response = model_response.final_text[0]
 
         # Strict instructions
-        instruction_list = state.metadata["instruction_id_list"]
-        all_kwargs = state.metadata["kwargs"]
-        prompt = state.input
+        instruction_list = doc.specific["instructions_id_list"]
+        all_kwargs = doc.specific["kwargs"]
+        prompt = doc.query
 
         # Loose instructions
         r = response.split("\n")
@@ -116,19 +117,12 @@ async def score(state: TaskState, target: Target):
 
             is_following_list_loose.append(is_following)
 
-        return Score(
-            value={
-                "prompt_level_strict_acc": int(all(is_following_list_strict)),
-                "prompt_level_loose_acc": int(all(is_following_list_loose)),
-            }
-        )
-
-    return score
-
-
-class IFEvalMetrics(SampleLevelComputation):
-    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict:
-        pass
+        return {
+            "prompt_level_strict_acc": int(all(is_following_list_strict)),
+            "inst_level_strict_acc": is_following_list_strict,
+            "prompt_level_loose_acc": int(all(is_following_list_loose)),
+            "inst_level_loose_acc": is_following_list_loose,
+        }
 
 
 @requires("langdetect")
@@ -155,12 +149,17 @@ def agg_inst_level_acc(items):
 ifeval = LightevalTaskConfig(
     name="ifeval",
     prompt_function=ifeval_prompt,
-    dataset_repo="google/IFEval",
-    dataset_subset="default",
-    dataset_split="train",
-    dataset_revision="main",
-    metrics=[],
-    system_prompt="FOLLOW THE INSTRUCTIONS STRICTLY.",
+    suite=["extended"],
+    hf_repo="google/IFEval",
+    hf_subset="default",
+    metrics=[ifeval_metrics],
+    hf_avail_splits=["train"],
+    evaluation_splits=["train"],
+    few_shots_split="train",
+    few_shots_select="random_sampling",
+    generation_size=1280,
+    stop_sequence=[],  # no stop sequence, will use eot token
+    version="0.1",
 )
 
 

src/lighteval/tasks/tasks/lcb/main.py

@@ -38,13 +38,13 @@
 from lighteval.metrics.metrics import Metrics, SampleLevelMetric
 from lighteval.metrics.metrics_sample import SampleLevelComputation
 from lighteval.models.model_output import ModelResponse
-from lighteval.tasks.extended.lcb.codegen_metrics import (
+from lighteval.tasks.lighteval_task import Doc, LightevalTaskConfig
+from lighteval.tasks.requests import SamplingMethod
+from lighteval.tasks.tasks.lcb.codegen_metrics import (
     codegen_metrics,
     extract_code,
     translate_private_test_cases,
 )
-from lighteval.tasks.lighteval_task import Doc, LightevalTaskConfig
-from lighteval.tasks.requests import SamplingMethod
 
 
 def prepare_prompt(line: dict[str, Any]) -> str:

src/lighteval/tasks/tasks/mix_eval/judge_prompts.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from lighteval.tasks.extended.mix_eval.prompts import parse_options
+from lighteval.tasks.tasks.mix_eval.prompts import parse_options
 
 
 def flow_judge_for_freeform_template(question, options, answer, gold):