fix registry mockup

Author: clefourrier

src/lighteval/pipeline.py

@@ -245,7 +245,7 @@ def _init_tasks_and_requests(self, tasks: str):
         # The registry contains all the potential tasks
         registry = Registry(tasks=tasks, custom_tasks=self.pipeline_parameters.custom_tasks_directory)
 
-        # load the tasks fro the configs and their datasets
+        # load the tasks from the configs and their datasets
         self.tasks_dict: dict[str, LightevalTask] = registry.load_tasks()
         LightevalTask.load_datasets(self.tasks_dict, self.pipeline_parameters.dataset_loading_processes)
         self.documents_dict = {

src/lighteval/tasks/lighteval_task.py

@@ -113,7 +113,7 @@ def __post_init__(self):
         self.evaluation_splits = tuple(self.evaluation_splits)
         self.suite = tuple(self.suite)
         self.stop_sequence = self.stop_sequence if self.stop_sequence is not None else ()
-        self.full_name = f"{self.name}|{self.num_fewshots}"
+        self.full_name = f"{self.name}|{self.num_fewshots}"  # todo clefourrier: this is likely incorrect
 
     def print(self):
         md_writer = MarkdownTableWriter()

tests/pipeline/test_reasoning_tags.py

@@ -61,11 +61,13 @@ def setUp(self):
             stop_sequence=["\n"],
             num_fewshots=0,
         )
+        self.input_task_name = "test|test_reasoning_task|0"
+        self.task_config_name = self.task_config.full_name
 
         # Create test documents with reasoning tags in expected responses
         self.test_docs = [
             Doc(
-                task_name="test|test_reasoning_task|0",
+                task_name=self.input_task_name,
                 query="What is 2+2?",
                 choices=["4"],
                 gold_index=[0],
@@ -77,7 +79,7 @@ def setUp(self):
         # Mock dataset
         self.mock_dataset = {"test": self.test_docs}
 
-    def _mock_task_registry(self, task_config, task_docs, responses_with_reasoning_tags):
+    def _mock_task_registry(self, input_task_name, task_config, task_docs, responses_with_reasoning_tags):
         """Create a fake registry for testing."""
 
         class FakeTask(LightevalTask):
@@ -96,13 +98,12 @@ class FakeRegistry(Registry):
             def __init__(
                 self, tasks: Optional[str] = None, custom_tasks: Optional[Union[str, Path, ModuleType]] = None
             ):
-                super().__init__(tasks=tasks, custom_tasks=custom_tasks)
+                self.tasks_list = [input_task_name]
+                # suite_name, task_name, few_shot = input_task_name.split("|")
+                self.task_to_configs = {input_task_name: [task_config]}
 
-            def get_tasks_configs(self, task: str):
-                return [task_config]
-
-            def get_tasks_from_configs(self, tasks_configs):
-                return {f"{task_config.suite[0]}|{task_config.full_name}": FakeTask(task_config)}
+            def load_tasks(self):
+                return {input_task_name: FakeTask(config=task_config)}
 
         # Create a DummyModel that returns responses with reasoning tags
         class TestDummyModel(DummyModel):
@@ -124,7 +125,7 @@ def test_remove_reasoning_tags_enabled(self):
         ]
 
         FakeRegistry, TestDummyModel = self._mock_task_registry(
-            self.task_config, self.test_docs, responses_with_reasoning
+            self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning
         )
 
         # Initialize accelerator if available
@@ -170,7 +171,7 @@ def test_remove_reasoning_tags_enabled_tags_as_string(self):
         ]
 
         FakeRegistry, TestDummyModel = self._mock_task_registry(
-            self.task_config, self.test_docs, responses_with_reasoning
+            self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning
         )
 
         # Initialize accelerator if available
@@ -216,7 +217,7 @@ def test_remove_reasoning_tags_enabled_default_tags(self):
         ]
 
         FakeRegistry, TestDummyModel = self._mock_task_registry(
-            self.task_config, self.test_docs, responses_with_reasoning
+            self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning
         )
 
         # Initialize accelerator if available
@@ -259,7 +260,7 @@ def test_remove_reasoning_tags_disabled(self):
         ]
 
         FakeRegistry, TestDummyModel = self._mock_task_registry(
-            self.task_config, self.test_docs, responses_with_reasoning
+            self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning
         )
 
         # Initialize accelerator if available
@@ -305,7 +306,7 @@ def test_custom_reasoning_tags(self):
         ]
 
         FakeRegistry, TestDummyModel = self._mock_task_registry(
-            self.task_config, self.test_docs, responses_with_reasoning
+            self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning
         )
 
         # Initialize accelerator if available
@@ -351,7 +352,7 @@ def test_multiple_reasoning_tags(self):
         ]
 
         FakeRegistry, TestDummyModel = self._mock_task_registry(
-            self.task_config, self.test_docs, responses_with_reasoning
+            self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning
         )
 
         # Initialize accelerator if available

tests/tasks/test_registry.py

@@ -95,7 +95,7 @@ def test_superset_with_subset_task():
     registry = Registry(tasks="original|mmlu|3,original|mmlu:abstract_algebra|5")
 
     # We have all mmlu tasks
-    assert registry.tasks_list == ["original|mmlu|3", "original|mmlu:abstract_algebra|5"]
+    assert set(registry.tasks_list) == {"original|mmlu|3", "original|mmlu:abstract_algebra|5"}
     assert len(registry.task_to_configs.keys()) == 57
 
     task_info: list[LightevalTaskConfig] = registry.task_to_configs["original|mmlu:abstract_algebra"]

tests/utils.py

@@ -98,6 +98,7 @@ def fake_evaluate_task(
     # Mock the Registry.get_task_dict method
 
     task_name = f"{lighteval_task.suite[0]}|{lighteval_task.name}"
+    task_name_fs = f"{lighteval_task.suite[0]}|{lighteval_task.name}|{n_fewshot}"
 
     task_dict = {task_name: lighteval_task}
     evaluation_tracker = EvaluationTracker(output_dir="outputs")
@@ -106,17 +107,17 @@ def fake_evaluate_task(
 
     class FakeRegistry(Registry):
         def __init__(self, tasks: Optional[str], custom_tasks: Optional[Union[str, Path, ModuleType]] = None):
-            super().__init__(tasks=tasks, custom_tasks=custom_tasks)
+            self.tasks_list = [task_name_fs]
+            self.task_to_configs = {task_name: [lighteval_task.config]}
 
-        def get_task_dict(self, task_names: list[str]):
-            return task_dict
+        def load_tasks(self):
+            return {lighteval_task.config.full_name: lighteval_task}
 
-        def get_tasks_configs(self, task: str):
-            config = lighteval_task.config
-            config.num_fewshots = n_fewshot
-            config.truncate_fewshots = False
-            config.full_name = f"{task_name}|{config.num_fewshots}"
-            return [config]
+        # def get_tasks_configs(self, task: str):
+        #    config = lighteval_task.config
+        #    config.num_fewshots = n_fewshot
+        #    config.full_name = f"{task_name}|{config.num_fewshots}"
+        #    return [config]
 
     # This is due to logger complaining we have no initialised the accelerator
     # It's hard to mock as it's global singleton