Merge branch 'main' into clem-add-ifbench2

Author: clefourrier

src/lighteval/tasks/registry.py

@@ -282,7 +282,13 @@ def _update_task_configs(self) -> dict[str, LightevalTaskConfig]:  # noqa: C901
         for task in self.tasks_list:
             metric_params_dict = {}
             try:
-                suite_name, task_name, few_shot = tuple(task.split("|"))
+                if task.count("|") == 3:
+                    logger.warning(
+                        "Deprecation warning: You provided 4 arguments in your task name, but we no longer support the `truncate_fewshot` option. We will ignore the parameter for now, but it will fail in a couple of versions, so you should change your task name to `suite|task|num_fewshot`."
+                    )
+                    suite_name, task_name, few_shot, _ = tuple(task.split("|"))
+                else:
+                    suite_name, task_name, few_shot = tuple(task.split("|"))
                 if "@" in task_name:
                     split_task_name = task_name.split("@")
                     task_name, metric_params = split_task_name[0], split_task_name[1:]
@@ -402,8 +408,6 @@ def create_task_config_dict(meta_table: list[LightevalTaskConfig] | None = None)
         Args:
             meta_table: meta_table containing tasks
                 configurations. If not provided, it will be loaded from TABLE_PATH.
-            cache_dir: Directory to store cached data. If not
-                provided, the default cache directory will be used.
 
         Returns:
             Dict[str, LightevalTaskConfig]: A dictionary of task names mapped to their corresponding LightevalTaskConfig.

tests/utils.py

@@ -113,12 +113,6 @@ def __init__(self, tasks: Optional[str], custom_tasks: Optional[Union[str, Path,
         def load_tasks(self):
             return {task_name_fs: lighteval_task}
 
-        # def get_tasks_configs(self, task: str):
-        #    config = lighteval_task.config
-        #    config.num_fewshots = n_fewshot
-        #    config.full_name = f"{task_name}|{config.num_fewshots}"
-        #    return [config]
-
     # This is due to logger complaining we have no initialised the accelerator
     # It's hard to mock as it's global singleton
     if is_accelerate_available():