diff --git a/src/lighteval/tasks/registry.py b/src/lighteval/tasks/registry.py
index 95914991c..cae642ff1 100644
--- a/src/lighteval/tasks/registry.py
+++ b/src/lighteval/tasks/registry.py
@@ -325,7 +325,7 @@ def _update_task_configs(self) -> dict[str, LightevalTaskConfig]:  # noqa: C901
                                 f"was not correctly parametrized. Forgot to set '{attribute}'."
                             )
 
-                task_to_configs[expanded_task].append(config)
+                task_to_configs[expanded_task + f"|{few_shot}"].append(config)
 
         return task_to_configs
 
diff --git a/src/lighteval/utils/cache_management.py b/src/lighteval/utils/cache_management.py
index 2059d2843..590854955 100644
--- a/src/lighteval/utils/cache_management.py
+++ b/src/lighteval/utils/cache_management.py
@@ -163,8 +163,10 @@ def _get_task_hash(self, full_task_name: str) -> str:
                 "The task registry was not provided to the cache config. We can't test if the current task has the same hash as the saved tasks."
             )
             return "NO_HASH"
-        task_suite, task_name, _ = full_task_name.split("|")
-        task_configs: list[LightevalTaskConfig] = sorted(self.registry.task_to_configs[f"{task_suite}|{task_name}"])
+        task_suite, task_name, few_shot = full_task_name.split("|")
+        task_configs: list[LightevalTaskConfig] = sorted(
+            self.registry.task_to_configs[f"{task_suite}|{task_name}|{few_shot}"]
+        )
         config_str = "|".join([task_config.__str__(lite=True) for task_config in task_configs])
         return hashlib.sha256(config_str.encode()).hexdigest()[:16]