chaange extended suite to lighteval

Author: NathanHB

src/lighteval/tasks/tasks/gsm8k.py

@@ -25,6 +25,16 @@
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 
 
+"""
+GSM8K is a dataset of 8,000+ high-quality, single-step arithmetic word problems.
+
+https://huggingface.co/datasets/openai/gsm8k
+
+languages: en
+fields: math, reasoning
+"""
+
+
 # gsm8k = LightevalTaskConfig_inspect(
 #     name="gsm8k",
 #     prompt_function=prompt.gsm8k,

src/lighteval/tasks/tasks/ifbench/main.py

@@ -104,7 +104,7 @@ def agg_inst_level_acc(items):
 ifbench_test = LightevalTaskConfig(
     name="ifbench_test",
     prompt_function=ifbench_prompt,
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="allenai/IFBench_test",
     hf_subset="default",
     metrics=[ifbench_metrics],
@@ -121,7 +121,7 @@ def agg_inst_level_acc(items):
 ifbench_multiturn = LightevalTaskConfig(
     name="ifbench_multiturn",
     prompt_function=ifbench_prompt,
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="allenai/IFBench_multi-turn",
     hf_subset="default",
     metrics=[ifbench_metrics],

src/lighteval/tasks/tasks/ifeval/main.py

@@ -149,7 +149,7 @@ def agg_inst_level_acc(items):
 ifeval = LightevalTaskConfig(
     name="ifeval",
     prompt_function=ifeval_prompt,
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="google/IFEval",
     hf_subset="default",
     metrics=[ifeval_metrics],

src/lighteval/tasks/tasks/lcb/main.py

@@ -154,7 +154,7 @@ def compute(self, model_response: ModelResponse, doc: Doc, **kwargs) -> dict:
     name = "lcb:codegeneration" if subset == "v4_v5" else f"lcb:codegeneration_{subset}"
     task = LightevalTaskConfig(
         name=name,
-        suite=["extended"],
+        suite=["lighteval"],
         prompt_function=lcb_codegeneration_prompt_fn,
         hf_repo="lighteval/code_generation_lite",
         hf_subset=subset,  # https://github.com/LiveCodeBench/LiveCodeBench/tree/main?tab=readme-ov-file#dataset-versions

src/lighteval/tasks/tasks/mix_eval/main.py

@@ -174,7 +174,7 @@ def mean_dv_5(x):
 mixeval_freeform_easy = LightevalTaskConfig(
     name="mixeval_easy:freeform",
     prompt_function=mixeval_freeform_prompt,
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="MixEval/MixEval",
     hf_subset="MixEval",
     metrics=[llm_judge_mixeval_freeform_flow_judge, llm_judge_mixeval_freeform_gpt_judge],
@@ -191,7 +191,7 @@ def mean_dv_5(x):
 mixeval_multichoice_easy = LightevalTaskConfig(
     name="mixeval_easy:multichoice",
     prompt_function=mixeval_multichoice_prompt,
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="MixEval/MixEval",
     hf_subset="MixEval",
     metrics=[llm_judge_mixeval_multichoice_flow_judge, llm_judge_mixeval_multichoice_gpt_judge],
@@ -207,7 +207,7 @@ def mean_dv_5(x):
 mixeval_freeform_hard = LightevalTaskConfig(
     name="mixeval_hard:freeform",
     prompt_function=mixeval_freeform_prompt,
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="MixEval/MixEval",
     hf_subset="MixEval_Hard",
     metrics=[llm_judge_mixeval_freeform_flow_judge, llm_judge_mixeval_freeform_gpt_judge],
@@ -224,7 +224,7 @@ def mean_dv_5(x):
 mixeval_multichoice_hard = LightevalTaskConfig(
     name="mixeval_hard:multichoice",
     prompt_function=mixeval_multichoice_prompt,
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="MixEval/MixEval",
     hf_subset="MixEval_Hard",
     metrics=[llm_judge_mixeval_multichoice_flow_judge, llm_judge_mixeval_multichoice_gpt_judge],

src/lighteval/tasks/tasks/mt_bench/main.py

@@ -80,7 +80,7 @@ def flow_judge_mt_bench_prompt(question, answer, options, gold):
 task = LightevalTaskConfig(
     name="mt_bench",
     prompt_function=mt_bench_prompt,  # must be defined in the file or imported from src/lighteval/tasks/tasks_prompt_formatting.py
-    suite=["extended"],
+    suite=["lighteval"],
     hf_repo="lighteval/mt-bench",
     hf_subset="default",
     hf_avail_splits=["train"],

src/lighteval/tasks/tasks/olympiade_bench/main.py

@@ -224,7 +224,7 @@ def olympiad_bench_prompt(line, task_name: str = None):
         LightevalTaskConfig(
             name="olympiad_bench:" + subset,
             prompt_function=olympiad_bench_prompt,
-            suite=["extended"],
+            suite=["lighteval"],
             hf_repo="Hothan/OlympiadBench",
             hf_subset=subset,
             metrics=[metric],

src/lighteval/tasks/tasks/tiny_benchmarks/main.py

@@ -249,7 +249,7 @@ def compute_corpus(self, y_input):
     task = LightevalTaskConfig(
         name=f"tiny:{name}",
         prompt_function=task["prompt"],
-        suite=["extended"],
+        suite=["lighteval"],
         hf_repo=task["dataset"],
         hf_subset=task["subset"],
         hf_avail_splits=task["splits"],