chaange extended suite to lighteval

Author: NathanHB

src/lighteval/tasks/default_prompts.py

@@ -330,7 +330,7 @@ def bbh_harness(line, task_name: str = None):
     )
 
 
-def bbh_lighteval(line, task_name: str = None):
+def bbh(line, task_name: str = None):
     line = {k: v for k, v in line.items() if v is not None}
 
     query = line.get("task_prefix", "")
@@ -349,16 +349,6 @@ def bbh_lighteval(line, task_name: str = None):
     )
 
 
-def bbh(line, instruction, choices, task_name: str = None):
-    return Doc(
-        task_name=task_name,
-        query=f"{instruction}Q: {line['input']}\nA:",
-        choices=choices,
-        gold_index=choices.index(line["target"]),
-        instruction=instruction,
-    )
-
-
 def bbh_boolean_expressions(line, task_name: str = None):
     instruction = "Evaluate the result of a random Boolean expression.\n\n"
     choices = ["False", "True"]

src/lighteval/tasks/tasks/babi_qa.py

@@ -25,9 +25,13 @@
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 
 
+"""
+helm task
+"""
+
 babi_qa = LightevalTaskConfig(
     name="babi_qa",
-    suite=["helm"],
+    suite=["lighteval"],
     prompt_function=prompt.babi_qa,
     hf_repo="facebook/babi_qa",
     hf_subset="en-valid-qa1",

src/lighteval/tasks/tasks/bigbench_hard.py

@@ -33,7 +33,7 @@
 causal_judgment = LightevalTaskConfig(
     name="bigbench_hard:causal_judgment",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="causal_judgement",
     hf_avail_splits=["train"],
@@ -49,7 +49,7 @@
 date_understanding = LightevalTaskConfig(
     name="bigbench_hard:date_understanding",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="date_understanding",
     hf_avail_splits=["train"],
@@ -65,7 +65,7 @@
 disambiguation_qa = LightevalTaskConfig(
     name="bigbench_hard:disambiguation_qa",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="disambiguation_qa",
     hf_avail_splits=["train"],
@@ -81,7 +81,7 @@
 geometric_shapes = LightevalTaskConfig(
     name="bigbench_hard:geometric_shapes",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="geometric_shapes",
     hf_avail_splits=["train"],
@@ -97,7 +97,7 @@
 logical_deduction_five_objects = LightevalTaskConfig(
     name="bigbench_hard:logical_deduction_five_objects",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="logical_deduction_five_objects",
     hf_avail_splits=["train"],
@@ -113,7 +113,7 @@
 logical_deduction_seven_objects = LightevalTaskConfig(
     name="bigbench_hard:logical_deduction_seven_objects",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="logical_deduction_seven_objects",
     hf_avail_splits=["train"],
@@ -129,7 +129,7 @@
 logical_deduction_three_objects = LightevalTaskConfig(
     name="bigbench_hard:logical_deduction_three_objects",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="logical_deduction_three_objects",
     hf_avail_splits=["train"],
@@ -145,7 +145,7 @@
 movie_recommendation = LightevalTaskConfig(
     name="bigbench_hard:movie_recommendation",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="movie_recommendation",
     hf_avail_splits=["train"],
@@ -161,7 +161,7 @@
 navigate = LightevalTaskConfig(
     name="bigbench_hard:navigate",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="navigate",
     hf_avail_splits=["train"],
@@ -177,7 +177,7 @@
 reasoning_about_colored_objects = LightevalTaskConfig(
     name="bigbench_hard:reasoning_about_colored_objects",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="reasoning_about_colored_objects",
     hf_avail_splits=["train"],
@@ -193,7 +193,7 @@
 ruin_names = LightevalTaskConfig(
     name="bigbench_hard:ruin_names",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="ruin_names",
     hf_avail_splits=["train"],
@@ -209,7 +209,7 @@
 salient_translation_error_detection = LightevalTaskConfig(
     name="bigbench_hard:salient_translation_error_detection",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="salient_translation_error_detection",
     hf_avail_splits=["train"],
@@ -225,7 +225,7 @@
 snarks = LightevalTaskConfig(
     name="bigbench_hard:snarks",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="snarks",
     hf_avail_splits=["train"],
@@ -241,7 +241,7 @@
 sports_understanding = LightevalTaskConfig(
     name="bigbench_hard:sports_understanding",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="sports_understanding",
     hf_avail_splits=["train"],
@@ -257,7 +257,7 @@
 temporal_sequences = LightevalTaskConfig(
     name="bigbench_hard:temporal_sequences",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="temporal_sequences",
     hf_avail_splits=["train"],
@@ -273,7 +273,7 @@
 tracking_shuffled_objects_five_objects = LightevalTaskConfig(
     name="bigbench_hard:tracking_shuffled_objects_five_objects",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="tracking_shuffled_objects_five_objects",
     hf_avail_splits=["train"],
@@ -289,7 +289,7 @@
 tracking_shuffled_objects_seven_objects = LightevalTaskConfig(
     name="bigbench_hard:tracking_shuffled_objects_seven_objects",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="tracking_shuffled_objects_seven_objects",
     hf_avail_splits=["train"],
@@ -305,7 +305,7 @@
 tracking_shuffled_objects_three_objects = LightevalTaskConfig(
     name="bigbench_hard:tracking_shuffled_objects_three_objects",
     suite=["lighteval"],
-    prompt_function=prompt.bbh_lighteval,
+    prompt_function=prompt.bbh,
     hf_repo="lighteval/bbh",
     hf_subset="tracking_shuffled_objects_three_objects",
     hf_avail_splits=["train"],

src/lighteval/tasks/tasks/commonsenseqa.py

@@ -25,9 +25,12 @@
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 
 
+"""
+helm task
+"""
 commonsenseqa = LightevalTaskConfig(
     name="commonsenseqa",
-    suite=["helm", "commonsense_scenario"],
+    suite=["lighteval"],
     prompt_function=prompt.commonsense_qa,
     hf_repo="commonsense_qa",
     hf_subset="default",

src/lighteval/tasks/tasks/covid_dialogue.py

@@ -25,9 +25,12 @@
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 
 
+"""
+helm task
+"""
 covid_dialogue = LightevalTaskConfig(
     name="covid_dialogue",
-    suite=["helm"],
+    suite=["lighteval"],
     prompt_function=prompt.covid_dialogue,
     hf_repo="lighteval/covid_dialogue",
     hf_subset="default",

src/lighteval/tasks/tasks/glue.py

@@ -25,7 +25,7 @@
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 
 
-glue_cola_lighteval = LightevalTaskConfig(
+glue_cola = LightevalTaskConfig(
     name="glue:cola",
     suite=["lighteval"],
     prompt_function=prompt.cola,
@@ -41,7 +41,7 @@
     version=0,
 )
 
-glue_mnli_lighteval = LightevalTaskConfig(
+glue_mnli = LightevalTaskConfig(
     name="glue:mnli",
     suite=["lighteval"],
     prompt_function=prompt.mnli,
@@ -57,7 +57,7 @@
     version=0,
 )
 
-glue_mnli_mismatched_lighteval = LightevalTaskConfig(
+glue_mnli_mismatched = LightevalTaskConfig(
     name="glue:mnli_mismatched",
     suite=["lighteval"],
     prompt_function=prompt.mnli,
@@ -73,7 +73,7 @@
     version=0,
 )
 
-glue_mrpc_lighteval = LightevalTaskConfig(
+glue_mrpc = LightevalTaskConfig(
     name="glue:mrpc",
     suite=["lighteval"],
     prompt_function=prompt.mrpc,
@@ -89,7 +89,7 @@
     version=0,
 )
 
-glue_qnli_lighteval = LightevalTaskConfig(
+glue_qnli = LightevalTaskConfig(
     name="glue:qnli",
     suite=["lighteval"],
     prompt_function=prompt.qnli,
@@ -105,7 +105,7 @@
     version=0,
 )
 
-glue_qqp_lighteval = LightevalTaskConfig(
+glue_qqp = LightevalTaskConfig(
     name="glue:qqp",
     suite=["lighteval"],
     prompt_function=prompt.qqp,
@@ -121,7 +121,7 @@
     version=0,
 )
 
-glue_rte_lighteval = LightevalTaskConfig(
+glue_rte = LightevalTaskConfig(
     name="glue:rte",
     suite=["lighteval"],
     prompt_function=prompt.rte,
@@ -137,7 +137,7 @@
     version=0,
 )
 
-glue_sst2_lighteval = LightevalTaskConfig(
+glue_sst2 = LightevalTaskConfig(
     name="glue:sst2",
     suite=["lighteval"],
     prompt_function=prompt.sst,
@@ -153,7 +153,7 @@
     version=0,
 )
 
-glue_stsb_lighteval = LightevalTaskConfig(
+glue_stsb = LightevalTaskConfig(
     name="glue:stsb",
     suite=["lighteval"],
     prompt_function=prompt.stsb,
@@ -169,7 +169,7 @@
     version=0,
 )
 
-glue_wnli_lighteval = LightevalTaskConfig(
+glue_wnli = LightevalTaskConfig(
     name="glue:wnli",
     suite=["lighteval"],
     prompt_function=prompt.wnli,
@@ -185,7 +185,7 @@
     version=0,
 )
 
-super_glue_boolq_lighteval = LightevalTaskConfig(
+super_glue_boolq = LightevalTaskConfig(
     name="super_glue:boolq",
     suite=["lighteval"],
     prompt_function=prompt.boolq_harness,
@@ -201,7 +201,7 @@
     version=0,
 )
 
-super_glue_cb_lighteval = LightevalTaskConfig(
+super_glue_cb = LightevalTaskConfig(
     name="super_glue:cb",
     suite=["lighteval"],
     prompt_function=prompt.cb,
@@ -217,7 +217,7 @@
     version=0,
 )
 
-super_glue_copa_lighteval = LightevalTaskConfig(
+super_glue_copa = LightevalTaskConfig(
     name="super_glue:copa",
     suite=["lighteval"],
     prompt_function=prompt.copa,
@@ -233,7 +233,7 @@
     version=0,
 )
 
-super_glue_rte_lighteval = LightevalTaskConfig(
+super_glue_rte = LightevalTaskConfig(
     name="super_glue:rte",
     suite=["lighteval"],
     prompt_function=prompt.rte,
@@ -249,7 +249,7 @@
     version=0,
 )
 
-super_glue_multirc_lighteval = LightevalTaskConfig(
+super_glue_multirc = LightevalTaskConfig(
     name="super_glue:multirc",
     suite=["lighteval"],
     prompt_function=prompt.multirc,
@@ -265,7 +265,7 @@
     version=0,
 )
 
-super_glue_wic_lighteval = LightevalTaskConfig(
+super_glue_wic = LightevalTaskConfig(
     name="super_glue:wic",
     suite=["lighteval"],
     prompt_function=prompt.wic,
@@ -281,7 +281,7 @@
     version=0,
 )
 
-super_glue_wsc_lighteval = LightevalTaskConfig(
+super_glue_wsc = LightevalTaskConfig(
     name="super_glue:wsc",
     suite=["lighteval"],
     prompt_function=prompt.wsc,