Restore log-likelihood HellaSwag and PIQA variants

src/lighteval/tasks/tasks/hellaswag.py

@@ -21,9 +21,12 @@
 
 from string import ascii_uppercase
 
+from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric
 from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import LogProbCharNorm
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
+from lighteval.tasks.templates.hellaswag import hellaswag_preprocess
 
 
 def hellaswag_prompt(line, task_name: str = None):
@@ -42,6 +45,16 @@ def hellaswag_prompt(line, task_name: str = None):
     )
 
 
+def hellaswag_harness_prompt(line, task_name: str = None):
+    ctx = f"{line['ctx_a']} {line['ctx_b'].capitalize()} "
+    return Doc(
+        task_name=task_name,
+        query=hellaswag_preprocess(f"{line['activity_label']}: {ctx}"),
+        choices=[hellaswag_preprocess(ending) for ending in line["endings"]],
+        gold_index=int(line["label"]) if line["label"] != "" else -1,
+    )
+
+
 hellaswag = LightevalTaskConfig(
     name="hellaswag",
     prompt_function=hellaswag_prompt,
@@ -59,6 +72,25 @@ def hellaswag_prompt(line, task_name: str = None):
     version=0,
 )
 
+hellaswag_harness = LightevalTaskConfig(
+    name="hellaswag_harness",
+    prompt_function=hellaswag_harness_prompt,
+    hf_repo="Rowan/hellaswag",
+    hf_subset="default",
+    hf_avail_splits=["train", "test", "validation"],
+    evaluation_splits=["validation"],
+    few_shots_split=None,
+    few_shots_select="random_sampling_from_train",
+    generation_size=-1,
+    metrics=[
+        LogLikelihoodAccMetric(),
+        LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
+    ],
+    stop_sequence=["\n"],
+    version=0,
+)
+
 TASKS_TABLE = [
     hellaswag,
+    hellaswag_harness,
 ]

src/lighteval/tasks/tasks/piqa.py

@@ -21,7 +21,9 @@
 
 from string import ascii_uppercase
 
+from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric
 from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import LogProbCharNorm
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
@@ -44,6 +46,15 @@ def piqa_prompt(line, task_name: str = None):
     )
 
 
+def piqa_harness_prompt(line, task_name: str = None):
+    return Doc(
+        task_name=task_name,
+        query=f"Question: {line['goal']}\nAnswer:",
+        choices=[f" {line['sol1']}", f" {line['sol2']}"],
+        gold_index=int(line["label"]),
+    )
+
+
 piqa = LightevalTaskConfig(
     name="piqa",
     prompt_function=piqa_prompt,
@@ -61,6 +72,25 @@ def piqa_prompt(line, task_name: str = None):
     version=0,
 )
 
+piqa_harness = LightevalTaskConfig(
+    name="piqa_harness",
+    prompt_function=piqa_harness_prompt,
+    hf_repo="ybisk/piqa",
+    hf_subset="plain_text",
+    hf_avail_splits=["train", "test", "validation"],
+    evaluation_splits=["validation"],
+    few_shots_split=None,
+    few_shots_select=None,
+    generation_size=-1,
+    metrics=[
+        LogLikelihoodAccMetric(),
+        LogLikelihoodAccMetric(normalization=LogProbCharNorm(ignore_first_space=True)),
+    ],
+    stop_sequence=["\n"],
+    version=0,
+)
+
 TASKS_TABLE = [
     piqa,
+    piqa_harness,
 ]

tests/unit/tasks/test_hellaswag_piqa.py

@@ -0,0 +1,105 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import pytest
+
+from lighteval.metrics.normalizations import LogProbCharNorm
+from lighteval.tasks.lighteval_task import LightevalTask
+from lighteval.tasks.requests import SamplingMethod
+from lighteval.tasks.tasks.hellaswag import (
+    TASKS_TABLE as HELLASWAG_TASKS_TABLE,
+)
+from lighteval.tasks.tasks.hellaswag import (
+    hellaswag,
+    hellaswag_harness,
+    hellaswag_harness_prompt,
+)
+from lighteval.tasks.tasks.piqa import (
+    TASKS_TABLE as PIQA_TASKS_TABLE,
+)
+from lighteval.tasks.tasks.piqa import (
+    piqa,
+    piqa_harness,
+    piqa_harness_prompt,
+)
+
+
+def test_hellaswag_harness_prompt():
+    line = {
+        "activity_label": "Removing ice",
+        "ctx_a": "A person grabs a pick.",
+        "ctx_b": "they chip at the ice",
+        "endings": ["The ice breaks [title] away.", "The ice grows."],
+        "label": "0",
+    }
+
+    doc = hellaswag_harness_prompt(line, "hellaswag_harness")
+
+    assert doc.query == "Removing ice: A person grabs a pick. They chip at the ice "
+    assert doc.choices == ["The ice breaks. away.", "The ice grows."]
+    assert doc.gold_index == 0
+
+
+def test_piqa_harness_prompt():
+    line = {
+        "goal": "Keep a door open",
+        "sol1": "Use a doorstop",
+        "sol2": "Lock the door",
+        "label": 0,
+    }
+
+    doc = piqa_harness_prompt(line, "piqa_harness")
+
+    assert doc.query == "Question: Keep a door open\nAnswer:"
+    assert doc.choices == [" Use a doorstop", " Lock the door"]
+    assert doc.gold_index == 0
+
+
+@pytest.mark.parametrize(
+    ("config", "ignore_first_space"),
+    [
+        (hellaswag_harness, False),
+        (piqa_harness, True),
+    ],
+)
+def test_harness_tasks_use_loglikelihood(config, ignore_first_space):
+    task = LightevalTask(config)
+
+    assert task.sampling_methods == [SamplingMethod.LOGPROBS]
+    assert task.generation_size == -1
+    assert [metric.metric_name for metric in task.metrics] == ["acc", "acc_norm"]
+
+    normalization = task.metrics[1].sample_level_fn.logprob_normalization
+    assert isinstance(normalization, LogProbCharNorm)
+    assert normalization.ignore_first_space is ignore_first_space
+
+
+@pytest.mark.parametrize("config", [hellaswag, piqa])
+def test_existing_tasks_remain_generative(config):
+    task = LightevalTask(config)
+
+    assert task.sampling_methods == [SamplingMethod.GENERATIVE]
+
+
+def test_harness_tasks_are_exported():
+    assert {config.name for config in HELLASWAG_TASKS_TABLE} == {"hellaswag", "hellaswag_harness"}
+    assert {config.name for config in PIQA_TASKS_TABLE} == {"piqa", "piqa_harness"}