diff --git a/src/lighteval/tasks/tasks/hellaswag.py b/src/lighteval/tasks/tasks/hellaswag.py index bb948f749..9e0ae18c1 100644 --- a/src/lighteval/tasks/tasks/hellaswag.py +++ b/src/lighteval/tasks/tasks/hellaswag.py @@ -21,9 +21,12 @@ from string import ascii_uppercase +from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric from lighteval.metrics.metrics import Metrics +from lighteval.metrics.normalizations import LogProbCharNorm from lighteval.tasks.lighteval_task import LightevalTaskConfig from lighteval.tasks.requests import Doc +from lighteval.tasks.templates.hellaswag import hellaswag_preprocess def hellaswag_prompt(line, task_name: str = None): @@ -42,6 +45,16 @@ def hellaswag_prompt(line, task_name: str = None): ) +def hellaswag_harness_prompt(line, task_name: str = None): + ctx = f"{line['ctx_a']} {line['ctx_b'].capitalize()} " + return Doc( + task_name=task_name, + query=hellaswag_preprocess(f"{line['activity_label']}: {ctx}"), + choices=[hellaswag_preprocess(ending) for ending in line["endings"]], + gold_index=int(line["label"]) if line["label"] != "" else -1, + ) + + hellaswag = LightevalTaskConfig( name="hellaswag", prompt_function=hellaswag_prompt, @@ -59,6 +72,25 @@ def hellaswag_prompt(line, task_name: str = None): version=0, ) +hellaswag_harness = LightevalTaskConfig( + name="hellaswag_harness", + prompt_function=hellaswag_harness_prompt, + hf_repo="Rowan/hellaswag", + hf_subset="default", + hf_avail_splits=["train", "test", "validation"], + evaluation_splits=["validation"], + few_shots_split=None, + few_shots_select="random_sampling_from_train", + generation_size=-1, + metrics=[ + LogLikelihoodAccMetric(), + LogLikelihoodAccMetric(normalization=LogProbCharNorm()), + ], + stop_sequence=["\n"], + version=0, +) + TASKS_TABLE = [ hellaswag, + hellaswag_harness, ] diff --git a/src/lighteval/tasks/tasks/piqa.py b/src/lighteval/tasks/tasks/piqa.py index 4aa20719c..116ba3bb0 100644 --- a/src/lighteval/tasks/tasks/piqa.py +++ b/src/lighteval/tasks/tasks/piqa.py @@ -21,7 +21,9 @@ from string import ascii_uppercase +from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric from lighteval.metrics.metrics import Metrics +from lighteval.metrics.normalizations import LogProbCharNorm from lighteval.tasks.lighteval_task import LightevalTaskConfig from lighteval.tasks.requests import Doc @@ -44,6 +46,15 @@ def piqa_prompt(line, task_name: str = None): ) +def piqa_harness_prompt(line, task_name: str = None): + return Doc( + task_name=task_name, + query=f"Question: {line['goal']}\nAnswer:", + choices=[f" {line['sol1']}", f" {line['sol2']}"], + gold_index=int(line["label"]), + ) + + piqa = LightevalTaskConfig( name="piqa", prompt_function=piqa_prompt, @@ -61,6 +72,25 @@ def piqa_prompt(line, task_name: str = None): version=0, ) +piqa_harness = LightevalTaskConfig( + name="piqa_harness", + prompt_function=piqa_harness_prompt, + hf_repo="ybisk/piqa", + hf_subset="plain_text", + hf_avail_splits=["train", "test", "validation"], + evaluation_splits=["validation"], + few_shots_split=None, + few_shots_select=None, + generation_size=-1, + metrics=[ + LogLikelihoodAccMetric(), + LogLikelihoodAccMetric(normalization=LogProbCharNorm(ignore_first_space=True)), + ], + stop_sequence=["\n"], + version=0, +) + TASKS_TABLE = [ piqa, + piqa_harness, ] diff --git a/tests/unit/tasks/test_hellaswag_piqa.py b/tests/unit/tasks/test_hellaswag_piqa.py new file mode 100644 index 000000000..9d42b893a --- /dev/null +++ b/tests/unit/tasks/test_hellaswag_piqa.py @@ -0,0 +1,105 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import pytest + +from lighteval.metrics.normalizations import LogProbCharNorm +from lighteval.tasks.lighteval_task import LightevalTask +from lighteval.tasks.requests import SamplingMethod +from lighteval.tasks.tasks.hellaswag import ( + TASKS_TABLE as HELLASWAG_TASKS_TABLE, +) +from lighteval.tasks.tasks.hellaswag import ( + hellaswag, + hellaswag_harness, + hellaswag_harness_prompt, +) +from lighteval.tasks.tasks.piqa import ( + TASKS_TABLE as PIQA_TASKS_TABLE, +) +from lighteval.tasks.tasks.piqa import ( + piqa, + piqa_harness, + piqa_harness_prompt, +) + + +def test_hellaswag_harness_prompt(): + line = { + "activity_label": "Removing ice", + "ctx_a": "A person grabs a pick.", + "ctx_b": "they chip at the ice", + "endings": ["The ice breaks [title] away.", "The ice grows."], + "label": "0", + } + + doc = hellaswag_harness_prompt(line, "hellaswag_harness") + + assert doc.query == "Removing ice: A person grabs a pick. They chip at the ice " + assert doc.choices == ["The ice breaks. away.", "The ice grows."] + assert doc.gold_index == 0 + + +def test_piqa_harness_prompt(): + line = { + "goal": "Keep a door open", + "sol1": "Use a doorstop", + "sol2": "Lock the door", + "label": 0, + } + + doc = piqa_harness_prompt(line, "piqa_harness") + + assert doc.query == "Question: Keep a door open\nAnswer:" + assert doc.choices == [" Use a doorstop", " Lock the door"] + assert doc.gold_index == 0 + + +@pytest.mark.parametrize( + ("config", "ignore_first_space"), + [ + (hellaswag_harness, False), + (piqa_harness, True), + ], +) +def test_harness_tasks_use_loglikelihood(config, ignore_first_space): + task = LightevalTask(config) + + assert task.sampling_methods == [SamplingMethod.LOGPROBS] + assert task.generation_size == -1 + assert [metric.metric_name for metric in task.metrics] == ["acc", "acc_norm"] + + normalization = task.metrics[1].sample_level_fn.logprob_normalization + assert isinstance(normalization, LogProbCharNorm) + assert normalization.ignore_first_space is ignore_first_space + + +@pytest.mark.parametrize("config", [hellaswag, piqa]) +def test_existing_tasks_remain_generative(config): + task = LightevalTask(config) + + assert task.sampling_methods == [SamplingMethod.GENERATIVE] + + +def test_harness_tasks_are_exported(): + assert {config.name for config in HELLASWAG_TASKS_TABLE} == {"hellaswag", "hellaswag_harness"} + assert {config.name for config in PIQA_TASKS_TABLE} == {"piqa", "piqa_harness"}