From 547e136c50f979e6b27be5c0edf1e079edcf6455 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Date: Tue, 14 Jul 2026 14:12:52 -0700 Subject: [PATCH] Fix asdiv prompt truncating multi-character gold answers asdiv_prompt set choices to a bare string (line["answer"].split(" (")[0]) rather than a list. Doc.choices is a list[str] and Doc.get_golds() indexes it with the gold index, so a bare string was indexed by character: gold "35" became "3", "128" became "1". ASDiv answers are typically multi-character (multi-digit numbers, decimals), so exact_match scored correct model outputs as wrong. Every sibling generative task wraps the value in a list; asdiv was the only one that did not. Wrap the extracted answer in a list so the full gold answer is preserved. --- src/lighteval/tasks/tasks/asdiv.py | 2 +- tests/unit/tasks/test_asdiv.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/unit/tasks/test_asdiv.py diff --git a/src/lighteval/tasks/tasks/asdiv.py b/src/lighteval/tasks/tasks/asdiv.py index fe0632655..70a6b5d0d 100644 --- a/src/lighteval/tasks/tasks/asdiv.py +++ b/src/lighteval/tasks/tasks/asdiv.py @@ -31,7 +31,7 @@ def asdiv_prompt(line, task_name: str = None): return Doc( task_name=task_name, query=f"{line['body']}\nQuestion:{line['question']}\nAnswer:", - choices=line["answer"].split(" (")[0], + choices=[line["answer"].split(" (")[0]], gold_index=[0], ) diff --git a/tests/unit/tasks/test_asdiv.py b/tests/unit/tasks/test_asdiv.py new file mode 100644 index 000000000..1150757e0 --- /dev/null +++ b/tests/unit/tasks/test_asdiv.py @@ -0,0 +1,15 @@ +from lighteval.tasks.tasks.asdiv import asdiv_prompt + + +def test_asdiv_prompt_keeps_full_gold_answer(): + # asdiv stored choices as a bare string, so Doc.get_golds() indexed the + # string and truncated a multi-character answer to its first character + # (e.g. "35" -> "3"). choices must be a list so the gold answer survives. + line = { + "body": "There are some apples.", + "question": "How many apples are there?", + "answer": "35 (apples)", + } + doc = asdiv_prompt(line) + assert doc.choices == ["35"] + assert doc.get_golds() == ["35"]