Skip to content

Commit 4c38951

Browse files
NathanHBNathanHB
committed
run precomit hook
1 parent 6cc3c04 commit 4c38951

95 files changed

Lines changed: 327 additions & 2404 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ Lighteval supports **7,000+ evaluation tasks** across multiple domains and langu
6262

6363
### 🌍 **Multilingual Evaluation**
6464
- **Cross-lingual**: XTREME, Flores200 (200 languages), XCOPA, XQuAD
65-
- **Language-specific**:
65+
- **Language-specific**:
6666
- **Arabic**: ArabicMMLU
6767
- **Filipino**: FilBench
6868
- **French**: IFEval-fr, GPQA-fr, BAC-fr

‎src/lighteval/main_inspect.py‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -53,24 +53,44 @@ def get_task(lighteval_task_config: LightevalTaskConfig):
5353

5454

5555
model_args = {
56-
"max-tokens", "system-message", "temperature", "top-p", "top-k", "frequence-penalty",
57-
"presence-penalty", "logit-bias", "seed", "stop-seqs", "num-choices", "best-of", "log-probs", "top-logprobs",
58-
"cache-prompt", "reasoning-effort", "reasoning-tokens", "reasoning-history", "response-format", "parallel-tool-calls", "max-tool-output",
59-
"internal-tools", "max-retries", "timeout"
56+
"max-tokens",
57+
"system-message",
58+
"temperature",
59+
"top-p",
60+
"top-k",
61+
"frequence-penalty",
62+
"presence-penalty",
63+
"logit-bias",
64+
"seed",
65+
"stop-seqs",
66+
"num-choices",
67+
"best-of",
68+
"log-probs",
69+
"top-logprobs",
70+
"cache-prompt",
71+
"reasoning-effort",
72+
"reasoning-tokens",
73+
"reasoning-history",
74+
"response-format",
75+
"parallel-tool-calls",
76+
"max-tool-output",
77+
"internal-tools",
78+
"max-retries",
79+
"timeout",
6080
}
6181

6282

6383
def main():
6484
MODEL = ["openai/gpt-4o"]
6585
all_tasks = [
66-
#default_tasks.gsm8k_lighteval,
67-
#default_tasks.aime25,
68-
#default_tasks.aime24,
69-
#default_tasks.math_500,
86+
# default_tasks.gsm8k_lighteval,
87+
# default_tasks.aime25,
88+
# default_tasks.aime24,
89+
# default_tasks.math_500,
7090
default_tasks.gsm_plus,
71-
#default_tasks.gpqa_diamond,
72-
#default_tasks.gpqa_extended,
73-
#default_tasks.gpqa_main,
91+
# default_tasks.gpqa_diamond,
92+
# default_tasks.gpqa_extended,
93+
# default_tasks.gpqa_main,
7494
] # default_tasksifeval]
7595
all_tasks = [get_task(task) for task in all_tasks]
7696

‎src/lighteval/metrics/metrics.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@
2525

2626
import numpy as np
2727
from aenum import Enum
28-
from inspect_ai.scorer import Score, Target, accuracy, exact, scorer, stderr
29-
from inspect_ai.solver import TaskState
3028

3129
from lighteval.metrics.dynamic_metrics import MultilingualExtractiveMatchMetric
3230
from lighteval.metrics.harness_compatibility.drop import DropMetrics
@@ -68,8 +66,6 @@
6866
ExprExtractionConfig,
6967
IndicesExtractionConfig,
7068
LatexExtractionConfig,
71-
extract_target_from_pred,
72-
get_extraction_regexes,
7369
)
7470
from lighteval.metrics.utils.metric_utils import (
7571
CorpusLevelMetric,

‎src/lighteval/metrics/utils/extractive_match_utils.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,9 @@ def lazy_indices_regex(
346346

347347
def get_extraction_regexes(
348348
# target_types: Sequence[ExtractionTarget], language: Language, len_choices: int = 1
349-
formatted_doc: Doc, target_types: Sequence[ExtractionTarget], language: Language
349+
formatted_doc: Doc,
350+
target_types: Sequence[ExtractionTarget],
351+
language: Language,
350352
) -> list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]:
351353
extraction_regexes: list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]] = [
352354
(lazy_latex_regex(target_type, language), target_type)

‎src/lighteval/tasks/default_prompts.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -900,6 +900,7 @@ def gpqa(line, task_name: str = None):
900900
# target=LETTER_INDICES[gold_index],
901901
# )
902902

903+
903904
def gpqa_instruct(line, task_name: str = None):
904905
"""Prompt template adapted from simple-evals: https://github.com/openai/simple-evals/blob/83ed7640a7d9cd26849bcb3340125002ef14abbe/common.py#L14"""
905906
gold_index = random.randint(0, 3)
@@ -933,7 +934,7 @@ def gsm_plus(record):
933934
# they are a bit trickier to eval with regular text extraction.
934935

935936
return Sample(
936-
input=record['question'],
937+
input=record["question"],
937938
target=record["answer"],
938939
)
939940

@@ -1442,20 +1443,22 @@ def lsat_qa(line, task_name: str = None):
14421443
)
14431444

14441445

1445-
def math_500(record):
1446+
def math_500(line, task_name: str = None):
14461447
# Prompt template adapted from
14471448
# - simple-evals: https://github.com/openai/simple-evals/blob/6e84f4e2aed6b60f6a0c7b8f06bbbf4bfde72e58/math_eval.py#L17
14481449
# - Llama 3: https://huggingface.co/datasets/meta-llama/Llama-3.2-1B-Instruct-evals/viewer/Llama-3.2-1B-Instruct-evals__math__details?views%5B%5D=llama_32_1b_instruct_evals__math__details
14491450
# Note that it is important to have the final answer in a box for math-verify to work correctly
1450-
14511451
MATH_QUERY_TEMPLATE = """
14521452
Solve the following math problem efficiently and clearly. The last line of your response should be of the following format: 'Therefore, the final answer is: $\\boxed{{ANSWER}}$. I hope it is correct' (without quotes) where ANSWER is just the final number or expression that solves the problem. Think step by step before answering.
14531453
1454+
{Question}
14541455
""".strip()
14551456

1456-
return Sample(
1457-
input=record["problem"],
1458-
target=record["solution"],
1457+
return Doc(
1458+
task_name=task_name,
1459+
query=MATH_QUERY_TEMPLATE.format(Question=line["problem"]),
1460+
gold_index=0,
1461+
choices=[line["solution"]],
14591462
)
14601463

14611464

‎src/lighteval/tasks/multilingual/tasks/acva.py‎

Lines changed: 1 addition & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -20,49 +20,17 @@
2020
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2121
# SOFTWARE.
2222

23-
from functools import partial
24-
from itertools import permutations
25-
26-
from langcodes import Language as LangCodeLanguage
27-
from langcodes import standardize_tag
2823

2924
from lighteval.metrics.dynamic_metrics import (
3025
LogLikelihoodAccMetric,
3126
MultilingualQuasiExactMatchMetric,
32-
MultilingualQuasiF1ScoreMetric,
3327
)
34-
from lighteval.metrics.metrics import Metrics
35-
from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm
36-
from lighteval.tasks.default_prompts import LETTER_INDICES
3728
from lighteval.tasks.lighteval_task import LightevalTaskConfig
38-
from lighteval.tasks.multilingual.adapters import (
39-
agieval_adapter,
40-
alghafa_adapter,
41-
ceval_adapter,
42-
enem_adapter,
43-
get_m3exam_adapter,
44-
get_mkqa_adapter,
45-
sciqa_adapter,
46-
thai_exams_adapter,
47-
winogrand_adapter,
48-
xcodah_adapter,
49-
)
50-
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset
5129
from lighteval.tasks.templates.boolq import get_boolq_prompt_function
52-
from lighteval.tasks.templates.continuation import get_continuation_prompt_function
53-
from lighteval.tasks.templates.copa import get_copa_prompt_function
54-
from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function
55-
from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
56-
from lighteval.tasks.templates.nli import get_nli_prompt_function
57-
from lighteval.tasks.templates.qa import get_qa_prompt_function
58-
from lighteval.tasks.templates.translation import get_translation_prompt_function
5930
from lighteval.tasks.templates.utils.formulation import (
6031
CFFormulation,
61-
HybridFormulation,
62-
MCFFormulation,
6332
)
64-
from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
65-
from lighteval.utils.language import Language, iso_639_3_ind_to_iso_639_3_macro, manage_duplicate_language_codes
33+
from lighteval.utils.language import Language
6634

6735

6836
TASKS_TABLE = []

‎src/lighteval/tasks/multilingual/tasks/afri_mgsm.py‎

Lines changed: 1 addition & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -20,49 +20,13 @@
2020
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2121
# SOFTWARE.
2222

23-
from functools import partial
24-
from itertools import permutations
25-
26-
from langcodes import Language as LangCodeLanguage
27-
from langcodes import standardize_tag
2823

2924
from lighteval.metrics.dynamic_metrics import (
30-
LogLikelihoodAccMetric,
3125
MultilingualQuasiExactMatchMetric,
32-
MultilingualQuasiF1ScoreMetric,
3326
)
34-
from lighteval.metrics.metrics import Metrics
35-
from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm
36-
from lighteval.tasks.default_prompts import LETTER_INDICES
3727
from lighteval.tasks.lighteval_task import LightevalTaskConfig
38-
from lighteval.tasks.multilingual.adapters import (
39-
agieval_adapter,
40-
alghafa_adapter,
41-
ceval_adapter,
42-
enem_adapter,
43-
get_m3exam_adapter,
44-
get_mkqa_adapter,
45-
sciqa_adapter,
46-
thai_exams_adapter,
47-
winogrand_adapter,
48-
xcodah_adapter,
49-
)
50-
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset
51-
from lighteval.tasks.templates.boolq import get_boolq_prompt_function
52-
from lighteval.tasks.templates.continuation import get_continuation_prompt_function
53-
from lighteval.tasks.templates.copa import get_copa_prompt_function
54-
from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function
55-
from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
56-
from lighteval.tasks.templates.nli import get_nli_prompt_function
5728
from lighteval.tasks.templates.qa import get_qa_prompt_function
58-
from lighteval.tasks.templates.translation import get_translation_prompt_function
59-
from lighteval.tasks.templates.utils.formulation import (
60-
CFFormulation,
61-
HybridFormulation,
62-
MCFFormulation,
63-
)
64-
from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
65-
from lighteval.utils.language import Language, iso_639_3_ind_to_iso_639_3_macro, manage_duplicate_language_codes
29+
from lighteval.utils.language import Language
6630

6731

6832
# African MGSM: MGSM for African Languages

‎src/lighteval/tasks/multilingual/tasks/afri_mmlu.py‎

Lines changed: 2 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -21,48 +21,21 @@
2121
# SOFTWARE.
2222

2323
from functools import partial
24-
from itertools import permutations
25-
26-
from langcodes import Language as LangCodeLanguage
27-
from langcodes import standardize_tag
2824

2925
from lighteval.metrics.dynamic_metrics import (
3026
LogLikelihoodAccMetric,
31-
MultilingualQuasiExactMatchMetric,
32-
MultilingualQuasiF1ScoreMetric,
3327
)
34-
from lighteval.metrics.metrics import Metrics
3528
from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm
3629
from lighteval.tasks.default_prompts import LETTER_INDICES
3730
from lighteval.tasks.lighteval_task import LightevalTaskConfig
38-
from lighteval.tasks.multilingual.adapters import (
39-
agieval_adapter,
40-
alghafa_adapter,
41-
ceval_adapter,
42-
enem_adapter,
43-
get_m3exam_adapter,
44-
get_mkqa_adapter,
45-
sciqa_adapter,
46-
thai_exams_adapter,
47-
winogrand_adapter,
48-
xcodah_adapter,
49-
)
50-
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset
51-
from lighteval.tasks.templates.boolq import get_boolq_prompt_function
52-
from lighteval.tasks.templates.continuation import get_continuation_prompt_function
53-
from lighteval.tasks.templates.copa import get_copa_prompt_function
54-
from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function
31+
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation
5532
from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
56-
from lighteval.tasks.templates.nli import get_nli_prompt_function
57-
from lighteval.tasks.templates.qa import get_qa_prompt_function
58-
from lighteval.tasks.templates.translation import get_translation_prompt_function
5933
from lighteval.tasks.templates.utils.formulation import (
6034
CFFormulation,
6135
HybridFormulation,
6236
MCFFormulation,
6337
)
64-
from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
65-
from lighteval.utils.language import Language, iso_639_3_ind_to_iso_639_3_macro, manage_duplicate_language_codes
38+
from lighteval.utils.language import Language
6639

6740

6841
# African MMLU: African Massive Multitask Language Understanding

‎src/lighteval/tasks/multilingual/tasks/afri_xnli.py‎

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -20,49 +20,21 @@
2020
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2121
# SOFTWARE.
2222

23-
from functools import partial
24-
from itertools import permutations
25-
26-
from langcodes import Language as LangCodeLanguage
27-
from langcodes import standardize_tag
2823

2924
from lighteval.metrics.dynamic_metrics import (
3025
LogLikelihoodAccMetric,
31-
MultilingualQuasiExactMatchMetric,
32-
MultilingualQuasiF1ScoreMetric,
3326
)
34-
from lighteval.metrics.metrics import Metrics
35-
from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm
36-
from lighteval.tasks.default_prompts import LETTER_INDICES
27+
from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm
3728
from lighteval.tasks.lighteval_task import LightevalTaskConfig
38-
from lighteval.tasks.multilingual.adapters import (
39-
agieval_adapter,
40-
alghafa_adapter,
41-
ceval_adapter,
42-
enem_adapter,
43-
get_m3exam_adapter,
44-
get_mkqa_adapter,
45-
sciqa_adapter,
46-
thai_exams_adapter,
47-
winogrand_adapter,
48-
xcodah_adapter,
49-
)
50-
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset
51-
from lighteval.tasks.templates.boolq import get_boolq_prompt_function
52-
from lighteval.tasks.templates.continuation import get_continuation_prompt_function
53-
from lighteval.tasks.templates.copa import get_copa_prompt_function
54-
from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function
55-
from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
29+
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation
5630
from lighteval.tasks.templates.nli import get_nli_prompt_function
57-
from lighteval.tasks.templates.qa import get_qa_prompt_function
58-
from lighteval.tasks.templates.translation import get_translation_prompt_function
5931
from lighteval.tasks.templates.utils.formulation import (
6032
CFFormulation,
6133
HybridFormulation,
6234
MCFFormulation,
6335
)
64-
from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
65-
from lighteval.utils.language import Language, iso_639_3_ind_to_iso_639_3_macro, manage_duplicate_language_codes
36+
from lighteval.utils.language import Language
37+
6638

6739
# African XNLI: African XNLI
6840
# From https://arxiv.org/abs/2406.03368. Human translated MMLU.

‎src/lighteval/tasks/multilingual/tasks/arabic_arc.py‎

Lines changed: 3 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,49 +20,23 @@
2020
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2121
# SOFTWARE.
2222

23-
from functools import partial
24-
from itertools import permutations
25-
26-
from langcodes import Language as LangCodeLanguage
27-
from langcodes import standardize_tag
2823

2924
from lighteval.metrics.dynamic_metrics import (
3025
LogLikelihoodAccMetric,
31-
MultilingualQuasiExactMatchMetric,
32-
MultilingualQuasiF1ScoreMetric,
3326
)
34-
from lighteval.metrics.metrics import Metrics
35-
from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm
36-
from lighteval.tasks.default_prompts import LETTER_INDICES
27+
from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm
3728
from lighteval.tasks.lighteval_task import LightevalTaskConfig
3829
from lighteval.tasks.multilingual.adapters import (
39-
agieval_adapter,
4030
alghafa_adapter,
41-
ceval_adapter,
42-
enem_adapter,
43-
get_m3exam_adapter,
44-
get_mkqa_adapter,
45-
sciqa_adapter,
46-
thai_exams_adapter,
47-
winogrand_adapter,
48-
xcodah_adapter,
4931
)
50-
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset
51-
from lighteval.tasks.templates.boolq import get_boolq_prompt_function
52-
from lighteval.tasks.templates.continuation import get_continuation_prompt_function
53-
from lighteval.tasks.templates.copa import get_copa_prompt_function
54-
from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function
32+
from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation
5533
from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
56-
from lighteval.tasks.templates.nli import get_nli_prompt_function
57-
from lighteval.tasks.templates.qa import get_qa_prompt_function
58-
from lighteval.tasks.templates.translation import get_translation_prompt_function
5934
from lighteval.tasks.templates.utils.formulation import (
6035
CFFormulation,
6136
HybridFormulation,
6237
MCFFormulation,
6338
)
64-
from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
65-
from lighteval.utils.language import Language, iso_639_3_ind_to_iso_639_3_macro, manage_duplicate_language_codes
39+
from lighteval.utils.language import Language
6640

6741

6842
TASKS_TABLE = []

0 commit comments

Comments
 (0)