Skip to content

Commit f4b0e27

Browse files
NathanHBNathanHB
committed
add docstring for all multilingual tasks
1 parent 1ed1602 commit f4b0e27

7 files changed

Lines changed: 93 additions & 81 deletions

File tree

‎src/lighteval/tasks/tasks/hle/main.py‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,3 @@
1-
import logging
2-
import math
3-
from typing import List, Literal
4-
5-
import numpy as np
6-
from aenum import extend_enum
7-
from pydantic import BaseModel
8-
9-
from lighteval.metrics.metrics import Metrics
10-
from lighteval.metrics.metrics_sample import JudgeLLM
11-
from lighteval.metrics.utils.metric_utils import CorpusLevelMetricGrouping
12-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
13-
from lighteval.tasks.requests import Doc, SamplingMethod
14-
15-
161
"""
172
abstract:
183
Humanity's Last Exam (HLE) is a global collaborative effort, with questions from
@@ -30,6 +15,20 @@
3015
https://arxiv.org/abs/2501.14249
3116
"""
3217

18+
import logging
19+
import math
20+
from typing import List, Literal
21+
22+
import numpy as np
23+
from aenum import extend_enum
24+
from pydantic import BaseModel
25+
26+
from lighteval.metrics.metrics import Metrics
27+
from lighteval.metrics.metrics_sample import JudgeLLM
28+
from lighteval.metrics.utils.metric_utils import CorpusLevelMetricGrouping
29+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
30+
from lighteval.tasks.requests import Doc, SamplingMethod
31+
3332

3433
logger = logging.getLogger(__name__)
3534

@@ -42,8 +41,7 @@ class ExtractedAnswer(BaseModel):
4241
strict: Literal[True] # 100% reliability
4342

4443

45-
"""Adaptation from https://github.com/centerforaisafety/hle/blob/main/hle_eval/run_judge_results.py
46-
"""
44+
# Adaptation from https://github.com/centerforaisafety/hle/blob/main/hle_eval/run_judge_results.py
4745

4846

4947
def get_judge_prompt(question: str, answer: str, gold: str, **kwargs):

‎src/lighteval/tasks/tasks/ifbench/main.py‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,3 @@
1-
import numpy as np
2-
from aenum import extend_enum
3-
4-
from lighteval.metrics.metrics import Metrics
5-
from lighteval.metrics.metrics_sample import SampleLevelComputation
6-
from lighteval.metrics.utils.metric_utils import (
7-
SampleLevelMetricGrouping,
8-
)
9-
from lighteval.models.model_output import ModelResponse
10-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
11-
from lighteval.tasks.requests import Doc, SamplingMethod
12-
from lighteval.tasks.tasks.ifbench import evaluation_lib
13-
14-
151
"""
162
abstract:
173
Challenging benchmark for precise instruction following.
@@ -26,6 +12,19 @@
2612
https://arxiv.org/abs/2507.02833
2713
"""
2814

15+
import numpy as np
16+
from aenum import extend_enum
17+
18+
from lighteval.metrics.metrics import Metrics
19+
from lighteval.metrics.metrics_sample import SampleLevelComputation
20+
from lighteval.metrics.utils.metric_utils import (
21+
SampleLevelMetricGrouping,
22+
)
23+
from lighteval.models.model_output import ModelResponse
24+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
25+
from lighteval.tasks.requests import Doc, SamplingMethod
26+
from lighteval.tasks.tasks.ifbench import evaluation_lib
27+
2928

3029
def ifbench_prompt(line, task_name: str = ""):
3130
return Doc(

‎src/lighteval/tasks/tasks/ifeval/main.py‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,3 @@
1-
import numpy as np
2-
3-
import lighteval.tasks.tasks.ifeval.instructions_registry as instructions_registry
4-
from lighteval.metrics.metrics_sample import SampleLevelComputation
5-
from lighteval.metrics.utils.metric_utils import (
6-
SampleLevelMetricGrouping,
7-
)
8-
from lighteval.models.model_output import ModelResponse
9-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
10-
from lighteval.tasks.requests import Doc, SamplingMethod
11-
from lighteval.utils.imports import requires
12-
13-
141
"""
152
abstract:
163
Very specific task where there are no precise outputs but instead we test if the
@@ -26,6 +13,18 @@
2613
https://arxiv.org/abs/2311.07911
2714
"""
2815

16+
import numpy as np
17+
18+
import lighteval.tasks.tasks.ifeval.instructions_registry as instructions_registry
19+
from lighteval.metrics.metrics_sample import SampleLevelComputation
20+
from lighteval.metrics.utils.metric_utils import (
21+
SampleLevelMetricGrouping,
22+
)
23+
from lighteval.models.model_output import ModelResponse
24+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
25+
from lighteval.tasks.requests import Doc, SamplingMethod
26+
from lighteval.utils.imports import requires
27+
2928

3029
# Very specific task where there are no precise outputs but instead we test if the format obeys rules
3130
@requires("langdetect")

‎src/lighteval/tasks/tasks/mix_eval/main.py‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,3 @@
1-
import logging
2-
import re
3-
4-
import numpy as np
5-
6-
from lighteval.metrics.metrics_sample import JudgeLLMMixEval
7-
from lighteval.metrics.utils.metric_utils import SampleLevelMetricGrouping
8-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
9-
from lighteval.tasks.requests import Doc, SamplingMethod
10-
from lighteval.tasks.tasks.mix_eval.judge_prompts import (
11-
flow_judge_for_freeform_template,
12-
flow_judge_for_multichoice_template,
13-
gpt_judge_for_closeended_freeform,
14-
gpt_judge_for_closeended_multiplechoice,
15-
)
16-
from lighteval.tasks.tasks.mix_eval.prompts import construct_prompt_freeform, construct_prompt_multichoice
17-
18-
191
"""
202
abstract:
213
Ground-truth-based dynamic benchmark derived from off-the-shelf benchmark
@@ -34,6 +16,23 @@
3416
https://mixeval.github.io/
3517
"""
3618

19+
import logging
20+
import re
21+
22+
import numpy as np
23+
24+
from lighteval.metrics.metrics_sample import JudgeLLMMixEval
25+
from lighteval.metrics.utils.metric_utils import SampleLevelMetricGrouping
26+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
27+
from lighteval.tasks.requests import Doc, SamplingMethod
28+
from lighteval.tasks.tasks.mix_eval.judge_prompts import (
29+
flow_judge_for_freeform_template,
30+
flow_judge_for_multichoice_template,
31+
gpt_judge_for_closeended_freeform,
32+
gpt_judge_for_closeended_multiplechoice,
33+
)
34+
from lighteval.tasks.tasks.mix_eval.prompts import construct_prompt_freeform, construct_prompt_multichoice
35+
3736

3837
logger = logging.getLogger(__name__)
3938

‎src/lighteval/tasks/tasks/mt_bench/main.py‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,32 @@
1-
# ruff: noqa: F405, F403, F401, I001
2-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
3-
from lighteval.tasks.requests import Doc, SamplingMethod
1+
"""
2+
abstract:
3+
MT-Bench is a multi-turn conversational benchmark for evaluating language
4+
models. It consists of 80 high-quality multi-turn questions across 8 common
5+
categories (writing, roleplay, reasoning, math, coding, extraction, STEM,
6+
humanities). Model responses are evaluated by a judge LLM.
7+
8+
languages:
9+
en
10+
11+
tags:
12+
conversational, generation, multi-turn
13+
14+
paper:
15+
https://arxiv.org/abs/2402.14762
16+
"""
17+
18+
import re
19+
20+
import numpy as np
21+
422
from lighteval.metrics.metrics_sample import JudgeLLMMTBench
523
from lighteval.metrics.utils.metric_utils import SampleLevelMetricGrouping
24+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
25+
from lighteval.tasks.requests import Doc, SamplingMethod
626
from lighteval.tasks.tasks.mt_bench.judge_prompt_templates import (
727
flow_judge_prompt_mt_bench_with_ref,
828
flow_judge_prompt_mt_bench_without_ref,
929
)
10-
import re
11-
import numpy as np
1230

1331

1432
def mt_bench_prompt(line, task_name: str = ""):

‎src/lighteval/tasks/tasks/olympiade_bench/main.py‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,3 @@
1-
import numpy as np
2-
3-
from lighteval.metrics.dynamic_metrics import (
4-
ExprExtractionConfig,
5-
LatexExtractionConfig,
6-
MultilingualExtractiveMatchMetric,
7-
)
8-
from lighteval.metrics.metrics import SampleLevelMetric, SamplingMethod
9-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
10-
from lighteval.tasks.requests import Doc
11-
from lighteval.utils.language import Language
12-
13-
141
"""
152
abstract:
163
OlympiadBench is a benchmark for evaluating the performance of language models
@@ -26,6 +13,19 @@
2613
https://arxiv.org/abs/2402.14008
2714
"""
2815

16+
import numpy as np
17+
18+
from lighteval.metrics.dynamic_metrics import (
19+
ExprExtractionConfig,
20+
LatexExtractionConfig,
21+
MultilingualExtractiveMatchMetric,
22+
)
23+
from lighteval.metrics.metrics import SampleLevelMetric, SamplingMethod
24+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
25+
from lighteval.tasks.requests import Doc
26+
from lighteval.utils.language import Language
27+
28+
2929
chinese_answer_type_dict = {"Numerical": "数值", "Expression": "表达式", "Equation": "方程", "Interval": "区间"}
3030
english_answer_type_dict = {
3131
"Numerical": "a numerical value",

‎src/lighteval/tasks/tasks/tiny_benchmarks/main.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# ruff: noqa: F405, F403, F401
21
"""
32
abstract:
43
TinyBenchmarks is a benchmark for evaluating the performance of language models

0 commit comments

Comments
 (0)