Skip to content

Commit 57fe390

Browse files
NathanHBNathanHB
committed
remove license notice and put docstring on top of file
1 parent c980ddb commit 57fe390

96 files changed

Lines changed: 410 additions & 2513 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ line-length = 119
55
[tool.ruff.lint]
66
# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
77
# Never enforce `E501` (line length violations).
8-
ignore = ["E501", "D100", "D101", "D102", "D103", "D104", "D415", "D105", "DOC501", "DOC201"]
8+
ignore = ["E501", "D100", "D101", "D102", "D103", "D104", "D415", "D105", "DOC501", "DOC201", "CPY001"]
99
select = ["C", "E", "F", "I", "W", "CPY", "D417", "DOC"]
1010
preview = true
1111

‎src/lighteval/tasks/tasks/agieval.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
AGIEval is a human-centric benchmark specifically designed to evaluate the
@@ -45,6 +18,11 @@
4518
https://arxiv.org/abs/2304.06364
4619
"""
4720

21+
import lighteval.tasks.default_prompts as prompt
22+
from lighteval.metrics.metrics import Metrics
23+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
24+
25+
4826
agieval_aqua_rat = LightevalTaskConfig(
4927
name="agieval:aqua-rat",
5028
suite=["lighteval"],

‎src/lighteval/tasks/tasks/aime.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
The American Invitational Mathematics Examination (AIME) is a prestigious,
@@ -42,6 +15,11 @@
4215
https://maa.org/aime-thresholds-are-available/
4316
"""
4417

18+
import lighteval.tasks.default_prompts as prompt
19+
from lighteval.metrics.metrics import Metrics
20+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
21+
22+
4523
aime24 = LightevalTaskConfig(
4624
name="aime24",
4725
suite=["lighteval"],

‎src/lighteval/tasks/tasks/anli.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
The Adversarial Natural Language Inference (ANLI) is a new large-scale NLI
@@ -40,6 +13,11 @@
4013
https://arxiv.org/abs/1910.14599
4114
"""
4215

16+
import lighteval.tasks.default_prompts as prompt
17+
from lighteval.metrics.metrics import Metrics
18+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
19+
20+
4321
anli_r1 = LightevalTaskConfig(
4422
name="anli:r1",
4523
suite=["lighteval"],

‎src/lighteval/tasks/tasks/arc.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
7,787 genuine grade-school level, multiple-choice science questions, assembled
@@ -40,6 +13,11 @@
4013
https://arxiv.org/abs/1803.05457
4114
"""
4215

16+
import lighteval.tasks.default_prompts as prompt
17+
from lighteval.metrics.metrics import Metrics
18+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
19+
20+
4321
arc_challenge = LightevalTaskConfig(
4422
name="arc:challenge",
4523
suite=["lighteval"],

‎src/lighteval/tasks/tasks/arc_agi_2.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
ARC-AGI tasks are a series of three to five input and output tasks followed by a
@@ -46,6 +19,11 @@
4619
https://arcprize.org/guide
4720
"""
4821

22+
import lighteval.tasks.default_prompts as prompt
23+
from lighteval.metrics.metrics import Metrics
24+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
25+
26+
4927
arc_agi_2 = LightevalTaskConfig(
5028
name="arc_agi_2",
5129
suite=["lighteval"],

‎src/lighteval/tasks/tasks/arithmetic.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
A small battery of 10 tests that involve asking language models a simple
@@ -37,6 +10,11 @@
3710
https://arxiv.org/abs/2005.14165
3811
"""
3912

13+
import lighteval.tasks.default_prompts as prompt
14+
from lighteval.metrics.metrics import Metrics
15+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
16+
17+
4018
arithmetic_1dc = LightevalTaskConfig(
4119
name="arithmetic:1dc",
4220
suite=["lighteval"],

‎src/lighteval/tasks/tasks/asdiv.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
ASDiv is a dataset for arithmetic reasoning that contains 2,000+ questions
@@ -37,6 +10,11 @@
3710
https://arxiv.org/abs/2410.12853
3811
"""
3912

13+
import lighteval.tasks.default_prompts as prompt
14+
from lighteval.metrics.metrics import Metrics
15+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
16+
17+
4018
asdiv = LightevalTaskConfig(
4119
name="asdiv",
4220
suite=["lighteval"],

‎src/lighteval/tasks/tasks/babi_qa.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
The bAbI benchmark for measuring understanding and reasoning, evaluates reading
@@ -37,6 +10,11 @@
3710
https://arxiv.org/abs/1502.05698
3811
"""
3912

13+
import lighteval.tasks.default_prompts as prompt
14+
from lighteval.metrics.metrics import Metrics
15+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
16+
17+
4018
babi_qa = LightevalTaskConfig(
4119
name="babi_qa",
4220
suite=["lighteval"],

‎src/lighteval/tasks/tasks/bbq.py‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,3 @@
1-
# MIT License
2-
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
22-
23-
import lighteval.tasks.default_prompts as prompt
24-
from lighteval.metrics.metrics import Metrics
25-
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26-
27-
281
"""
292
abstract:
303
The Bias Benchmark for Question Answering (BBQ) for measuring social bias in
@@ -37,6 +10,11 @@
3710
https://arxiv.org/abs/2110.08193
3811
"""
3912

13+
import lighteval.tasks.default_prompts as prompt
14+
from lighteval.metrics.metrics import Metrics
15+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
16+
17+
4018
bbq = LightevalTaskConfig(
4119
name="bbq",
4220
suite=["lighteval"],

0 commit comments

Comments
 (0)