Skip to content

Commit 1ed1602

Browse files
NathanHBNathanHB
committed
add docstring for all multilingual tasks
1 parent ee081f2 commit 1ed1602

76 files changed

Lines changed: 809 additions & 1681 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/lighteval/tasks/multilingual/tasks/acva.py‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,13 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
Acva multilingual benchmark.
24
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
5+
languages:
6+
arabic
227
8+
tags:
9+
knowledge, multilingual, multiple-choice
10+
"""
2311

2412
from lighteval.metrics.dynamic_metrics import (
2513
LogLikelihoodAccMetric,

‎src/lighteval/tasks/multilingual/tasks/afri_mgsm.py‎

Lines changed: 11 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,17 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
African MGSM: MGSM for African Languages
24
3-
# Copyright (c) 2024 The HuggingFace Team
5+
languages:
6+
amharic, ewe, french, hausa, igbo, kinyarwanda, lingala, luganda, oromo, shona,
7+
sotho, swahili, twi, wolof, xhosa, yoruba, zulu
48
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
9+
tags:
10+
math, multilingual, reasoning
2211
12+
paper:
13+
https://arxiv.org/abs/2406.03368.
14+
"""
2315

2416
from lighteval.metrics.dynamic_metrics import (
2517
MultilingualQuasiExactMatchMetric,
@@ -29,9 +21,6 @@
2921
from lighteval.utils.language import Language
3022

3123

32-
# African MGSM: MGSM for African Languages
33-
# From https://arxiv.org/abs/2406.03368. Human translated MGSM.
34-
3524
TASKS_TABLE = []
3625

3726

‎src/lighteval/tasks/multilingual/tasks/afri_mmlu.py‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,17 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
African MMLU: African Massive Multitask Language Understanding
24
3-
# Copyright (c) 2024 The HuggingFace Team
5+
languages:
6+
amharic, ewe, french, hausa, igbo, kinyarwanda, lingala, luganda, oromo, shona,
7+
sotho, swahili, twi, wolof, xhosa, yoruba, zulu
48
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
9+
tags:
10+
knowledge, multilingual, multiple-choice
1111
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
12+
paper:
13+
https://arxiv.org/abs/2406.03368.
14+
"""
2215

2316
from functools import partial
2417

@@ -38,9 +31,6 @@
3831
from lighteval.utils.language import Language
3932

4033

41-
# African MMLU: African Massive Multitask Language Understanding
42-
# From https://arxiv.org/abs/2406.03368. Human translated MMLU.
43-
4434
TASKS_TABLE = []
4535

4636

‎src/lighteval/tasks/multilingual/tasks/afri_xnli.py‎

Lines changed: 11 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,17 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
African XNLI: African XNLI
24
3-
# Copyright (c) 2024 The HuggingFace Team
5+
languages:
6+
amharic, ewe, french, hausa, igbo, kinyarwanda, lingala, luganda, oromo, shona,
7+
sotho, swahili, twi, wolof, xhosa, yoruba, zulu
48
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
9+
tags:
10+
classification, multilingual, nli
2211
12+
paper:
13+
https://arxiv.org/abs/2406.03368.
14+
"""
2315

2416
from lighteval.metrics.dynamic_metrics import (
2517
LogLikelihoodAccMetric,
@@ -36,9 +28,6 @@
3628
from lighteval.utils.language import Language
3729

3830

39-
# African XNLI: African XNLI
40-
# From https://arxiv.org/abs/2406.03368. Human translated MMLU.
41-
4231
TASKS_TABLE = []
4332

4433

‎src/lighteval/tasks/multilingual/tasks/arabic_arc.py‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,13 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
Arabic Arc multilingual benchmark.
24
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
5+
languages:
6+
arabic
227
8+
tags:
9+
multilingual, multiple-choice, reasoning
10+
"""
2311

2412
from lighteval.metrics.dynamic_metrics import (
2513
LogLikelihoodAccMetric,

‎src/lighteval/tasks/multilingual/tasks/arabic_mmlu.py‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,13 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
Arabic Mmlu multilingual benchmark.
24
3-
# Copyright (c) 2024 The HuggingFace Team
4-
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
5+
languages:
6+
arabic
227
8+
tags:
9+
knowledge, multilingual, multiple-choice
10+
"""
2311

2412
from lighteval.metrics.dynamic_metrics import (
2513
LogLikelihoodAccMetric,

‎src/lighteval/tasks/multilingual/tasks/arcd.py‎

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,16 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
ARCD: Arabic Reading Comprehension Dataset.
24
3-
# Copyright (c) 2024 The HuggingFace Team
5+
languages:
6+
arabic
47
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
8+
tags:
9+
multilingual, multiple-choice, qa, reasoning
2210
11+
paper:
12+
https://arxiv.org/pdf/1906.05394
13+
"""
2314

2415
from lighteval.metrics.dynamic_metrics import (
2516
MultilingualQuasiExactMatchMetric,

‎src/lighteval/tasks/multilingual/tasks/belebele.py‎

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,19 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
Belebele: A large-scale reading comprehension dataset covering 122 languages.
24
3-
# Copyright (c) 2024 The HuggingFace Team
5+
languages:
6+
arabic, armenian, bengali, cyrillic, devanagari, ethiopic, georgian, greek,
7+
gujarati, gurmukhi, chinese (simplified), chinese (traditional), hangul, hebrew,
8+
japanese, khmer, kannada, lao, latin, malayalam, myanmar, odia, sinhala, tamil,
9+
telugu, thai, tibetan
410
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
11+
tags:
12+
multilingual, multiple-choice, reading-comprehension
2213
14+
paper:
15+
https://arxiv.org/abs/2308.16884
16+
"""
2317

2418
from langcodes import Language as LangCodeLanguage
2519

@@ -38,9 +32,6 @@
3832
from lighteval.utils.language import iso_639_3_ind_to_iso_639_3_macro
3933

4034

41-
# Belebele: A large-scale reading comprehension dataset covering 122 languages.
42-
# https://arxiv.org/abs/2308.16884
43-
4435
TASKS_TABLE = []
4536

4637

‎src/lighteval/tasks/multilingual/tasks/c3.py‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,17 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
C3: A Chinese Challenge Corpus for Cross-lingual and Cross-modal Tasks Reading
4+
comprehension task part of clue.
25
3-
# Copyright (c) 2024 The HuggingFace Team
6+
languages:
7+
chinese
48
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
9+
tags:
10+
multilingual, multiple-choice, reasoning
2211
12+
paper:
13+
https://arxiv.org/abs/2004.05986
14+
"""
2315

2416
from lighteval.metrics.dynamic_metrics import (
2517
LogLikelihoodAccMetric,

‎src/lighteval/tasks/multilingual/tasks/ceval.py‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,13 @@
1-
# MIT License
1+
"""
2+
abstract:
3+
Ceval multilingual benchmark.
24
3-
# Copyright (c) 2024 The HuggingFace Team
5+
languages:
6+
chinese
47
5-
# Permission is hereby granted, free of charge, to any person obtaining a copy
6-
# of this software and associated documentation files (the "Software"), to deal
7-
# in the Software without restriction, including without limitation the rights
8-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9-
# copies of the Software, and to permit persons to whom the Software is
10-
# furnished to do so, subject to the following conditions:
11-
12-
# The above copyright notice and this permission notice shall be included in all
13-
# copies or substantial portions of the Software.
14-
15-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21-
# SOFTWARE.
8+
tags:
9+
knowledge, multilingual, multiple-choice
10+
"""
2211

2312
from functools import partial
2413

0 commit comments

Comments
 (0)