Skip to content

Commit 971e082

Browse files
clefourrierclefourrier
committed
Merge branch 'main' into test_mmlu_redux_2
2 parents 96df0e7 + 7ed2636 commit 971e082

148 files changed

Lines changed: 5399 additions & 4630 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ Here’s a quick command to evaluate using the Accelerate backend:
9090
```shell
9191
lighteval accelerate \
9292
"model_name=gpt2" \
93-
"leaderboard|truthfulqa:mc|0|0"
93+
"leaderboard|truthfulqa:mc|0"
9494
```
9595

9696
## 🙏 Acknowledgements

‎community_tasks/arabic_evals.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -843,6 +843,7 @@ def compute(self, responses: list[str], formatted_docs: list[Doc], **kwargs) ->
843843
Args:
844844
responses (list[str]): The predicted answers
845845
formatted_docs (list[Doc]): Documents containing questions and gold answers
846+
kwargs: Additional keyword arguments (not used)
846847
847848
Returns:
848849
dict[str, float]: Dictionary containing evaluation scores

‎community_tasks/custom_task_classification_grammar_task.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
Example usage:
3333
TGI endpoint evaluation:
3434
```bash
35-
uv run --active --extra litellm --extra tgi lighteval endpoint tgi examples/model_configs/tgi_model.yaml "custom|emotion_classification|0|0"
35+
uv run --active --extra litellm --extra tgi lighteval endpoint tgi examples/model_configs/tgi_model.yaml "custom|emotion_classification|0"
3636
--custom-tasks examples/custom_tasks_templates/custom_task_classification_grammar_task.py
3737
--output-dir results
3838
--save-details
@@ -449,8 +449,8 @@ def get_emotion_classification_grammar() -> TextGenerationInputGrammarType:
449449

450450
print("\nUsage Examples:")
451451
print(
452-
f" TGI: uv run lighteval endpoint tgi config/tgi/tgi.yaml 'custom|{task.name}|0|0' --custom-tasks {__file__} --output-dir results --override-batch-size 1 --use-chat-template --save-details --no-public-run --max-samples 10"
452+
f" TGI: uv run lighteval endpoint tgi config/tgi/tgi.yaml 'custom|{task.name}|0' --custom-tasks {__file__} --output-dir results --override-batch-size 1 --use-chat-template --save-details --no-public-run --max-samples 10"
453453
)
454454
print(
455-
f" Full: uv run lighteval endpoint tgi config/tgi/tgi.yaml 'custom|{task.name}|5|1' --custom-tasks {__file__} --output-dir results --override-batch-size 1 --use-chat-template --save-details --no-public-run"
455+
f" Full: uv run lighteval endpoint tgi config/tgi/tgi.yaml 'custom|{task.name}|5' --custom-tasks {__file__} --output-dir results --override-batch-size 1 --use-chat-template --save-details --no-public-run"
456456
)

‎community_tasks/filipino_evals.py‎

Lines changed: 41 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
from langcodes import Language as LangCodeLanguage
4343
from langcodes import standardize_tag
4444

45-
from lighteval.metrics.dynamic_metrics import loglikelihood_acc_metric
45+
from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric
4646
from lighteval.metrics.metrics import Metrics
4747
from lighteval.metrics.normalizations import (
4848
LogProbCharNorm,
@@ -87,8 +87,8 @@
8787
metrics=get_metrics_for_formulation(
8888
formulation,
8989
[
90-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
91-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
90+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
91+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
9292
],
9393
),
9494
)
@@ -117,8 +117,8 @@
117117
metrics=get_metrics_for_formulation(
118118
formulation,
119119
[
120-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
121-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
120+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
121+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
122122
],
123123
),
124124
)
@@ -154,9 +154,9 @@
154154
metrics=get_metrics_for_formulation(
155155
formulation,
156156
[
157-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
158-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
159-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
157+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
158+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
159+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
160160
],
161161
),
162162
version=0,
@@ -191,9 +191,9 @@
191191
metrics=get_metrics_for_formulation(
192192
formulation,
193193
[
194-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
195-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
196-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
194+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
195+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
196+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
197197
],
198198
),
199199
hf_avail_splits=["test"],
@@ -275,9 +275,9 @@ def filipino_dengue_pfn(line, task_name: str) -> Doc:
275275
metrics=get_metrics_for_formulation(
276276
formulation,
277277
[
278-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
279-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
280-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
278+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
279+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
280+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
281281
],
282282
),
283283
hf_avail_splits=["train", "test"],
@@ -327,9 +327,9 @@ def filipino_dengue_pfn(line, task_name: str) -> Doc:
327327
metrics=get_metrics_for_formulation(
328328
formulation,
329329
[
330-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
331-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
332-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
330+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
331+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
332+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
333333
],
334334
),
335335
)
@@ -360,9 +360,9 @@ def filipino_dengue_pfn(line, task_name: str) -> Doc:
360360
metrics=get_metrics_for_formulation(
361361
formulation,
362362
[
363-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
364-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
365-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
363+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
364+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
365+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
366366
],
367367
),
368368
hf_avail_splits=["test"],
@@ -396,9 +396,9 @@ def filipino_dengue_pfn(line, task_name: str) -> Doc:
396396
hf_subset="default",
397397
evaluation_splits=["tl"],
398398
metrics=[
399-
loglikelihood_acc_metric(normalization=None),
400-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
401-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
399+
LogLikelihoodAccMetric(normalization=None),
400+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
401+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
402402
],
403403
)
404404
for formulation in [HybridFormulation(), MCFFormulation()]
@@ -427,9 +427,9 @@ def filipino_dengue_pfn(line, task_name: str) -> Doc:
427427
metrics=get_metrics_for_formulation(
428428
formulation,
429429
[
430-
loglikelihood_acc_metric(normalization=None),
431-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
432-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
430+
LogLikelihoodAccMetric(normalization=None),
431+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
432+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
433433
],
434434
),
435435
trust_dataset=True,
@@ -509,9 +509,9 @@ def create_sib200_task(language: Language, formulation):
509509
metrics=get_metrics_for_formulation(
510510
formulation,
511511
[
512-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
513-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
514-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
512+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
513+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
514+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
515515
],
516516
),
517517
hf_avail_splits=["test", "validation"],
@@ -565,9 +565,9 @@ def prepare_stingray_semantic_appropriateness(line: dict[str, str]) -> dict[str,
565565
metrics=get_metrics_for_formulation(
566566
formulation,
567567
[
568-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
569-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
570-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
568+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
569+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
570+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
571571
],
572572
),
573573
hf_avail_splits=["test"],
@@ -595,9 +595,9 @@ def prepare_stingray_semantic_appropriateness(line: dict[str, str]) -> dict[str,
595595
metrics=get_metrics_for_formulation(
596596
formulation,
597597
[
598-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
599-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
600-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
598+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
599+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
600+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
601601
],
602602
),
603603
hf_avail_splits=["test"],
@@ -718,9 +718,9 @@ def prepare_stingray_semantic_appropriateness(line: dict[str, str]) -> dict[str,
718718
metrics=get_metrics_for_formulation(
719719
formulation,
720720
[
721-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
722-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
723-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
721+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
722+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
723+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
724724
],
725725
),
726726
version=0,
@@ -762,9 +762,9 @@ def create_universalner_task(language: Language, formulation):
762762
metrics=get_metrics_for_formulation(
763763
formulation,
764764
[
765-
loglikelihood_acc_metric(normalization=LogProbTokenNorm()),
766-
loglikelihood_acc_metric(normalization=LogProbCharNorm()),
767-
loglikelihood_acc_metric(normalization=LogProbPMINorm()),
765+
LogLikelihoodAccMetric(normalization=LogProbTokenNorm()),
766+
LogLikelihoodAccMetric(normalization=LogProbCharNorm()),
767+
LogLikelihoodAccMetric(normalization=LogProbPMINorm()),
768768
],
769769
),
770770
version=0,

0 commit comments

Comments
 (0)