diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml
index a777cf097..d3b9c9d9b 100644
--- a/docs/source/_toctree.yml
+++ b/docs/source/_toctree.yml
@@ -46,8 +46,6 @@
title: Model Configs
- local: package_reference/pipeline
title: Pipeline
- - local: package_reference/models_outputs
- title: Model's Output
title: Main classes
- local: package_reference/metrics
title: Metrics
@@ -55,4 +53,8 @@
title: Tasks
- local: package_reference/logging
title: Logging
+ - local: package_reference/models_outputs
+ title: ModelResponse
+ - local: package_reference/doc
+ title: Doc
title: Reference
diff --git a/docs/source/adding-a-custom-task.mdx b/docs/source/adding-a-custom-task.mdx
index 448941aa3..1eeebd7ea 100644
--- a/docs/source/adding-a-custom-task.mdx
+++ b/docs/source/adding-a-custom-task.mdx
@@ -1,37 +1,49 @@
# Adding a Custom Task
-To add a new task, first either open an issue, to determine whether it will be
-integrated in the core evaluations of lighteval, in the extended tasks, or the
-community tasks, and add its dataset on the hub.
-
-- Core evaluations are evaluations that only require standard logic in their
- metrics and processing, and that we will add to our test suite to ensure non
- regression through time. They already see high usage in the community.
-- Extended evaluations are evaluations that require custom logic in their
- metrics (complex normalisation, an LLM as a judge, ...), that we added to
- facilitate the life of users. They already see high usage in the community.
-- Community evaluations are submissions by the community of new tasks.
+Lighteval provides a flexible framework for creating custom evaluation tasks. This guide explains how to create and integrate new tasks into the evaluation system.
+
+## Task Categories
+
+Before creating a custom task, consider which category it belongs to:
+
+### Core Evaluations
+Core evaluations are evaluations that only require standard logic in their
+metrics and processing, and that we will add to our test suite to ensure non-regression through time. They already see high usage in the community.
+
+### Extended Evaluations
+Extended evaluations are evaluations that require custom logic in their
+metrics (complex normalization, an LLM as a judge, etc.), that we added to
+facilitate the life of users. They already see high usage in the community.
+
+### Community Evaluations
+Community evaluations are submissions by the community of new tasks.
A popular community evaluation can move to become an extended or core evaluation over time.
> [!TIP]
-> You can find examples of custom tasks in the community_task directory.
+> You can find examples of custom tasks in the [community_tasks](https://github.com/huggingface/lighteval/tree/main/community_tasks) directory.
-## Step by step creation of a custom task
+## Step-by-Step Creation of a Custom Task
> [!WARNING]
-> To contribute your custom metric to the lighteval repo, you would first need
+> To contribute your custom task to the Lighteval repository, you would first need
> to install the required dev dependencies by running `pip install -e .[dev]`
> and then run `pre-commit install` to install the pre-commit hooks.
-First, create a python file under the `community_tasks` directory.
+### Step 1: Create the Task File
+
+First, create a Python file under the `community_tasks` directory.
+
+### Step 2: Define the Prompt Function
You need to define a prompt function that will convert a line from your
dataset to a document to be used for evaluation.
```python
+from lighteval.tasks.requests import Doc
+
# Define as many as you need for your different tasks
-def prompt_fn(line, task_name: str = None):
+def prompt_fn(line: dict, task_name: str):
"""Defines how to go from a dataset line to a doc object.
Follow examples in src/lighteval/tasks/default_prompts.py, or get more info
about what this function should do in the README.
@@ -44,47 +56,68 @@ def prompt_fn(line, task_name: str = None):
)
```
-Then, you need to choose a metric: you can either use an existing one (defined
-in [`lighteval.metrics.metrics.Metrics`]) or [create a custom one](adding-a-new-metric)).
-[//]: # (TODO: Replace lighteval.metrics.metrics.Metrics with ~metrics.metrics.Metrics once its autodoc is added)
+### Step 3: Choose or Create Metrics
+
+You can either use an existing metric (defined in [`lighteval.metrics.metrics.Metrics`]) or [create a custom one](adding-a-new-metric).
+
+#### Using Existing Metrics
+
+```python
+from lighteval.metrics import Metrics
+
+# Use an existing metric
+metric = Metrics.ACCURACY
+```
+
+#### Creating Custom Metrics
```python
+from lighteval.metrics.utils.metric_utils import SampleLevelMetric
+import numpy as np
+
custom_metric = SampleLevelMetric(
metric_name="my_custom_metric_name",
higher_is_better=True,
- category=SamplingMethod.{GENERATIVE,LOGPROBS},
- sample_level_fn=lambda x: x, # how to compute score for one sample
- corpus_level_fn=np.mean, # How to aggregate the samples metrics
+ category="accuracy",
+ sample_level_fn=lambda x: x, # How to compute score for one sample
+ corpus_level_fn=np.mean, # How to aggregate the sample metrics
)
```
-Then, you need to define your task using [`~tasks.lighteval_task.LightevalTaskConfig`].
-You can define a task with or without subsets.
-To define a task with no subsets:
+### Step 4: Define Your Task
+
+You can define a task with or without subsets using [`~tasks.lighteval_task.LightevalTaskConfig`].
+
+#### Simple Task (No Subsets)
```python
-# This is how you create a simple task (like hellaswag) which has one single subset
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+
+# This is how you create a simple task (like HellaSwag) which has one single subset
# attached to it, and one evaluation possible.
task = LightevalTaskConfig(
name="myothertask",
- prompt_function=prompt_fn, # must be defined in the file or imported from src/lighteval/tasks/tasks_prompt_formatting.py
+ prompt_function=prompt_fn, # Must be defined in the file or imported
suite=["community"],
- hf_repo="",
+ hf_repo="your_dataset_repo_on_hf",
hf_subset="default",
- hf_avail_splits=[],
- evaluation_splits=[],
- few_shots_split=None,
- few_shots_select=None,
- metrics=[], # select your metric in Metrics
+ hf_avail_splits=["train", "test"],
+ evaluation_splits=["test"],
+ few_shots_split="train",
+ few_shots_select="random_sampling_from_train",
+ metrics=[metric], # Select your metric in Metrics
+ generation_size=256,
+ stop_sequence=["\n", "Question:"],
)
```
-If you want to create a task with multiple subset, add them to the
+#### Task with Multiple Subsets
+
+If you want to create a task with multiple subsets, add them to the
`SAMPLE_SUBSETS` list and create a task for each subset.
```python
-SAMPLE_SUBSETS = [] # list of all the subsets to use for this eval
-
+SAMPLE_SUBSETS = ["subset1", "subset2", "subset3"] # List of all the subsets to use for this eval
class CustomSubsetTask(LightevalTaskConfig):
def __init__(
@@ -95,33 +128,43 @@ class CustomSubsetTask(LightevalTaskConfig):
super().__init__(
name=name,
hf_subset=hf_subset,
- prompt_function=prompt_fn, # must be defined in the file or imported from src/lighteval/tasks/tasks_prompt_formatting.py
- hf_repo="",
- metric=[custom_metric], # select your metric in Metrics or use your custom_metric
- hf_avail_splits=[],
- evaluation_splits=[],
- few_shots_split=None,
- few_shots_select=None,
+ prompt_function=prompt_fn, # Must be defined in the file or imported
+ hf_repo="your_dataset_name",
+ metrics=[custom_metric], # Select your metric in Metrics or use your custom_metric
+ hf_avail_splits=["train", "test"],
+ evaluation_splits=["test"],
+ few_shots_split="train",
+ few_shots_select="random_sampling_from_train",
suite=["community"],
- generation_size=-1,
- stop_sequence=None,
+ generation_size=256,
+ stop_sequence=["\n", "Question:"],
)
+
SUBSET_TASKS = [CustomSubsetTask(name=f"mytask:{subset}", hf_subset=subset) for subset in SAMPLE_SUBSETS]
```
+### Step 5: Add Tasks to the Table
+
Then you need to add your task to the `TASKS_TABLE` list.
```python
# STORE YOUR EVALS
-# tasks with subset:
+# Tasks with subsets:
TASKS_TABLE = SUBSET_TASKS
-# tasks without subset:
+# Tasks without subsets:
# TASKS_TABLE = [task]
```
-Once your file is created you can then run the evaluation with the following command:
+### Step 6: Creating a requirement file
+
+If your task has requirements, you need to create a `requirement.txt` file with
+only the required dependencies so that anyone can run your task.
+
+## Running Your Custom Task
+
+Once your file is created, you can run the evaluation with the following command:
```bash
lighteval accelerate \
@@ -129,3 +172,19 @@ lighteval accelerate \
"community|{custom_task}|{fewshots}" \
--custom-tasks {path_to_your_custom_task_file}
```
+
+### Example Usage
+
+```bash
+# Run a custom task with zero-shot evaluation
+lighteval accelerate \
+ "model_name=openai-community/gpt2" \
+ "community|myothertask|0|0" \
+ --custom-tasks community_tasks/my_custom_task.py
+
+# Run a custom task with few-shot evaluation
+lighteval accelerate \
+ "model_name=openai-community/gpt2" \
+ "community|myothertask|3|1" \
+ --custom-tasks community_tasks/my_custom_task.py
+```
diff --git a/docs/source/adding-a-new-metric.mdx b/docs/source/adding-a-new-metric.mdx
index 970298a9c..b17caac10 100644
--- a/docs/source/adding-a-new-metric.mdx
+++ b/docs/source/adding-a-new-metric.mdx
@@ -1,39 +1,70 @@
# Adding a New Metric
-First, check if you can use one of the parametrized functions in
+## Before You Start
+
+### Two different types of metrics
+
+There are two types of metrics in Lighteval:
+
+#### Sample-Level Metrics
+- **Purpose**: Evaluate individual samples/predictions
+- **Input**: Takes a `Doc` (document/sample) and `ModelResponse` (model's prediction)
+- **Output**: Returns a float or boolean value for that specific sample
+- **Example**: Checking if a model's answer matches the correct answer for one question
+
+#### Corpus-Level Metrics
+- **Purpose**: Compute final scores across the entire dataset/corpus
+- **Input**: Takes the results from all sample-level evaluations
+- **Output**: Returns a single score representing overall performance
+- **Examples**:
+ - Simple aggregation: Calculating average accuracy across all test samples
+ - Complex metrics: BLEU score where sample-level metric prepares data (tokenization, etc.) and corpus-level metric computes the actual BLEU score
+
+### Check Existing Metrics
+
+First, check if you can use one of the parameterized functions in
[Corpus Metrics](package_reference/metrics#corpus-metrics) or
[Sample Metrics](package_reference/metrics#sample-metrics).
-If not, you can use the `custom_task` system to register your new metric:
+If not, you can use the `custom_task` system to register your new metric.
> [!TIP]
-> To see an example of a custom metric added along with a custom task, look at the IFEval custom task.
-
+> To see an example of a custom metric added along with a custom task, look at the [IFEval custom task](https://github.com/huggingface/lighteval/tree/main/examples/custom_tasks/ifeval).
> [!WARNING]
-> To contribute your custom metric to the lighteval repo, you would first need
+> To contribute your custom metric to the Lighteval repository, you would first need
> to install the required dev dependencies by running `pip install -e .[dev]`
> and then run `pre-commit install` to install the pre-commit hooks.
-- Create a new Python file which should contain the full logic of your metric.
-- The file also needs to start with these imports
+## Creating a Custom Metric
+
+### Step 1: Create the Metric File
+
+Create a new Python file which should contain the full logic of your metric.
+The file also needs to start with these imports:
```python
from aenum import extend_enum
from lighteval.metrics import Metrics
```
-You need to define a sample level metric, all sample level metrics will have the same signature, taking a
+### Step 2: Define the Sample-Level Metric
+
+You need to define a sample-level metric. All sample-level metrics will have the same signature, taking a
[`~lighteval.types.Doc`] and a [`~lighteval.types.ModelResponse`]. The metric should return a float or a
boolean.
+#### Single Metric Example
+
```python
def custom_metric(doc: Doc, model_response: ModelResponse) -> bool:
response = model_response.text[0]
return response == doc.choices[doc.gold_index]
```
-Here the sample level metric only returns one metric, if you want to return multiple metrics per sample you need to return a dictionary with the metrics as keys and the values as values.
+#### Multiple Metrics Example
+
+If you want to return multiple metrics per sample, you need to return a dictionary with the metrics as keys and the values as values:
```python
def custom_metric(doc: Doc, model_response: ModelResponse) -> dict:
@@ -41,7 +72,9 @@ def custom_metric(doc: Doc, model_response: ModelResponse) -> dict:
return {"accuracy": response == doc.choices[doc.gold_index], "other_metric": 0.5}
```
-Then, you can define an aggregation function if needed, a common aggregation function is `np.mean`.
+### Step 3: Define Aggregation Function (Optional)
+
+You can define an aggregation function if needed. A common aggregation function is `np.mean`:
```python
def agg_function(items):
@@ -50,44 +83,82 @@ def agg_function(items):
return score
```
-Finally, you can define your metric. If it's a sample level metric, you can use the following code
+### Step 4: Create the Metric Object
+
+#### Single Metric
+
+If it's a sample-level metric, you can use the following code
with [`~metrics.utils.metric_utils.SampleLevelMetric`]:
```python
my_custom_metric = SampleLevelMetric(
- metric_name={custom_metric_name},
- higher_is_better={either True or False},
- category={SamplingMethod},
+ metric_name="custom_accuracy",
+ higher_is_better=True,
+ category="accuracy",
sample_level_fn=custom_metric,
corpus_level_fn=agg_function,
)
```
+#### Multiple Metrics
+
If your metric defines multiple metrics per sample, you can use the following code
with [`~metrics.utils.metric_utils.SampleLevelMetricGrouping`]:
```python
custom_metric = SampleLevelMetricGrouping(
- metric_name={submetric_names},
- higher_is_better={n: {True or False} for n in submetric_names},
- category={SamplingMethod},
+ metric_name=["accuracy", "response_length", "confidence"],
+ higher_is_better={
+ "accuracy": True,
+ "response_length": False, # Shorter responses might be better
+ "confidence": True
+ },
+ category="comprehensive",
sample_level_fn=custom_metric,
corpus_level_fn={
"accuracy": np.mean,
- "other_metric": agg_function,
+ "response_length": np.mean,
+ "confidence": np.mean,
},
)
```
-To finish, add the following, so that it adds your metric to our metrics list
-when loaded as a module.
+### Step 5: Register the Metric
+
+To finish, add the following code so that it adds your metric to our metrics list
+when loaded as a module:
```python
# Adds the metric to the metric list!
-extend_enum(Metrics, "metric_name", metric_function)
+extend_enum(Metrics, "CUSTOM_ACCURACY", my_custom_metric)
+
if __name__ == "__main__":
print("Imported metric")
```
-You can then give your custom metric to lighteval by using `--custom-tasks
-path_to_your_file` when launching it.
+## Using Your Custom Metric
+
+### With Custom Tasks
+
+You can then give your custom metric to Lighteval by using `--custom-tasks
+path_to_your_file` when launching it after adding it to the task config.
+
+```bash
+lighteval accelerate \
+ "model_name=openai-community/gpt2" \
+ "leaderboard|truthfulqa:mc|0|0" \
+ --custom-tasks path_to_your_metric_file.py
+```
+
+```python
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+
+task = LightevalTaskConfig(
+ name="my_custom_task",
+ suite=["community"],
+ metric=[my_custom_metric], # Use your custom metric here
+ prompt_function=my_prompt_function,
+ hf_repo="my_dataset",
+ evaluation_splits=["test"]
+)
+```
diff --git a/docs/source/available-tasks.mdx b/docs/source/available-tasks.mdx
index 5b80c082e..080ee0fc9 100644
--- a/docs/source/available-tasks.mdx
+++ b/docs/source/available-tasks.mdx
@@ -1,13 +1,26 @@
# Available Tasks
-You can get a list of all the available tasks by running:
+## Discovering Available Tasks
+
+### List All Tasks
+
+You can get a list of all available tasks by running:
```bash
lighteval tasks list
```
-You can also inspect a specific task by running:
+This command will display all tasks organized by their suites (e.g., leaderboard, lighteval, community).
+
+### Inspect Specific Tasks
+
+You can inspect a specific task to see its configuration, metrics, and requirements by running:
```bash
lighteval tasks inspect
```
+
+For example:
+```bash
+lighteval tasks inspect "leaderboard|truthfulqa:mc|0|0"
+```
diff --git a/docs/source/contributing-to-multilingual-evaluations.mdx b/docs/source/contributing-to-multilingual-evaluations.mdx
index 828a9f471..8493d844d 100644
--- a/docs/source/contributing-to-multilingual-evaluations.mdx
+++ b/docs/source/contributing-to-multilingual-evaluations.mdx
@@ -1,58 +1,147 @@
-# Contributing to multilingual evaluations
+# Contributing to Multilingual Evaluations
-## Contributing a small translation
+Lighteval supports multilingual evaluations through a comprehensive system of translation literals and language-adapted templates.
+
+## Contributing Translation Literals
+
+### What Are Translation Literals?
We define 19 `literals`, basic keywords or punctuation signs used when creating evaluation prompts in an automatic manner, such as `yes`, `no`, `because`, etc.
-We welcome translations in your language!
+These literals are essential for:
+- **Consistent prompt formatting** across languages
+- **Automatic prompt generation** for multilingual tasks
+- **Proper localization** of evaluation templates
+
+### How to Contribute Translations
+
+We welcome translations in your language! To contribute:
+
+1. **Open the translation literals file**: [translation_literals.py](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/utils/translation_literals.py)
-To contribute, you'll need to
-1. Open the [translation_literals](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/utils/translation_literals.py) file
-2. Edit the file to add or expand the literal for your language of interest.
+2. **Edit the file** to add or expand the literal for your language of interest
+
+3. **Open a PR** with your modifications
+
+### Translation Literals Structure
```python
- Language.ENGLISH: TranslationLiterals(
- language=Language.ENGLISH,
- question_word="question", # Usage: "Question: How are you?"
- answer="answer", # Usage: "Answer: I am fine"
- confirmation_word="right", # Usage: "He is smart, right?"
- yes="yes", # Usage: "Yes, he is"
- no="no", # Usage: "No, he is not"
- also="also", # Usage: "Also, she is smart."
- cause_word="because", # Usage: "She is smart, because she is tall"
- effect_word="therefore", # Usage: "He is tall therefore he is smart"
- or_word="or", # Usage: "He is tall or small"
- true="true", # Usage: "He is smart, true, false or neither?"
- false="false", # Usage: "He is smart, true, false or neither?"
- neither="neither", # Usage: "He is smart, true, false or neither?"
- # Punctuation and spacing: only adjust if your language uses something different than in English
- full_stop=".",
- comma=",",
- question_mark="?",
- exclamation_mark="!",
- word_space=" ",
- sentence_space=" ",
- colon=":",
- # The first characters of your alphabet used in enumerations, if different from English
- indices=["A", "B", "C", ...]
- )
+Language.ENGLISH: TranslationLiterals(
+ language=Language.ENGLISH,
+ question_word="question", # Usage: "Question: How are you?"
+ answer="answer", # Usage: "Answer: I am fine"
+ confirmation_word="right", # Usage: "He is smart, right?"
+ yes="yes", # Usage: "Yes, he is"
+ no="no", # Usage: "No, he is not"
+ also="also", # Usage: "Also, she is smart."
+ cause_word="because", # Usage: "She is smart, because she is tall"
+ effect_word="therefore", # Usage: "He is tall therefore he is smart"
+ or_word="or", # Usage: "He is tall or small"
+ true="true", # Usage: "He is smart, true, false or neither?"
+ false="false", # Usage: "He is smart, true, false or neither?"
+ neither="neither", # Usage: "He is smart, true, false or neither?"
+ # Punctuation and spacing: only adjust if your language uses something different than in English
+ full_stop=".",
+ comma=",",
+ question_mark="?",
+ exclamation_mark="!",
+ word_space=" ",
+ sentence_space=" ",
+ colon=":",
+ # The first characters of your alphabet used in enumerations, if different from English
+ indices=["A", "B", "C", ...]
+)
+```
+
+## Contributing New Multilingual Tasks
+
+### Prerequisites
+
+Before creating a new multilingual task, you should:
+
+1. **Read the custom task guide**: [Adding a Custom Task](adding-a-custom-task)
+2. **Understand multilingual task structure**: Review the [multilingual tasks](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/multilingual/tasks.py) file
+3. **Browse available templates**: Check the [templates directory](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/templates)
+
+### Key Concepts
+
+#### Language-Adapted Templates
+For multilingual evaluations, the `prompt_function` should be implemented using language-adapted templates. These templates handle:
+- **Correct formatting** for each language
+- **Consistent usage** of language-adjusted prompt anchors (e.g., Question/Answer)
+- **Proper punctuation** and spacing conventions
+
+#### Template Types
+Available template types include:
+- **XNLI**: Natural language inference tasks - [`get_nli_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/nli.py#L162)
+- **COPA**: Causal reasoning tasks - [`get_copa_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/copa.py#L76)
+- **Multiple Choice**: Standard multiple choice questions - [`get_mcq_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/multichoice.py#L81)
+- **Question Answering**: Open-ended question answering - [`get_qa_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/qa.py#L46)
+- **Custom**: Specialized task templates
+
+#### Formulation Types
+
+##### Multiple Choice Formulation (MCF)
+Used for standard multiple choice questions where the model selects from lettered options:
+```python
+MCFFormulation()
+```
+
+**Example output:**
+```
+Question: What is the capital of France?
+A. London
+B. Paris
+C. Berlin
+D. Rome
+Answer: | A/B/C/D
+```
+
+##### Classification Formulation (CF)
+Used for classification tasks where the model generates the answer directly:
+```python
+CFFormulation()
+```
+
+**Example output:**
+```
+Question: What is the capital of France?
+Answer: | Paris
```
-3. Open a PR with your modifications! And voilà!
+##### Hybrid Formulation
+Used for tasks that present choices but expect the full answer text:
+```python
+HybridFormulation()
+```
-## Contributing a new multilingual task
+**Example output:**
+```
+Question: What is the capital of France?
+A. London
+B. Paris
+C. Berlin
+D. Rome
+Answer: | Paris
+```
-You should first read our guide on [adding a custom task](adding-a-custom-task), to better understand the different parameters we use.
-Then, you should take a look at the current [multilingual tasks](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/multilingual/tasks.py) file, to understand how they are defined. For multilingual evaluations the `prompt_function` should be implemented by language-adapted template. The template will take care of correct formatting, correct and consistent usage of language adjusted prompt anchors (e.g Question/Answer) and punctuation.
+### Creating Your Multilingual Task
-Browse the list of all templates [here](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/templates) to see which are the most adapted to your own task.
+#### Step 1: Create the Task File
+Create a Python file following the custom task guide structure.
-Then, when ready, to define your own task, you should:
-1. create a Python file as indicated in the above guide
-2. import the relevant templates for your task type (XNLI, Copa, Multiple choice, Question Answering, etc)
-3. define one or a list of tasks for each relevant language and evaluation formulation (for multichoice) using our parametrizable [`~tasks.lighteval_task.LightevalTaskConfig`] class
+#### Step 2: Import Required Components
+```python
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+from lighteval.tasks.multilingual.language import Language
+from lighteval.tasks.multilingual.formulations import MCFFormulation, CFFormulation, HybridFormulation
+from lighteval.tasks.multilingual.templates import get_template_prompt_function
+from lighteval.tasks.multilingual.metrics import get_metrics_for_formulation, loglikelihood_acc_metric
+from lighteval.tasks.multilingual.normalization import LogProbTokenNorm, LogProbCharNorm
+```
+#### Step 3: Define Your Tasks
```python
your_tasks = [
LightevalTaskConfig(
@@ -72,14 +161,14 @@ your_tasks = [
# In this function, you choose which template to follow and for which language and formulation
prompt_function=get_template_prompt_function(
language=language,
- # then use the adapter to define the mapping between the
+ # Use the adapter to define the mapping between the
# keys of the template (left), and the keys of your dataset
# (right)
# To know which template keys are required and available,
# consult the appropriate adapter type and doc-string.
adapter=lambda line: {
"key": line["relevant_key"],
- ...
+ # Add more mappings as needed
},
formulation=formulation,
),
@@ -93,15 +182,30 @@ your_tasks = [
hf_avail_splits=["train"],
)
for language in [
- Language.YOUR_LANGUAGE, ...
+ Language.YOUR_LANGUAGE, # Add your target languages
+ # Language.SPANISH,
+ # Language.FRENCH,
+ # etc.
]
for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()]
]
```
-4. then, you can go back to the guide to test if your task is correctly implemented!
+
+#### Step 4: Test Your Implementation
+Follow the custom task guide to test if your task is correctly implemented.
> [!TIP]
> All [`~tasks.lighteval_task.LightevalTaskConfig`] parameters are strongly typed, including the inputs to the template function. Make sure to take advantage of your IDE's functionality to make it easier to correctly fill these parameters.
+### Validation Checklist
+- [ ] Translation literals are accurate and complete
+- [ ] Task works correctly across all target languages
+- [ ] Metrics are appropriate for the task type
+- [ ] Documentation is clear and comprehensive
+- [ ] Code follows project conventions
+
+### Getting Help
-Once everything is good, open a PR, and we'll be happy to review it!
+- **GitHub Issues**: Report bugs or ask questions
+- **Discussions**: Join community discussions
+- **Documentation**: Review existing guides and examples
diff --git a/docs/source/evaluating-a-custom-model.mdx b/docs/source/evaluating-a-custom-model.mdx
index 97a30aa53..3bd934340 100644
--- a/docs/source/evaluating-a-custom-model.mdx
+++ b/docs/source/evaluating-a-custom-model.mdx
@@ -1,12 +1,14 @@
-# Custom Model
+# Evaluating Custom Models
Lighteval allows you to evaluate custom model implementations by creating a custom model class that inherits from `LightevalModel`.
-This is useful when you want to evaluate models that aren't directly supported by the standard backends and providers (transformers, vllm, etc), or
-if you want to add your own pre/post processing.
+This is useful when you want to evaluate models that aren't directly supported by the standard backends and providers (Transformers, VLLM, etc.), or
+if you want to add your own pre/post-processing logic.
## Creating a Custom Model
-1. Create a Python file containing your custom model implementation. The model must inherit from `LightevalModel` and implement all required methods.
+### Step 1: Create Your Model Implementation
+
+Create a Python file containing your custom model implementation. The model must inherit from `LightevalModel` and implement all required methods.
Here's a basic example:
@@ -41,14 +43,16 @@ class MyCustomModel(LightevalModel):
pass
```
-2. The custom model file should contain exactly one class that inherits from `LightevalModel`. This class will be automatically detected and instantiated when loading the model.
+### Step 2: Model File Requirements
+
+The custom model file should contain exactly one class that inherits from `LightevalModel`. This class will be automatically detected and instantiated when loading the model.
> [!TIP]
> You can find a complete example of a custom model implementation in `examples/custom_models/google_translate_model.py`.
## Running the Evaluation
-You can evaluate your custom model using either the command line interface or the Python API.
+You can evaluate your custom model using either the command-line interface or the Python API.
### Using the Command Line
@@ -61,16 +65,16 @@ lighteval custom \
```
The command takes three required arguments:
-- The model name (used for tracking in results/logs)
-- The path to your model implementation file
-- The tasks to evaluate on (same format as other backends)
+- **Model name**: Used for tracking in results/logs
+- **Model implementation file path**: Path to your Python file containing the custom model
+- **Tasks**: Tasks to evaluate on (same format as other backends)
### Using the Python API
```python
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.models.custom.custom_model import CustomModelConfig
-from lighteval.pipeline import Pipeline, PipelineParameters
+from lighteval.pipeline import Pipeline, PipelineParameters, ParallelismManager
# Set up evaluation tracking
evaluation_tracker = EvaluationTracker(
@@ -85,7 +89,7 @@ pipeline_params = PipelineParameters(
# Configure your custom model
model_config = CustomModelConfig(
- model="my-custom-model",
+ model_name="my-custom-model",
model_definition_file_path="path/to/my_model.py"
)
@@ -105,9 +109,56 @@ pipeline.save_and_push_results()
Your custom model must implement these core methods:
-- `greedy_until`: For generating text until a stop sequence or max tokens is reached - this is used for generative evaluations
-- `loglikelihood`: For computing log probabilities of specific continuations - this is used for multiple choice logprob evaluations
-- `loglikelihood_rolling`: For computing rolling log probabilities of sequences - this is used for perplexity metrics
+### `greedy_until`
+For generating text until a stop sequence or max tokens is reached. This is used for generative evaluations.
+
+```python
+def greedy_until(self, docs: List[Doc]) -> List[ModelResponse]:
+ """
+ Generate text until stop sequence or max tokens.
+
+ Args:
+ docs: List of documents containing prompts and generation parameters
+
+ Returns:
+ List of model responses with generated text
+ """
+ pass
+```
+
+### `loglikelihood`
+For computing log probabilities of specific continuations. This is used for multiple choice logprob evaluations.
+
+```python
+def loglikelihood(self, docs: List[Doc]) -> List[ModelResponse]:
+ """
+ Compute log probabilities of continuations.
+
+ Args:
+ docs: List of documents containing context and continuation pairs
+
+ Returns:
+ List of model responses with log probabilities
+ """
+ pass
+```
+
+### `loglikelihood_rolling`
+For computing rolling log probabilities of sequences. This is used for perplexity metrics.
+
+```python
+def loglikelihood_rolling(self, docs: List[Doc]) -> List[ModelResponse]:
+ """
+ Compute rolling log probabilities of sequences.
+
+ Args:
+ docs: List of documents containing text sequences
+
+ Returns:
+ List of model responses with rolling log probabilities
+ """
+ pass
+```
See the `LightevalModel` base class documentation for detailed method signatures and requirements.
@@ -116,33 +167,43 @@ See the `LightevalModel` base class documentation for detailed method signatures
Lighteval includes a caching system that can significantly speed up evaluations by storing and reusing model predictions.
To enable caching in your custom model:
-1. **Import caching components**:
- ```python
- from lighteval.utils.cache_management import SampleCache, cached
- ```
+### Step 1: Import Caching Components
+```python
+from lighteval.utils.cache_management import SampleCache, cached
+```
+
+### Step 2: Initialize Cache in Constructor
+```python
+def __init__(self, config):
+ super().__init__(config)
+ # Your initialization code...
+ self._cache = SampleCache(config)
+```
-2. **Initialize cache in constructor**:
- ```python
- def __init__(self, config):
- # Your initialization code...
- self._cache = SampleCache(config)
- ```
+### Step 3: Add Cache Decorators
+Add cache decorators to your prediction methods:
+```python
+@cached("predictions")
+def greedy_until(self, docs: List[Doc]) -> List[ModelResponse]:
+ # Your implementation...
+```
-3. **Add cache decorators** to your prediction methods:
- ```python
- @cached("predictions")
- def greedy_until(self, docs: List[Doc]) -> List[ModelResponse]:
- # Your implementation...
- ```
+For detailed information about the caching system, see the [Caching Documentation](caching).
-For detailed information about the caching system, see the [Caching Documentation](./caching.mdx).
+## Troubleshooting
-## Best Practices
+### Common Issues
-1. **Error Handling**: Implement robust error handling in your model methods to gracefully handle edge cases.
+1. **Import Errors**: Ensure all required dependencies are installed
+2. **Method Signature Errors**: Verify your methods match the expected signatures
+3. **Caching Issues**: Check that cache decorators are applied correctly
+4. **Performance Issues**: Consider implementing batching and caching
-2. **Batching**: Consider implementing efficient batching in your model methods to improve performance.
+### Debugging Tips
-3. **Documentation**: Add clear docstrings to your model class and methods explaining any specific requirements or limitations.
+- Use the `--max-samples` flag to test with a small dataset
+- Enable detailed logging to see what's happening
+- Test individual methods in isolation
+- Check the example implementations for reference
-4. **Caching**: Enable caching to speed up repeated evaluations and development iterations.
+For more detailed information about custom model implementation, see the [Model Reference](package_reference/models).
diff --git a/docs/source/index.mdx b/docs/source/index.mdx
index 71809d622..c9c1c7c30 100644
--- a/docs/source/index.mdx
+++ b/docs/source/index.mdx
@@ -1,19 +1,70 @@
# Lighteval
-🤗 Lighteval is your all-in-one toolkit for evaluating LLMs across multiple
-backends—whether it's
-[transformers](https://github.com/huggingface/transformers),
-[tgi](https://github.com/huggingface/text-generation-inference),
-[inference providers](https://huggingface.co/docs/huggingface_hub/en/guides/inference),
-[vllm](https://github.com/vllm-project/vllm), or
-[nanotron](https://github.com/huggingface/nanotron)-with
-ease. Dive deep into your model’s performance by saving and exploring detailed,
-sample-by-sample results to debug and see how your models stack-up.
-
-Customization at your fingertips: letting you effortlessly create [new
-tasks](adding-a-custom-task) and
-[metrics](adding-a-new-metric)
-tailored to your needs, or browsing all our existing tasks and metrics.
-
-Seamlessly experiment, benchmark, and store your results on the Hugging Face
-Hub, S3, or locally.
+🤗 Lighteval is your all-in-one toolkit for evaluating Large Language Models
+(LLMs) across multiple backends with ease. Dive deep into your model's
+performance by saving and exploring detailed, sample-by-sample results to debug
+and see how your models stack up.
+
+## Key Features
+
+### 🚀 **Multi-Backend Support**
+Evaluate your models using the most popular and efficient inference backends:
+- `transformers`: Evaluate models on CPU or one or more GPUs using [🤗
+ Accelerate](https://github.com/huggingface/transformers)
+- `nanotron`: Evaluate models in distributed settings using [⚡️
+ Nanotron](https://github.com/huggingface/nanotron)
+- `vllm`: Evaluate models on one or more GPUs using [🚀
+ VLLM](https://github.com/vllm-project/vllm)
+- `custom`: Evaluate custom models (can be anything)
+- `sglang`: Evaluate models using [SGLang](https://github.com/sgl-project/sglang) as backend
+- `inference-endpoint`: Evaluate models using Hugging Face's [Inference Endpoints API](https://huggingface.co/inference-endpoints/dedicated)
+- `tgi`: Evaluate models using [🔗 Text Generation Inference](https://huggingface.co/docs/text-generation-inference/en/index) running locally
+- `litellm`: Evaluate models on any compatible API using [LiteLLM](https://www.litellm.ai/)
+- `inference-providers`: Evaluate models using [HuggingFace's inference providers](https://huggingface.co/docs/inference-providers/en/index) as backend**: Distributed training and evaluation
+
+### 📊 **Comprehensive Evaluation**
+- **Extensive Task Library**: 1000s pre-built evaluation tasks
+- **Custom Task Creation**: Build your own evaluation tasks
+- **Flexible Metrics**: Support for custom metrics and scoring
+- **Detailed Analysis**: Sample-by-sample results for deep insights
+
+### 🔧 **Easy Customization**
+Customization at your fingertips: create [new tasks](adding-a-custom-task),
+[metrics](adding-a-new-metric) or [model](evaluating-a-custom-model) tailored to your needs, or browse all our existing tasks and metrics.
+
+### ☁️ **Seamless Integration**
+Seamlessly experiment, benchmark, and store your results on the Hugging Face Hub, S3, or locally.
+
+## Quick Start
+
+### Installation
+
+```bash
+pip install lighteval
+```
+
+### Basic Usage
+
+```bash
+# Evaluate a model using Transformers backend
+lighteval accelerate \
+ "model_name=openai-community/gpt2" \
+ "leaderboard|truthfulqa:mc|0|0"
+```
+
+### Save Results
+
+```bash
+# Save locally
+lighteval accelerate \
+ "model_name=openai-community/gpt2" \
+ "leaderboard|truthfulqa:mc|0|0" \
+ --output-dir ./results
+
+# Push to Hugging Face Hub
+lighteval accelerate \
+ "model_name=openai-community/gpt2" \
+ "leaderboard|truthfulqa:mc|0|0" \
+ --push-to-hub \
+ --results-org your-username
+```
diff --git a/docs/source/installation.mdx b/docs/source/installation.mdx
index a57bcf6b7..0025c04f6 100644
--- a/docs/source/installation.mdx
+++ b/docs/source/installation.mdx
@@ -1,15 +1,26 @@
# Installation
-You can install Lighteval either from PyPi or from source.
+Lighteval can be installed from PyPI or from source. This guide covers all installation options and dependencies.
-## From PyPi
+## System Requirements
+
+- **Python**: 3.10 or higher
+- **PyTorch**: 2.0 or higher (but less than 3.0)
+- **CUDA**: Optional, for GPU acceleration
+
+## From PyPI
+
+The simplest way to install Lighteval is from PyPI:
```bash
pip install lighteval
```
-## From source
-Source install is mostly for people who intend to develop on `lighteval`
+This installs the core package with all essential dependencies for basic evaluation tasks.
+
+## From Source
+
+Source installation is recommended for developers who want to contribute to Lighteval or need the latest features:
```bash
git clone https://github.com/huggingface/lighteval.git
@@ -17,32 +28,44 @@ cd lighteval
pip install -e .
```
-## Extras
+## Optional Dependencies (Extras)
-Lighteval has optional dependencies that you can install by specifying the
-appropriate extras group.
-`pip install lighteval[]` or `pip install -e .[]`.
+Lighteval provides several optional dependency groups that you can install based on your needs. Use the format `pip install lighteval[]` or `pip install -e .[]` for source installation.
-If you want to use lighteval with `sglang`, try to follow [sglang install documentation](https://docs.sglang.ai/start/install.html).
+### Backend Extras
-| extra name | description |
-|--------------|---------------------------------------------------------------------------|
-| tgi | To use Text Generation Inference API to evaluate your model |
-| nanotron | To evaluate nanotron models |
-| quantization | To evaluate quantized models |
-| adapters | To evaluate adapters models (delta and peft) |
-| tensorboardX | To upload your results to tensorboard |
-| vllm | To use vllm as backend for inference |
-| sglang | To use sglang as backend for inference |
-| s3 | To upload results to s3 |
+| Extra | Description | Dependencies |
+|-------|-------------|--------------|
+| `vllm` | Use VLLM as backend for high-performance inference | vllm>=0.10.0, ray, more_itertools |
+| `tgi` | Use Text Generation Inference API | text-generation>=0.6.0 |
+| `litellm` | Use LiteLLM for unified API access | litellm, diskcache |
+| `optimum` | Use Optimum for optimized models | optimum==1.12.0 |
+| `quantization` | Evaluate quantized models | bitsandbytes>=0.41.0, auto-gptq>=0.4.2 |
+| `adapters` | Evaluate adapter models (PEFT, Delta) | peft==0.3.0 |
+| `nanotron` | Evaluate Nanotron models | nanotron, tensorboardX |
+### Task and Feature Extras
-## Hugging Face login
+| Extra | Description | Dependencies |
+|-------|-------------|--------------|
+| `extended_tasks` | Extended evaluation tasks | langdetect, openai>1.87, tiktoken |
+| `multilingual` | Multilingual evaluation support | stanza, spacy[ja,ko,th], jieba, pyvi |
+| `math` | Mathematical reasoning tasks | latex2sympy2_extended==1.0.6 |
-If you want to push your results to the Hugging Face Hub or evaluate your own
-private models, don't forget to add your access token to the environment
-variable `HF_TOKEN`. You can do this by running:
+### Storage and Logging Extras
-```bash
-huggingface-cli login
-```
+| Extra | Description | Dependencies |
+|-------|-------------|--------------|
+| `s3` | Upload results to S3 | s3fs |
+| `tensorboardX` | Upload results to TensorBoard | tensorboardX |
+| `wandb` | Log results to Weights & Biases | wandb |
+| `trackio` | Log results to Trackio | trackio |
+
+### Development Extras
+
+| Extra | Description | Dependencies |
+|-------|-------------|--------------|
+| `quality` | Code quality tools | ruff>=v0.11.0, pre-commit |
+| `tests` | Testing dependencies | pytest>=7.4.0, deepdiff |
+| `docs` | Documentation building | hf-doc-builder, watchdog |
+| `dev` | All development dependencies | Includes accelerate, quality, tests, multilingual, math, extended_tasks, vllm |
diff --git a/docs/source/package_reference/doc.mdx b/docs/source/package_reference/doc.mdx
new file mode 100644
index 000000000..0a666ec83
--- /dev/null
+++ b/docs/source/package_reference/doc.mdx
@@ -0,0 +1,3 @@
+# Doc
+
+[[autodoc]] tasks.requests.Doc
diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx
index 682a08cda..d93af7078 100644
--- a/docs/source/quicktour.mdx
+++ b/docs/source/quicktour.mdx
@@ -1,28 +1,45 @@
-# Quicktour
-
+# Quick Tour
> [!TIP]
> We recommend using the `--help` flag to get more information about the
> available options for each command.
> `lighteval --help`
-Lighteval can be used with a few different commands.
+Lighteval can be used with several different commands, each optimized for different evaluation scenarios.
+
+## Available Commands
-- `lighteval accelerate` : evaluate models on CPU or one or more GPUs using [🤗
+### Evaluation Backends
+
+- `lighteval accelerate`: Evaluate models on CPU or one or more GPUs using [🤗
Accelerate](https://github.com/huggingface/accelerate)
-- `lighteval nanotron`: evaluate models in distributed settings using [⚡️
+- `lighteval nanotron`: Evaluate models in distributed settings using [⚡️
Nanotron](https://github.com/huggingface/nanotron)
-- `lighteval vllm`: evaluate models on one or more GPUs using [🚀
+- `lighteval vllm`: Evaluate models on one or more GPUs using [🚀
VLLM](https://github.com/vllm-project/vllm)
-- `lighteval endpoint`
- - `inference-endpoint`: evaluate models using Hugging Face's [Inference Endpoints's API](https://huggingface.co/inference-endpoints/dedicated)
- - `tgi`: evaluate models using [🔗 Text Generation Inference](https://huggingface.co/docs/text-generation-inference/en/index) running locally.
- - `litellm`: evaluate models on any compatible API using [litellm](https://www.litellm.ai/)
+- `lighteval custom`: Evaluate custom models (can be anything)
+- `lighteval sglang`: Evaluate models using [SGLang](https://github.com/sgl-project/sglang) as backend
+- `lighteval endpoint`: Evaluate models using various endpoints as backend
+ - `lighteval endpoint inference-endpoint`: Evaluate models using Hugging Face's [Inference Endpoints API](https://huggingface.co/inference-endpoints/dedicated)
+ - `lighteval endpoint tgi`: Evaluate models using [🔗 Text Generation Inference](https://huggingface.co/docs/text-generation-inference/en/index) running locally
+ - `lighteval endpoint litellm`: Evaluate models on any compatible API using [LiteLLM](https://www.litellm.ai/)
+ - `lighteval endpoint inference-providers`: Evaluate models using [HuggingFace's inference providers](https://huggingface.co/docs/inference-providers/en/index) as backend
+
+### Evaluation Utils
-## Basic usage
+- `lighteval baseline`: Compute baselines for given tasks
+
+### Utils
+
+- `lighteval tasks`: List or inspect tasks
+ - `lighteval tasks list`: List all available tasks
+ - `lighteval tasks inspect`: Inspect a specific task to see its configuration and samples
+ - `lighteval tasks create`: Create a new task from a template
+
+## Basic Usage
To evaluate `GPT-2` on the Truthful QA benchmark with [🤗
- Accelerate](https://github.com/huggingface/accelerate) , run:
+ Accelerate](https://github.com/huggingface/accelerate), run:
```bash
lighteval accelerate \
@@ -32,8 +49,7 @@ lighteval accelerate \
Here, we first choose a backend (either `accelerate`, `nanotron`, `endpoint`, or `vllm`), and then specify the model and task(s) to run.
-The syntax for the model arguments is `key1=value1,key2=value2,etc`.
-Valid key-value pairs correspond with the backend configuration, and are detailed [below](#Model Arguments).
+### Task Specification
The syntax for the task specification might be a bit hard to grasp at first. The format is as follows:
@@ -56,13 +72,15 @@ All officially supported tasks can be found at the [tasks_list](available-tasks)
[extended folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/extended).
Moreover, community-provided tasks can be found in the
[community](https://github.com/huggingface/lighteval/tree/main/community_tasks) folder.
-For more details on the implementation of the tasks, such as how prompts are constructed, or which metrics are used, you can have a look at the
-[file](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/default_tasks.py)
-implementing them.
-Running multiple tasks is supported, either with a comma-separated list, or by specifying a file path.
+For more details on the implementation of the tasks, such as how prompts are constructed or which metrics are used, you can examine the
+[implementation file](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/default_tasks.py).
+
+### Running Multiple Tasks
+
+Running multiple tasks is supported, either with a comma-separated list or by specifying a file path.
The file should be structured like [examples/tasks/recommended_set.txt](https://github.com/huggingface/lighteval/blob/main/examples/tasks/recommended_set.txt).
-When specifying a path to file, it should start with `./`.
+When specifying a path to a file, it should start with `./`.
```bash
lighteval accelerate \
@@ -71,62 +89,23 @@ lighteval accelerate \
# or, e.g., "leaderboard|truthfulqa:mc|0,leaderboard|gsm8k|3"
```
-## Evaluate a model on one or more GPUs
-
-#### Data parallelism
-
-To evaluate a model on one or more GPUs, first create a multi-gpu config by running.
-
-```bash
-accelerate config
-```
-
-You can then evaluate a model using data parallelism on 8 GPUs like follows:
-
-```bash
-accelerate launch --multi_gpu --num_processes=8 -m \
- lighteval accelerate \
- "model_name=openai-community/gpt2" \
- "leaderboard|truthfulqa:mc|0"
-```
-
-Here, `--override_batch_size` defines the batch size per device, so the effective
-batch size will be `override_batch_size * num_gpus`.
-
-#### Pipeline parallelism
-
-To evaluate a model using pipeline parallelism on 2 or more GPUs, run:
-
-```bash
-lighteval accelerate \
- "model_name=openai-community/gpt2,model_parallel=True" \
- "leaderboard|truthfulqa:mc|0"
-```
-
-This will automatically use accelerate to distribute the model across the GPUs.
-
-> [!TIP]
-> Both data and pipeline parallelism can be combined by setting
-> `model_parallel=True` and using accelerate to distribute the data across the
-GPUs.
-
-## Backend configuration
+## Backend Configuration
-#### General information
+### General Information
The `model-args` argument takes a string representing a list of model
-argument. The arguments allowed vary depending on the backend you use and
-correspond to the fields of the model configs.
+arguments. The arguments allowed vary depending on the backend you use and
+correspond to the fields of the model configurations.
The model configurations can be found [here](./package_reference/models).
-All models allow you to post process your reasoning model predictions,
+All models allow you to post-process your reasoning model predictions
to remove the thinking tokens from the trace used to compute the metrics,
-using `--remove-reasoning-tags`, and `--reasoning-tags` to specify which
-reasoning tags to remove (defaults to and ).
+using `--remove-reasoning-tags` and `--reasoning-tags` to specify which
+reasoning tags to remove (defaults to `` and ``).
Here's an example with `mistralai/Magistral-Small-2507` which outputs custom
-think tokens.
+thinking tokens:
```bash
lighteval vllm \
@@ -136,23 +115,21 @@ lighteval vllm \
--reasoning-tags="[('[THINK]','[/THINK]')]"
```
+### Nanotron
-#### Nanotron
-
-To evaluate a model trained with nanotron on a single gpu.
+To evaluate a model trained with Nanotron on a single GPU:
> [!WARNING]
> Nanotron models cannot be evaluated without torchrun.
-
```bash
- torchrun --standalone --nnodes=1 --nproc-per-node=1 \
- src/lighteval/__main__.py nanotron \
- --checkpoint-config-path ../nanotron/checkpoints/10/config.yaml \
- --lighteval-config-path examples/nanotron/lighteval_config_override_template.yaml
- ```
+torchrun --standalone --nnodes=1 --nproc-per-node=1 \
+ src/lighteval/__main__.py nanotron \
+ --checkpoint-config-path ../nanotron/checkpoints/10/config.yaml \
+ --lighteval-config-path examples/nanotron/lighteval_config_override_template.yaml
+```
-The `nproc-per-node` argument should match the data, tensor and pipeline
+The `nproc-per-node` argument should match the data, tensor, and pipeline
parallelism configured in the `lighteval_config_template.yaml` file.
That is: `nproc-per-node = data_parallelism * tensor_parallelism *
pipeline_parallelism`.
diff --git a/docs/source/saving-and-reading-results.mdx b/docs/source/saving-and-reading-results.mdx
index a8a3fb2bc..580e739b5 100644
--- a/docs/source/saving-and-reading-results.mdx
+++ b/docs/source/saving-and-reading-results.mdx
@@ -1,71 +1,70 @@
-# Saving and reading results
+# Saving and Reading Results
-## Saving results locally
+Lighteval provides comprehensive logging and result management through the `EvaluationTracker` class. This system allows you to save results locally and optionally push them to various platforms for collaboration and analysis.
-Lighteval will automatically save results and evaluation details in the
-directory set with the `--output-dir` option. The results will be saved in
+## Saving Results Locally
+
+Lighteval automatically saves results and evaluation details in the
+directory specified with the `--output-dir` option. The results are saved in
`{output_dir}/results/{model_name}/results_{timestamp}.json`. [Here is an
example of a result file](#example-of-a-result-file). The output path can be
any [fsspec](https://filesystem-spec.readthedocs.io/en/latest/index.html)
-compliant path (local, s3, hf hub, gdrive, ftp, etc).
+compliant path (local, S3, Hugging Face Hub, Google Drive, FTP, etc.).
-To save the details of the evaluation, you can use the `--save-details`
-option. The details will be saved in a parquet file
+To save detailed evaluation information, you can use the `--save-details`
+option. The details are saved in Parquet files at
`{output_dir}/details/{model_name}/{timestamp}/details_{task}_{timestamp}.parquet`.
-If you want results to be saved in a custom path, you can set the `results-path-template` option.
-This allows you to set a string template for the path. The template need to contain the following
-variables: `output_dir`, `model_name`, `org`. For example
+If you want results to be saved in a custom path structure, you can set the `results-path-template` option.
+This allows you to specify a string template for the path. The template must contain the following
+variables: `output_dir`, `model_name`, `org`. For example:
`{output_dir}/{org}_{model}`. The template will be used to create the path for the results file.
-## Pushing results to the HuggingFace hub
+## Pushing Results to the Hugging Face Hub
-You can push the results and evaluation details to the HuggingFace hub. To do
-so, you need to set the `--push-to-hub` as well as the `--results-org`
-option. The results will be saved in a dataset with the name at
+You can push results and evaluation details to the Hugging Face Hub. To do
+so, you need to set the `--push-to-hub` option as well as the `--results-org`
+option. The results are saved in a dataset with the name
`{results_org}/{model_org}/{model_name}`. To push the details, you need to set
the `--save-details` option.
-The dataset created will be private by default, you can make it public by
-setting the `--public-run` option.
+The dataset created will be private by default. You can make it public by
+setting the `--public-run` option.
-## Pushing results to Tensorboard
+## Pushing Results to TensorBoard
-You can push the results to Tensorboard by setting `--push-to-tensorboard`.
-This will create a Tensorboard dashboard in a HF org set with the `--results-org`
+You can push results to TensorBoard by setting `--push-to-tensorboard`.
+This creates a TensorBoard dashboard in a Hugging Face organization specified with the `--results-org`
option.
+## Pushing Results to Weights & Biases or Trackio
-## Pushing results to Trackio or WandB
-
-You can push the results to WandB by setting `--wandb`. This will init a WandB
-run and log the results.
+You can push results to Weights & Biases by setting `--wandb`. This initializes a W&B
+run and logs the results.
-Wandb args need to be set in your env variables.
+W&B arguments need to be set in your environment variables:
-```
+```bash
export WANDB_PROJECT="lighteval"
```
-You can find a list of variable in the [wandb documentation](https://docs.wandb.ai/guides/track/environment-variables/).
+You can find a complete list of variables in the [W&B documentation](https://docs.wandb.ai/guides/track/environment-variables/).
-If trackio is available in your environment `pip install
-lighteval[trackio]`, it will be used to log and push the results on a
-huggingface dataset. Choose the dataset name and org with:
+If Trackio is available in your environment (`pip install lighteval[trackio]`), it will be used to log and push results to a
+Hugging Face dataset. Choose the dataset name and organization with:
-```
+```bash
export WANDB_SPACE_ID="org/name"
```
+## How to Load and Investigate Details
-
-## How to load and investigate details
-
-### Load from local detail files
+### Loading from Local Detail Files
```python
from datasets import load_dataset
import os
+import glob
output_dir = "evals_doc"
model_name = "HuggingFaceH4/zephyr-7b-beta"
@@ -73,7 +72,7 @@ timestamp = "latest"
task = "lighteval|gsm8k|0"
if timestamp == "latest":
- path = f"{output_dir}/details/{model_org}/{model_name}/*/"
+ path = f"{output_dir}/details/{model_name}/*/"
timestamps = glob.glob(path)
timestamp = sorted(timestamps)[-1].split("/")[-2]
print(f"Latest timestamp: {timestamp}")
@@ -87,7 +86,7 @@ for detail in details:
print(detail)
```
-### Load from the HuggingFace hub
+### Loading from the Hugging Face Hub
```python
from datasets import load_dataset
@@ -105,21 +104,83 @@ for detail in details:
print(detail)
```
+## Detail File Structure
The detail file contains the following columns:
-- __doc__: The doc used for the evaluation, this will contain the gold reference, the fewshots and other hyperparamters used for the task.
-- __model_response__: where you will find model generations, logprobs and the input that was sent to the model
-- __metric__: the value of the metrics for this sample
+- **`__doc__`**: The document used for evaluation, containing the gold reference, few-shot examples, and other hyperparameters used for the task.
+- **`__model_response__`**: Contains model generations, log probabilities, and the input that was sent to the model.
+- **`__metric__`**: The value of the metrics for this sample.
+
+## EvaluationTracker Configuration
+
+The `EvaluationTracker` class provides several configuration options for customizing how results are saved and pushed:
+
+### Basic Configuration
+
+```python
+from lighteval.logging.evaluation_tracker import EvaluationTracker
+
+tracker = EvaluationTracker(
+ output_dir="./results",
+ save_details=True,
+ push_to_hub=True,
+ hub_results_org="your_username",
+ public=False
+)
+```
+
+### Advanced Configuration
+
+```python
+tracker = EvaluationTracker(
+ output_dir="./results",
+ results_path_template="{output_dir}/custom/{org}_{model}",
+ save_details=True,
+ push_to_hub=True,
+ push_to_tensorboard=True,
+ hub_results_org="my-org",
+ tensorboard_metric_prefix="eval",
+ public=True,
+ use_wandb=True
+)
+```
+
+### Key Parameters
+
+- **`output_dir`**: Local directory to save evaluation results and logs
+- **`results_path_template`**: Template for results directory structure
+- **`save_details`**: Whether to save detailed evaluation records (default: True)
+- **`push_to_hub`**: Whether to push results to Hugging Face Hub (default: False)
+- **`push_to_tensorboard`**: Whether to push metrics to TensorBoard (default: False)
+- **`hub_results_org`**: Hugging Face Hub organization to push results to
+- **`tensorboard_metric_prefix`**: Prefix for TensorBoard metrics (default: "eval")
+- **`public`**: Whether to make Hub datasets public (default: False)
+- **`use_wandb`**: Whether to log to Weights & Biases or Trackio (default: False)
+
+## Result File Structure
+
+The main results file contains several sections:
+
+### General Configuration
+- **`config_general`**: Overall evaluation configuration including model information, timing, and system details
+- **`summary_general`**: General statistics about the evaluation run
+
+### Task-Specific Information
+- **`config_tasks`**: Configuration details for each evaluated task
+- **`summary_tasks`**: Task-specific statistics and metadata
+- **`versions`**: Version information for tasks and datasets
+
+### Results
+- **`results`**: Actual evaluation metrics and scores for each task
-## Example of a result file
+## Example of a Result File
```json
{
"config_general": {
"lighteval_sha": "203045a8431bc9b77245c9998e05fc54509ea07f",
"num_fewshot_seeds": 1,
- "override_batch_size": 1,
"max_samples": 1,
"job_id": "",
"start_time": 620979.879320166,
diff --git a/docs/source/use-huggingface-inference-endpoints-or-tgi-as-backend.mdx b/docs/source/use-huggingface-inference-endpoints-or-tgi-as-backend.mdx
index 232bcd73a..e0fa12e42 100644
--- a/docs/source/use-huggingface-inference-endpoints-or-tgi-as-backend.mdx
+++ b/docs/source/use-huggingface-inference-endpoints-or-tgi-as-backend.mdx
@@ -1,36 +1,36 @@
-# Evaluate the model on a server or container
+# Using Hugging Face Inference Endpoints or TGI as Backend
An alternative to launching the evaluation locally is to serve the model on a
TGI-compatible server/container and then run the evaluation by sending requests
to the server. The command is the same as before, except you specify a path to
-a yaml config file (detailed below):
+a YAML configuration file (detailed below):
```bash
lighteval endpoint {tgi,inference-endpoint} \
- "/path/to/config/file"\
-
+ "/path/to/config/file" \
+
```
There are two types of configuration files that can be provided for running on
the server:
-### Hugging Face Inference Endpoints
+## Hugging Face Inference Endpoints
-To launch a model using HuggingFace's Inference Endpoints, you need to provide
+To launch a model using Hugging Face's Inference Endpoints, you need to provide
the following file: `endpoint_model.yaml`. Lighteval will automatically deploy
the endpoint, run the evaluation, and finally delete the endpoint (unless you
specify an endpoint that was already launched, in which case the endpoint won't
be deleted afterwards).
-__configuration file example:__
+### Configuration File Example
```yaml
model_parameters:
- reuse_existing: false # if true, ignore all params in instance, and don't delete the endpoint after evaluation
-# endpoint_name: "llama-2-7B-lighteval" # needs to be lower case without special characters
+ reuse_existing: false # If true, ignore all params in instance, and don't delete the endpoint after evaluation
+ # endpoint_name: "llama-2-7B-lighteval" # Needs to be lowercase without special characters
model_name: "meta-llama/Llama-2-7b-hf"
- revision: "main" # defaults to "main"
- dtype: "float16" # can be any of "awq", "eetq", "gptq", "4bit' or "8bit" (will use bitsandbytes), "bfloat16" or "float16"
+ revision: "main" # Defaults to "main"
+ dtype: "float16" # Can be any of "awq", "eetq", "gptq", "4bit" or "8bit" (will use bitsandbytes), "bfloat16" or "float16"
accelerator: "gpu"
region: "eu-west-1"
vendor: "aws"
@@ -40,16 +40,15 @@ model_parameters:
endpoint_type: "protected"
namespace: null # The namespace under which to launch the endpoint. Defaults to the current user's namespace
image_url: null # Optionally specify the docker image to use when launching the endpoint model. E.g., launching models with later releases of the TGI container with support for newer models.
- env_vars:
- null # Optional environment variables to include when launching the endpoint. e.g., `MAX_INPUT_LENGTH: 2048`
+ env_vars: null # Optional environment variables to include when launching the endpoint. e.g., `MAX_INPUT_LENGTH: 2048`
```
-### Text Generation Inference (TGI)
+## Text Generation Inference (TGI)
-To use a model already deployed on a TGI server, for example on HuggingFace's
+To use a model already deployed on a TGI server, for example on Hugging Face's
serverless inference.
-__configuration file example:__
+### Configuration File Example
```yaml
model_parameters:
@@ -57,3 +56,101 @@ model_parameters:
inference_server_auth: null
model_id: null # Optional, only required if the TGI container was launched with model_id pointing to a local directory
```
+
+## Key Parameters
+
+### Hugging Face Inference Endpoints
+
+#### Model Configuration
+- `model_name`: The Hugging Face model ID to deploy
+- `revision`: Model revision (defaults to "main")
+- `dtype`: Data type for model weights ("float16", "bfloat16", "4bit", "8bit", etc.)
+- `framework`: Framework to use ("pytorch", "tensorflow")
+
+#### Infrastructure Settings
+- `accelerator`: Hardware accelerator ("gpu", "cpu")
+- `region`: AWS region for deployment
+- `vendor`: Cloud vendor ("aws", "azure", "gcp")
+- `instance_type`: Instance type (e.g., "nvidia-a10g", "nvidia-t4")
+- `instance_size`: Instance size ("x1", "x2", etc.)
+
+#### Endpoint Configuration
+- `endpoint_type`: Endpoint access level ("public", "protected", "private")
+- `namespace`: Organization namespace for deployment
+- `reuse_existing`: Whether to reuse an existing endpoint
+- `endpoint_name`: Custom endpoint name (lowercase, no special characters)
+
+#### Advanced Settings
+- `image_url`: Custom Docker image URL
+- `env_vars`: Environment variables for the endpoint
+
+### Text Generation Inference (TGI)
+
+#### Server Configuration
+- `inference_server_address`: URL of the TGI server
+- `inference_server_auth`: Authentication credentials
+- `model_id`: Model identifier (if using local model directory)
+
+## Usage Examples
+
+### Deploying a New Inference Endpoint
+
+```bash
+lighteval endpoint inference-endpoint \
+ "configs/endpoint_model.yaml" \
+ "lighteval|gsm8k|0|0"
+```
+
+### Using an Existing TGI Server
+
+```bash
+lighteval endpoint tgi \
+ "configs/tgi_server.yaml" \
+ "lighteval|gsm8k|0|0"
+```
+
+### Reusing an Existing Endpoint
+
+```yaml
+model_parameters:
+ reuse_existing: true
+ endpoint_name: "my-existing-endpoint"
+ # Other parameters will be ignored when reuse_existing is true
+```
+
+## Cost Management
+
+### Inference Endpoints
+- Endpoints are automatically deleted after evaluation (unless `reuse_existing: true`)
+- Costs are based on instance type and runtime
+- Monitor usage in the [Hugging Face billing dashboard](https://huggingface.co/settings/billing)
+
+### TGI Servers
+- No additional costs beyond your existing server infrastructure
+- Useful for cost-effective evaluation of already-deployed models
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Endpoint Deployment Failures**: Check instance availability in your region
+2. **Authentication Errors**: Ensure proper Hugging Face token permissions
+3. **Model Loading Errors**: Verify model name and revision are correct
+4. **Resource Constraints**: Choose appropriate instance type for your model size
+
+### Performance Tips
+
+- Use appropriate instance types for your model size
+- Consider using quantized models (4bit, 8bit) for cost savings
+- Reuse existing endpoints for multiple evaluations
+- Use serverless TGI for cost-effective evaluation
+
+### Error Handling
+
+Common error messages and solutions:
+- **"Instance not available"**: Try a different region or instance type
+- **"Model not found"**: Check the model name and revision
+- **"Insufficient permissions"**: Verify your Hugging Face token has endpoint deployment permissions
+- **"Endpoint already exists"**: Use `reuse_existing: true` or choose a different endpoint name
+
+For more detailed information about Hugging Face Inference Endpoints, see the [official documentation](https://huggingface.co/docs/inference-endpoints/).
diff --git a/docs/source/use-inference-providers-as-backend.mdx b/docs/source/use-inference-providers-as-backend.mdx
index 70b436a8a..aa9acd0ff 100644
--- a/docs/source/use-inference-providers-as-backend.mdx
+++ b/docs/source/use-inference-providers-as-backend.mdx
@@ -1,13 +1,12 @@
-# Inference Providers as backend
+# Using Inference Providers as Backend
-Lighteval allows to use Hugging Face's Inference Providers to evaluate llms on supported providers such as Black Forest Labs, Cerebras, Fireworks AI, Nebius, Together AI and many more.
-
-## Quick use
+Lighteval allows you to use Hugging Face's Inference Providers to evaluate LLMs on supported providers such as Black Forest Labs, Cerebras, Fireworks AI, Nebius, Together AI, and many more.
> [!WARNING]
-> Do not forget to set your HuggingFace API key.
+> Do not forget to set your Hugging Face API key.
> You can set it using the `HF_TOKEN` environment variable or by using the `huggingface-cli` command.
+## Basic Usage
```bash
lighteval endpoint inference-providers \
@@ -15,9 +14,9 @@ lighteval endpoint inference-providers \
"lighteval|gsm8k|0"
```
-## Using a config file
+## Using a Configuration File
-You can use config files to define the model and the provider to use.
+You can use configuration files to define the model and the provider to use.
```bash
lighteval endpoint inference-providers \
@@ -25,7 +24,7 @@ lighteval endpoint inference-providers \
"lighteval|gsm8k|0"
```
-with the following config file:
+With the following configuration file:
```yaml
model_parameters:
@@ -42,3 +41,53 @@ model_parameters:
By default, inference requests are billed to your personal account.
Optionally, you can charge them to an organization by setting `org_to_bill=""` (requires being a member of that organization).
+
+## Supported Providers
+
+Hugging Face Inference Providers supports a wide range of LLM providers see the [Inference Providers documentation](https://huggingface.co/docs/inference-providers/en/index) for the complete list.
+
+
+## Billing and Costs
+
+### Personal Account Billing
+By default, all inference requests are billed to your personal Hugging Face account. You can monitor your usage in the [Hugging Face billing dashboard](https://huggingface.co/settings/billing).
+
+### Organization Billing
+To bill requests to an organization:
+
+1. Ensure you are a member of the organization
+2. Add `org_to_bill=""` to your configuration
+3. The organization must have sufficient credits
+
+```yaml
+model_parameters:
+ model_name: "meta-llama/Llama-2-7b-chat-hf"
+ provider: "together"
+ org_to_bill: "my-organization"
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **API Key Errors**: Ensure your `HF_TOKEN` is set correctly
+2. **Provider Not Available**: Check that the provider supports your model
+3. **Billing Issues**: Verify you have sufficient credits or organization permissions
+4. **Timeout Errors**: Increase the timeout parameter for large models
+
+### Performance Tips
+
+- Use appropriate `parallel_calls_count` based on your needs
+- Monitor your usage to avoid unexpected costs
+- Consider using organization billing for team projects
+- Use appropriate timeout values for your model size
+
+### Error Handling
+
+Common error messages and solutions:
+- **"Provider not found"**: Check the provider name is correct
+- **"Model not available"**: Verify the model is supported by the provider
+- **"Insufficient credits"**: Add credits to your account or organization
+- **"Timeout"**: Increase the timeout parameter or use a smaller model
+
+For more detailed error handling and provider-specific information, refer to the [Hugging Face Inference Providers documentation](https://huggingface.co/docs/inference-endpoints/guides/inference_providers).
diff --git a/docs/source/use-litellm-as-backend.mdx b/docs/source/use-litellm-as-backend.mdx
index 36ecf841d..a6f55a84f 100644
--- a/docs/source/use-litellm-as-backend.mdx
+++ b/docs/source/use-litellm-as-backend.mdx
@@ -1,31 +1,32 @@
-# Litellm as backend
+# Using LiteLLM as Backend
-Lighteval allows to use litellm, a backend allowing you to call all LLM APIs
-using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure,
-OpenAI, Groq etc.].
+Lighteval allows you to use LiteLLM as a backend, enabling you to call all LLM APIs
+using the OpenAI format. LiteLLM supports various providers including Bedrock, Hugging Face, Vertex AI, Together AI, Azure,
+OpenAI, Groq, and many others.
-Documentation for available APIs and compatible endpoints can be found [here](https://docs.litellm.ai/docs/).
+> [!TIP]
+> Documentation for available APIs and compatible endpoints can be found [here](https://docs.litellm.ai/docs/).
-## Quick use
+## Basic Usage
```bash
lighteval endpoint litellm \
"provider=openai,model_name=gpt-3.5-turbo" \
- "lighteval|gsm8k|0" \
+ "lighteval|gsm8k|0"
```
-## Using a config file
+## Using a Configuration File
-Litellm allows generation with any OpenAI compatible endpoint, for example you
-can evaluate a model running on a local vllm server.
+LiteLLM allows generation with any OpenAI-compatible endpoint. For example, you
+can evaluate a model running on a local VLLM server.
-To do so you will need to use a config file like so:
+To do so, you will need to use a configuration file like this:
```yaml
model_parameters:
model_name: "openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
- base_url: "URL OF THE ENDPOINT YOU WANT TO USE"
- api_key: "" # remove or keep empty as needed
+ base_url: "URL_OF_THE_ENDPOINT_YOU_WANT_TO_USE"
+ api_key: "" # Remove or keep empty as needed
generation_parameters:
temperature: 0.5
max_new_tokens: 256
@@ -35,3 +36,61 @@ model_parameters:
repetition_penalty: 1.0
frequency_penalty: 0.0
```
+
+## Supported Providers
+
+LiteLLM supports a wide range of LLM providers:
+
+### Cloud Providers
+
+all cloud providers can be found in the [litellm documentation](https://docs.litellm.ai/docs/providers).
+
+### Local/On-Premise
+- **VLLM**: Local VLLM servers
+- **Hugging Face**: Local Hugging Face models
+- **Custom endpoints**: Any OpenAI-compatible API
+
+## Using with Local Models
+
+### VLLM Server
+To use with a local VLLM server:
+
+1. Start your VLLM server:
+```bash
+vllm serve HuggingFaceH4/zephyr-7b-beta --host 0.0.0.0 --port 8000
+```
+
+2. Configure LiteLLM to use the local server:
+```yaml
+model_parameters:
+ provider: "openai"
+ model_name: "HuggingFaceH4/zephyr-7b-beta"
+ base_url: "http://localhost:8000/v1"
+ api_key: ""
+```
+
+
+## Troubleshooting
+
+### Common Issues
+
+1. **API Key Errors**: Ensure your API key is correct and has sufficient permissions
+2. **Connection Errors**: Check that the base URL is accessible and the service is running
+3. **Model Not Found**: Verify the model name is correct for the specified provider
+4. **Rate Limiting**: Implement retry logic or reduce request frequency
+
+### Performance Tips
+
+- Implement retry logic for transient failures
+- Consider using batch processing for multiple evaluations
+- Monitor API usage and costs for cloud providers
+
+### Error Handling
+
+LiteLLM provides detailed error messages for common issues:
+- Invalid API keys
+- Model not available
+- Rate limiting
+- Network connectivity issues
+
+For more detailed error handling and debugging, refer to the [LiteLLM documentation](https://docs.litellm.ai/docs/).
diff --git a/docs/source/use-sglang-as-backend.mdx b/docs/source/use-sglang-as-backend.mdx
index a13c1b82a..260f074c6 100644
--- a/docs/source/use-sglang-as-backend.mdx
+++ b/docs/source/use-sglang-as-backend.mdx
@@ -1,7 +1,9 @@
-# Use SGLang as backend
+# Using SGLang as Backend
-Lighteval allows you to use `sglang` as backend allowing great speedups.
-To use, simply change the `model_args` to reflect the arguments you want to pass to sglang.
+Lighteval allows you to use SGLang as a backend, providing significant speedups for model evaluation.
+To use SGLang, simply change the `model_args` to reflect the arguments you want to pass to SGLang.
+
+## Basic Usage
```bash
lighteval sglang \
@@ -9,11 +11,14 @@ lighteval sglang \
"leaderboard|truthfulqa:mc|0"
```
-`sglang` is able to distribute the model across multiple GPUs using data
-parallelism and tensor parallelism.
-You can choose the parallelism method by setting in the `model_args`.
+## Parallelism Options
+
+SGLang can distribute the model across multiple GPUs using data parallelism and tensor parallelism.
+You can choose the parallelism method by setting the appropriate parameters in the `model_args`.
+
+### Tensor Parallelism
-For example if you have 4 GPUs you can split it across using `tp_size`:
+For example, if you have 4 GPUs, you can split the model across them using tensor parallelism with `tp_size`:
```bash
lighteval sglang \
@@ -21,7 +26,9 @@ lighteval sglang \
"leaderboard|truthfulqa:mc|0"
```
-Or, if your model fits on a single GPU, you can use `dp_size` to speed up the evaluation:
+### Data Parallelism
+
+If your model fits on a single GPU, you can use data parallelism with `dp_size` to speed up the evaluation:
```bash
lighteval sglang \
@@ -29,10 +36,10 @@ lighteval sglang \
"leaderboard|truthfulqa:mc|0"
```
-## Use a config file
+## Using a Configuration File
-For more advanced configurations, you can use a config file for the model.
-An example of a config file is shown below and can be found at `examples/model_configs/sglang_model_config.yaml`.
+For more advanced configurations, you can use a YAML configuration file for the model.
+An example configuration file is shown below and can be found at `examples/model_configs/sglang_model_config.yaml`.
```bash
lighteval sglang \
@@ -41,7 +48,7 @@ lighteval sglang \
```
> [!TIP]
-> Documentation for the config file of sglang can be found [here](https://docs.sglang.ai/backend/server_arguments.html)
+> Documentation for SGLang server arguments can be found [here](https://docs.sglang.ai/backend/server_arguments.html)
```yaml
model_parameters:
@@ -74,5 +81,70 @@ model_parameters:
```
> [!WARNING]
-> In the case of OOM issues, you might need to reduce the context size of the
-> model as well as reduce the `mem_fraction_static` and `chunked_prefill_size` parameter.
+> In case of out-of-memory (OOM) issues, you might need to reduce the context size of the
+> model as well as reduce the `mem_fraction_static` and `chunked_prefill_size` parameters.
+
+## Key SGLang Parameters
+
+### Memory Management
+- `mem_fraction_static`: Fraction of GPU memory to allocate for static tensors (default: 0.8)
+- `chunked_prefill_size`: Size of chunks for prefill operations (default: 4096)
+- `context_length`: Maximum context length for the model
+- `kv_cache_dtype`: Data type for key-value cache
+
+### Parallelism Settings
+- `tp_size`: Number of GPUs for tensor parallelism
+- `dp_size`: Number of GPUs for data parallelism
+
+### Model Configuration
+- `dtype`: Data type for model weights ("auto", "float16", "bfloat16", etc.)
+- `device`: Device to run the model on ("cuda", "cpu")
+- `trust_remote_code`: Whether to trust remote code from the model
+- `skip_tokenizer_init`: Skip tokenizer initialization for faster startup
+
+### Generation Parameters
+- `temperature`: Controls randomness in generation (0.0 = deterministic, 1.0 = random)
+- `top_p`: Nucleus sampling parameter
+- `top_k`: Top-k sampling parameter
+- `max_new_tokens`: Maximum number of tokens to generate
+- `repetition_penalty`: Penalty for repeating tokens
+- `presence_penalty`: Penalty for token presence
+- `frequency_penalty`: Penalty for token frequency
+
+## Advanced Features
+
+### Backend Selection
+SGLang supports different backends for sampling and attention:
+
+```yaml
+model_parameters:
+ sampling_backend: "vllm" # or "hf", "hf_legacy"
+ attention_backend: "flash" # or "xformers", "native"
+```
+
+### Optimizations
+- `pairwise_tokenization`: Enable pairwise tokenization for multiple choice tasks
+- `add_special_tokens`: Add special tokens during tokenization
+- `skip_tokenizer_init`: Skip tokenizer initialization for faster startup
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Out of Memory Errors**: Reduce `mem_fraction_static` or `chunked_prefill_size`
+2. **Model Loading Errors**: Check that the model name and configuration are correct
+3. **Performance Issues**: Ensure appropriate backend selection for your hardware
+
+### Performance Tips
+
+- Use tensor parallelism for large models that don't fit on a single GPU
+- Use data parallelism for smaller models to increase throughput
+- Adjust `mem_fraction_static` based on your GPU memory capacity
+- Consider using `float16` or `bfloat16` for faster inference
+- Use appropriate sampling and attention backends for your hardware
+
+### Memory Optimization
+
+- Reduce `mem_fraction_static` if you encounter OOM errors
+- Adjust `chunked_prefill_size` based on your memory constraints
+- Use `skip_tokenizer_init=True` for faster startup if tokenizer is not needed
diff --git a/docs/source/use-vllm-as-backend.mdx b/docs/source/use-vllm-as-backend.mdx
index 05ec1edde..ebca812e5 100644
--- a/docs/source/use-vllm-as-backend.mdx
+++ b/docs/source/use-vllm-as-backend.mdx
@@ -1,11 +1,12 @@
-# Use VLLM as backend
-
-Lighteval allows you to use `vllm` as backend allowing great speedups.
-To use, simply change the `model_args` to reflect the arguments you want to pass to vllm.
+# Using VLLM as Backend
+Lighteval allows you to use VLLM as a backend, providing significant speedups for model evaluation.
+To use VLLM, simply change the `model_args` to reflect the arguments you want to pass to VLLM.
> [!TIP]
-> Documentation for vllm engine args can be found [here](https://docs.vllm.ai/en/latest/serving/engine_args.html)
+> Documentation for VLLM engine arguments can be found [here](https://docs.vllm.ai/en/latest/serving/engine_args.html)
+
+## Basic Usage
```bash
lighteval vllm \
@@ -13,11 +14,14 @@ lighteval vllm \
"extended|ifeval|0"
```
-`vllm` is able to distribute the model across multiple GPUs using data
-parallelism, pipeline parallelism or tensor parallelism.
-You can choose the parallelism method by setting in the `model_args`.
+## Parallelism Options
+
+VLLM can distribute the model across multiple GPUs using data parallelism, pipeline parallelism, or tensor parallelism.
+You can choose the parallelism method by setting the appropriate parameters in the `model_args`.
-For example if you have 4 GPUs you can split it across using `tensor_parallelism`:
+### Tensor Parallelism
+
+For example, if you have 4 GPUs, you can split the model across them using tensor parallelism:
```bash
export VLLM_WORKER_MULTIPROC_METHOD=spawn && lighteval vllm \
@@ -25,7 +29,9 @@ export VLLM_WORKER_MULTIPROC_METHOD=spawn && lighteval vllm \
"extended|ifeval|0"
```
-Or, if your model fits on a single GPU, you can use `data_parallelism` to speed up the evaluation:
+### Data Parallelism
+
+If your model fits on a single GPU, you can use data parallelism to speed up the evaluation:
```bash
export VLLM_WORKER_MULTIPROC_METHOD=spawn && lighteval vllm \
@@ -33,10 +39,10 @@ export VLLM_WORKER_MULTIPROC_METHOD=spawn && lighteval vllm \
"extended|ifeval|0"
```
-## Use a config file
+## Using a Configuration File
-For more advanced configurations, you can use a config file for the model.
-An example of a config file is shown below and can be found at `examples/model_configs/vllm_model_config.yaml`.
+For more advanced configurations, you can use a YAML configuration file for the model.
+An example configuration file is shown below and can be found at `examples/model_configs/vllm_model_config.yaml`.
```bash
lighteval vllm \
@@ -76,50 +82,40 @@ model_parameters:
```
> [!WARNING]
-> In the case of OOM issues, you might need to reduce the context size of the
+> In case of out-of-memory (OOM) issues, you might need to reduce the context size of the
> model as well as reduce the `gpu_memory_utilization` parameter.
-## Dynamically changing the metric configuration
+## Key VLLM Parameters
-For special kinds of metrics like `Pass@K` or LiveCodeBench's `codegen` metric, you may need to pass specific values like the number of
-generations. This can be done in the `yaml` file in the following way:
+### Memory Management
+- `gpu_memory_utilization`: Controls how much GPU memory VLLM can use (default: 0.9)
+- `max_model_length`: Maximum sequence length for the model
+- `swap_space`: Amount of CPU memory to use for swapping (in GB)
-```yaml
-model_parameters:
- model_name: "HuggingFaceTB/SmolLM-1.7B-Instruct"
- revision: "main"
- dtype: "bfloat16"
- tensor_parallel_size: 1
- data_parallel_size: 1
- pipeline_parallel_size: 1
- gpu_memory_utilization: 0.9
- max_model_length: 2048
- swap_space: 4
- seed: 1
- trust_remote_code: True
- add_special_tokens: True
- multichoice_continuations_start_space: True
- pairwise_tokenization: True
- subfolder: null
- generation_parameters:
- presence_penalty: 0.0
- repetition_penalty: 1.0
- frequency_penalty: 0.0
- temperature: 1.0
- top_k: 50
- min_p: 0.0
- top_p: 1.0
- seed: 42
- stop_tokens: null
- max_new_tokens: 1024
- min_new_tokens: 0
-metric_options: # Optional metric arguments
- codegen_pass@1:16:
- num_samples: 16
-```
+### Parallelism Settings
+- `tensor_parallel_size`: Number of GPUs for tensor parallelism
+- `data_parallel_size`: Number of GPUs for data parallelism
+- `pipeline_parallel_size`: Number of GPUs for pipeline parallelism
+
+### Generation Parameters
+- `temperature`: Controls randomness in generation (0.0 = deterministic, 1.0 = random)
+- `top_p`: Nucleus sampling parameter
+- `top_k`: Top-k sampling parameter
+- `max_new_tokens`: Maximum number of tokens to generate
+- `repetition_penalty`: Penalty for repeating tokens
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Out of Memory Errors**: Reduce `gpu_memory_utilization` or `max_model_length`
+2. **Worker Process Issues**: Ensure `VLLM_WORKER_MULTIPROC_METHOD=spawn` is set for multi-GPU setups
+3. **Model Loading Errors**: Check that the model name and revision are correct
+
+### Performance Tips
-An optional key `metric_options` can be passed in the yaml file,
-using the name of the metric or metrics, as defined in the `Metric.metric_name`.
-In this case, the `codegen_pass@1:16` metric defined in our tasks will have the `num_samples` updated to 16,
-independently of the number defined by default.
+- Use tensor parallelism for large models that don't fit on a single GPU
+- Use data parallelism for smaller models to increase throughput
+- Adjust `gpu_memory_utilization` based on your GPU memory capacity
+- Consider using `bfloat16` or `float16` for faster inference
diff --git a/docs/source/using-the-python-api.mdx b/docs/source/using-the-python-api.mdx
index 28927d8de..847eb4559 100644
--- a/docs/source/using-the-python-api.mdx
+++ b/docs/source/using-the-python-api.mdx
@@ -1,6 +1,6 @@
# Using the Python API
-Lighteval can be used from a custom python script. To evaluate a model you will need to set up an
+Lighteval can be used from a custom Python script. To evaluate a model, you will need to set up an
[`~logging.evaluation_tracker.EvaluationTracker`], [`~pipeline.PipelineParameters`],
a [`model`](package_reference/models) or a [`model_config`](package_reference/model_config),
and a [`~pipeline.Pipeline`].
@@ -26,19 +26,19 @@ def main():
output_dir="./results",
save_details=True,
push_to_hub=True,
- hub_results_org="your user name",
+ hub_results_org="your_username", # Replace with your actual username
)
pipeline_params = PipelineParameters(
launcher_type=ParallelismManager.ACCELERATE,
- custom_tasks_directory=None, # if using a custom task
- # Remove the 2 parameters below once your configuration is tested
+ custom_tasks_directory=None, # Set to path if using custom tasks
+ # Remove the parameter below once your configuration is tested
max_samples=10
)
model_config = VLLMModelConfig(
- model_name="HuggingFaceH4/zephyr-7b-beta",
- dtype="float16",
+ model_name="HuggingFaceH4/zephyr-7b-beta",
+ dtype="float16",
)
task = "helm|mmlu|5"
@@ -57,3 +57,47 @@ def main():
if __name__ == "__main__":
main()
```
+
+## Key Components
+
+### EvaluationTracker
+The `EvaluationTracker` handles logging and saving evaluation results. It can save results locally and optionally push them to the Hugging Face Hub.
+
+### PipelineParameters
+`PipelineParameters` configures how the evaluation pipeline runs, including parallelism settings and task configuration.
+
+### Model Configuration
+Model configurations define the model to be evaluated, including the model name, data type, and other model-specific parameters. Different backends (VLLM, Transformers, etc.) have their own configuration classes.
+
+### Pipeline
+The `Pipeline` orchestrates the entire evaluation process, taking the tasks, model configuration, and parameters to run the evaluation.
+
+## Running Multiple Tasks
+
+You can evaluate on multiple tasks by providing a comma-separated list or a file path:
+
+```python
+# Multiple tasks as comma-separated string
+tasks = "helm|mmlu|5|1,helm|truthfulqa:mc|0|0"
+
+# Or load from a file
+tasks = "./path/to/tasks.txt"
+
+pipeline = Pipeline(
+ tasks=tasks,
+ # ... other parameters
+)
+```
+
+## Custom Tasks
+
+To use custom tasks, set the `custom_tasks_directory` parameter to the path containing your custom task definitions:
+
+```python
+pipeline_params = PipelineParameters(
+ custom_tasks_directory="./path/to/custom/tasks",
+ # ... other parameters
+)
+```
+
+For more information on creating custom tasks, see the [Adding a Custom Task](adding-a-custom-task) guide.
diff --git a/examples/model_configs/vllm_model_config.yaml b/examples/model_configs/vllm_model_config.yaml
index ec41d3fda..8518b78cb 100644
--- a/examples/model_configs/vllm_model_config.yaml
+++ b/examples/model_configs/vllm_model_config.yaml
@@ -21,7 +21,7 @@ model_parameters:
presence_penalty: 0.0
repetition_penalty: 1.0
frequency_penalty: 0.0
- temperature: 0.0
+ temperature: 0.1
top_k: null
min_p: 0.0
top_p: 0.9
diff --git a/pyproject.toml b/pyproject.toml
index 97a1745d6..69733f665 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,11 +5,13 @@ line-length = 119
[tool.ruff.lint]
# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
# Never enforce `E501` (line length violations).
-ignore = ["E501"]
-select = ["C", "E", "F", "I", "W", "CPY"]
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+ignore = ["E501", "D100", "D101", "D102", "D103", "D104", "D415", "D105", "DOC501", "DOC201"]
+select = ["C", "E", "F", "I", "W", "CPY", "D417", "DOC"]
preview = true
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
[tool.ruff.lint.isort]
lines-after-imports = 2
known-first-party = ["lighteval"]
diff --git a/src/lighteval/cli_args.py b/src/lighteval/cli_args.py
index 472941ad7..30e85a1a9 100644
--- a/src/lighteval/cli_args.py
+++ b/src/lighteval/cli_args.py
@@ -20,8 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
-"""
-Common CLI argument types for LightEval main files.
+"""Common CLI argument types for LightEval main files.
This module exports pre-defined argument types to reduce redundancy across main_*.py files.
"""
diff --git a/src/lighteval/data.py b/src/lighteval/data.py
index eae2bb39a..50e841c6c 100644
--- a/src/lighteval/data.py
+++ b/src/lighteval/data.py
@@ -47,8 +47,7 @@ def __init__(
requests: list[Doc],
num_dataset_splits: int,
):
- """
- This dataset class uses dynamic batching to speed up the generation.
+ """This dataset class uses dynamic batching to speed up the generation.
Each request is sorted by the length of the prompt + the length of the
continuation. Then, the dataset is split into num_dataset_splits splits.
The first split will contain the longest requests, the second split will
@@ -87,11 +86,10 @@ def init_split_limits(self, num_dataset_splits):
return num_dataset_splits, splits_indices
def get_original_order(self, new_arr: list) -> list:
- """
- Get the original order of the data.
+ """Get the original order of the data.
Args:
- newarr (list): Array containing any kind of data that needs to be
+ new_arr (list): Array containing any kind of data that needs to be
reset in the original order.
Returns:
@@ -110,8 +108,7 @@ def get_original_order(self, new_arr: list) -> list:
return original_order
def splits_iterator(self) -> Iterator[Subset]:
- """
- Iterator that yields the dataset splits based on the split limits.
+ """Iterator that yields the dataset splits based on the split limits.
Yields:
Subset: A subset of the dataset.
@@ -124,8 +121,7 @@ def splits_iterator(self) -> Iterator[Subset]:
yield Subset(self, range(split_start, split_end))
def __getitem__(self, index) -> Doc:
- """
- Get an item from the dataset.
+ """Get an item from the dataset.
Args:
index (int): The index of the item.
@@ -136,8 +132,7 @@ def __getitem__(self, index) -> Doc:
return self.sorted_data[index]
def __len__(self) -> int:
- """
- Get the length of current split the dataset.
+ """Get the length of current split the dataset.
All splits have the same length, except the last one which might be
shorter.
@@ -147,8 +142,7 @@ def __len__(self) -> int:
return len(self.sorted_data)
def __iter__(self) -> Iterator[Doc]:
- """
- Iterator that yields the items of the dataset depending on the split we
+ """Iterator that yields the items of the dataset depending on the split we
are currently in. For instance, if we are in split 0, we will get the
items from index 0 to self.split_size, if we are in split 1, we will get
the items from index self.split_size to 2 * self.split_size, etc. Used
@@ -166,8 +160,7 @@ def _sorting_criteria(self, doc: Doc):
class LoglikelihoodDataset(DynamicBatchDataset):
def _sorting_criteria(self, doc: Doc) -> int:
- """
- Collates the input data for batching.
+ """Collates the input data for batching.
the negative sign on len(toks) sorts descending - this has a few
advantages:
@@ -180,10 +173,10 @@ def _sorting_criteria(self, doc: Doc) -> int:
- any OOMs will happen right away rather than near the end
Args:
- x (tuple): A tuple containing the input data.
+ doc (Doc): The document containing query and choices.
Returns:
- tuple: A tuple containing the sorted input data.
+ int: Negative sum of query length and max choice length for sorting.
"""
len_doc_query = len(doc.query)
max_len_choices = max(len(choice) for choice in doc.choices) if doc.choices else 0
@@ -233,14 +226,13 @@ def init_split_limits(self, num_dataset_splits):
return num_dataset_splits, splits_indices
def _sorting_criteria(self, doc: Doc) -> tuple[int, bool, tuple, int, int]:
- """
- Collate function for generating batches.
+ """Collate function for generating batches.
Args:
- x (Any): The input data.
+ doc (Doc): The document containing generation parameters.
Returns:
- Any: The collated data.
+ tuple: A tuple containing (num_samples, use_logits, stop_sequences, gen_length, negative_total_length).
"""
query = doc.query
gen_length = doc.generation_size
@@ -261,8 +253,7 @@ def _sorting_criteria(self, doc: Doc) -> tuple[int, bool, tuple, int, int]:
class GenerativeTaskDatasetNanotron(GenerativeTaskDataset):
def __getitem__(self, index) -> tuple[int, Doc]:
- """
- Get an item from the dataset depending on the split we are currently in.
+ """Get an item from the dataset depending on the split we are currently in.
For instance, if we are in split 0, we will get the item at index 0, if
we are in split 1, we will get the item at index self.split_size, etc.
Used for dynamic batching.
diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py
index 2bbe9bfdf..108877601 100644
--- a/src/lighteval/logging/evaluation_tracker.py
+++ b/src/lighteval/logging/evaluation_tracker.py
@@ -59,8 +59,7 @@
class EnhancedJSONEncoder(json.JSONEncoder):
- """
- Provides a proper json encoding for the loggers and trackers json dumps.
+ """Provides a proper json encoding for the loggers and trackers json dumps.
Notably manages the json encoding of dataclasses.
"""
@@ -91,38 +90,50 @@ def default(self, o):
class EvaluationTracker:
- """Keeps track of the overall evaluation process and relevant information.
+ """Tracks and manages evaluation results, metrics, and logging for model evaluations.
+
+ The EvaluationTracker coordinates multiple specialized loggers to track different aspects of model evaluation:
- The [`~logging.evaluation_tracker.EvaluationTracker`] contains specific loggers for experiments details
- ([`~logging.evaluation_tracker.DetailsLogger`]), metrics ([`~logging.evaluation_tracker.MetricsLogger`]), task versions
- ([`~logging.evaluation_tracker.VersionsLogger`]) as well as for the general configurations of both the
- specific task ([`~logging.evaluation_tracker.TaskConfigLogger`]) and overall evaluation run
- ([`~logging.evaluation_tracker.GeneralConfigLogger`]). It compiles the data from these loggers and
- writes it to files, which can be published to the Hugging Face hub if
- requested.
+ - Details Logger (DetailsLogger): Records per-sample evaluation details and predictions
+ - Metrics Logger (MetricsLogger): Tracks aggregate evaluation metrics and scores
+ - Versions Logger (VersionsLogger): Records task and dataset versions
+ - General Config Logger (GeneralConfigLogger): Stores overall evaluation configuration
+ - Task Config Logger (TaskConfigLogger): Maintains per-task configuration details
+
+ The tracker can save results locally and optionally push them to:
+ - Hugging Face Hub as datasets
+ - TensorBoard for visualization
+ - Trackio or Weights & Biases for experiment tracking
Args:
- output_dir (`str`): Local folder path where you want results to be saved.
- results_path_template (`str`, *optional*): template to use for the results output directory. for example,
- `"{output_dir}/results_this_time_it_will_work/{org}_{model}"` will create a folder named `results` in the output directory
- with the model name and the organization name.
- save_details (`bool`, defaults to True): If True, details are saved to the `output_dir`.
- push_to_hub (`bool`, defaults to False): If True, details are pushed to the hub.
- Results are pushed to `{hub_results_org}/details__{sanitized model_name}` for the model `model_name`, a public dataset,
- if `public` is True else `{hub_results_org}/details__{sanitized model_name}_private`, a private dataset.
- push_to_tensorboard (`bool`, defaults to False): If True, will create and push the results for a tensorboard folder on the hub.
- hub_results_org (`str`, *optional*): The organisation to push the results to.
- See more details about the datasets organisation in [`EvaluationTracker.save`].
- tensorboard_metric_prefix (`str`, defaults to "eval"): Prefix for the metrics in the tensorboard logs.
- public (`bool`, defaults to False): If True, results and details are pushed to public orgs.
- nanotron_run_info ([`~nanotron.config.GeneralArgs`], *optional*): Reference to information about Nanotron models runs.
-
- **Attributes**:
- - **details_logger** ([`~logging.info_loggers.DetailsLogger`]) -- Logger for experiment details.
- - **metrics_logger** ([`~logging.info_loggers.MetricsLogger`]) -- Logger for experiment metrics.
- - **versions_logger** ([`~logging.info_loggers.VersionsLogger`]) -- Logger for task versions.
- - **general_config_logger** ([`~logging.info_loggers.GeneralConfigLogger`]) -- Logger for general configuration.
- - **task_config_logger** ([`~logging.info_loggers.TaskConfigLogger`]) -- Logger for task configuration.
+ output_dir (str): Local directory to save evaluation results and logs
+ results_path_template (str, optional): Template for results directory structure.
+ Example: "{output_dir}/results/{org}_{model}"
+ save_details (bool, defaults to True): Whether to save detailed evaluation records
+ push_to_hub (bool, defaults to False): Whether to push results to HF Hub
+ push_to_tensorboard (bool, defaults to False): Whether to push metrics to TensorBoard
+ hub_results_org (str, optional): HF Hub organization to push results to
+ tensorboard_metric_prefix (str, defaults to "eval"): Prefix for TensorBoard metrics
+ public (bool, defaults to False): Whether to make Hub datasets public
+ nanotron_run_info (GeneralArgs, optional): Nanotron model run information
+ use_wandb (bool, defaults to False): Whether to log to Weights & Biases or Trackio if available
+
+ Example:
+ ```python
+ tracker = EvaluationTracker(
+ output_dir="./eval_results",
+ push_to_hub=True,
+ hub_results_org="my-org",
+ save_details=True
+ )
+
+ # Log evaluation results
+ tracker.metrics_logger.add_metric("accuracy", 0.85)
+ tracker.details_logger.add_detail(task_name="qa", prediction="Paris")
+
+ # Save all results
+ tracker.save()
+ ```
"""
def __init__(
@@ -350,6 +361,9 @@ def generate_final_dict(self) -> dict:
"""Aggregates and returns all the logger's experiment information in a dictionary.
This function should be used to gather and display said information at the end of an evaluation run.
+
+ Returns:
+ dict: Dictionary containing all experiment information including config, results, versions, and summaries
"""
to_dump = {
"config_general": asdict(self.general_config_logger),
diff --git a/src/lighteval/logging/info_loggers.py b/src/lighteval/logging/info_loggers.py
index 055670657..64019ecf8 100644
--- a/src/lighteval/logging/info_loggers.py
+++ b/src/lighteval/logging/info_loggers.py
@@ -25,7 +25,6 @@
import os
import time
from dataclasses import asdict, dataclass, field
-from typing import Union
import git
import xxhash
@@ -47,24 +46,39 @@
@dataclass(init=False)
class GeneralConfigLogger:
- """Logger for the evaluation parameters.
+ """Tracks general configuration and runtime information for model evaluations.
+
+ This logger captures key configuration parameters, model details, and timing information
+ to ensure reproducibility and provide insights into the evaluation process.
Attributes:
- lighteval_sha (str): Current commit sha of lighteval used for the evaluation (for reproducibility purposes)
- num_fewshot_seeds (int): Number of seeds for the few-shot sampling.
- If equal to or below 1, the experiment is done once only, with a single few-shot seed (equal to 0).
- If above, the experiment is reproduced several times, with a different sampling/shuffling for the few-shot examples, which follows what is done in HELM for example.
- override_batch_size (int): Manages the batch size.
- If strictly positive, its value is used as the batch size for all experiments.
- Else, the batch size is automatically inferred depending on what fits in memory.
- max_samples (int): If set, cuts the number of samples per task to `max_samples`.
- Note: This should only be used for debugging purposes!
- job_id (int): If the evaluation suite is launched as a slurm job, stores the current job id.
- Purely informative parameter used to retrieve scheduler logs.
- start_time (float): Start time of the experiment. Logged at class init.
- end_time (float): End time of the experiment. Logged when calling [`GeneralConfigLogger.log_end_time`]
- total_evaluation_time_secondes (str): Inferred total evaluation time in seconds (from the start and end times).
- model_config (ModelConfig): Model configuration
+ lighteval_sha (str): Git commit SHA of lighteval used for evaluation, enabling exact version reproducibility.
+ Set to "?" if not in a git repository.
+
+ num_fewshot_seeds (int): Number of random seeds used for few-shot example sampling.
+ - If <= 1: Single evaluation with seed=0
+ - If > 1: Multiple evaluations with different few-shot samplings (HELM-style)
+
+ max_samples (int, optional): Maximum number of samples to evaluate per task.
+ Only used for debugging - truncates each task's dataset.
+
+ job_id (int, optional): Slurm job ID if running on a cluster.
+ Used to cross-reference with scheduler logs.
+
+ start_time (float): Unix timestamp when evaluation started.
+ Automatically set during logger initialization.
+
+ end_time (float): Unix timestamp when evaluation completed.
+ Set by calling log_end_time().
+
+ total_evaluation_time_secondes (str): Total runtime in seconds.
+ Calculated as end_time - start_time.
+
+ model_config (ModelConfig): Complete model configuration settings.
+ Contains model architecture, tokenizer, and generation parameters.
+
+ model_name (str): Name identifier for the evaluated model.
+ Extracted from model_config.
"""
# general
@@ -92,18 +106,14 @@ def __init__(self) -> None:
def log_args_info(
self,
num_fewshot_seeds: int,
- max_samples: Union[None, int],
+ max_samples: int | None,
job_id: str,
) -> None:
- """
- Logs the information about the arguments passed to the method.
+ """Logs the information about the arguments passed to the method.
Args:
num_fewshot_seeds (int): number of few-shot seeds.
- override_batch_size (Union[None, int]): overridden batch size.
- If strictly positive, its value is used as the batch size for all experiments.
- Else, the batch size is automatically inferred depending on what fits in memory.
- max_samples (Union[None, int]): maximum number of samples, if None, use all the samples available.
+ max_samples (int | None): maximum number of samples, if None, use all the samples available.
job_id (str): job ID, used to retrieve logs.
"""
self.num_fewshot_seeds = num_fewshot_seeds
@@ -111,12 +121,10 @@ def log_args_info(
self.job_id = job_id
def log_model_info(self, model_config: ModelConfig) -> None:
- """
- Logs the model information.
+ """Logs the model information.
Args:
model_config: the model config used to initialize the model.
-
"""
self.model_config = model_config
self.model_name = model_config.model_name
@@ -199,8 +207,7 @@ class CompiledDetailOverAllTasks:
@dataclass
class Hash:
- """
- Hashes important values for one sample ([`Doc`]) of one task ([`LightevalTask`])
+ """Hashes important values for one sample ([`Doc`]) of one task ([`LightevalTask`])
Attributes:
example (str): Hash of the [`Doc.query`]
@@ -217,8 +224,7 @@ class Hash:
@dataclass
class CompiledHash:
- """
- Hashes the aggregated hash values for all the sample ([`Doc`]) of one task ([`LightevalTask`])
+ """Hashes the aggregated hash values for all the sample ([`Doc`]) of one task ([`LightevalTask`])
Attributes:
example (str): Aggregated hash of all the [`Doc.query`] hashes for all samples of the current task.
@@ -255,12 +261,9 @@ def log(
Args:
task_name (str): Name of the current task of interest.
- task (LightevalTask): Current task of interest.
doc (Doc): Current sample that we want to store.
- outputs (list[ModelResponse]): Model outputs for the current sample
- metrics (_type_): Model scores for said sample on the current task's metrics.
- llm_as_prompt_judgement (tuple[str, str]): Tuple containing the
- prompt passed to the judge and the judgement for the current sample when using llm-as-judge metric.
+ model_response (ModelResponse): Model outputs for the current sample
+ metrics (dict): Model scores for said sample on the current task's metrics.
"""
detail = self.Detail(doc, model_response, metrics)
self.details[task_name].append(detail)
@@ -272,11 +275,7 @@ def log(
self.hashes[task_name].append(hash)
def aggregate(self):
- """
- Aggregate the details and hashes for each task and then for all tasks.
- We end up with a dict of compiled details for each task and a dict of compiled details for all tasks.
- """
-
+ """Hashes the details for each task and then for all tasks."""
for task_name in self.hashes:
compiled_hash = self.CompiledHash()
compiled_hash.hash_examples = xxhash.xxh64(
@@ -329,15 +328,12 @@ def log(self, task_name: str, metrics: dict) -> None:
self.metrics_values[task_name][metric_name].append(metric_value)
def aggregate(self, task_dict: dict[str, LightevalTask], bootstrap_iters: int = 1000): # noqa: C901
- """
- Aggregate the metrics for each task and then for all tasks.
+ """Aggregate the metrics for each task and then for all tasks.
Args:
task_dict (dict[str, LightevalTask]): used to determine what aggregation function to use for each metric
bootstrap_iters (int, optional): Number of runs used to run the statistical bootstrap. Defaults to 1000.
-
"""
-
for task_name, metrics in self.metrics_values.items():
task = task_dict[task_name]
diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py
index 1e5726f86..1559fd85d 100644
--- a/src/lighteval/main_accelerate.py
+++ b/src/lighteval/main_accelerate.py
@@ -78,8 +78,10 @@ def accelerate( # noqa C901
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate models using accelerate and transformers as backend.
+ """Evaluate models using accelerate and transformers as backend.
+
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
"""
import yaml
diff --git a/src/lighteval/main_baseline.py b/src/lighteval/main_baseline.py
index 2768a8a4d..f082af726 100644
--- a/src/lighteval/main_baseline.py
+++ b/src/lighteval/main_baseline.py
@@ -37,8 +37,7 @@ def baseline(
output_dir: output_dir.type = output_dir.default,
max_samples: max_samples.type = max_samples.default,
):
- """
- Compute baselines for given tasks.
+ """Compute baselines for given tasks.
It has been tested with generative and accuracy tasks, but may not work correctly for other task types.
diff --git a/src/lighteval/main_custom.py b/src/lighteval/main_custom.py
index 14507ae8d..1cef8f3dc 100644
--- a/src/lighteval/main_custom.py
+++ b/src/lighteval/main_custom.py
@@ -72,8 +72,10 @@ def custom(
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate custom models (can be anything).
+ """Evaluate custom models (can be anything).
+
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
"""
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py
index 7d40f1661..060b93822 100644
--- a/src/lighteval/main_endpoint.py
+++ b/src/lighteval/main_endpoint.py
@@ -84,8 +84,10 @@ def inference_endpoint(
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate models using inference-endpoints as backend.
+ """Evaluate models using inference-endpoints as backend.
+
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
"""
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.models.endpoints.endpoint_model import InferenceEndpointModelConfig, ServerlessEndpointModelConfig
@@ -165,8 +167,10 @@ def tgi(
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate models using TGI as backend.
+ """Evaluate models using TGI as backend.
+
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
"""
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.models.endpoints.tgi_model import TGIModelConfig
@@ -246,10 +250,11 @@ def litellm(
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate models using LiteLLM as backend.
- """
+ """Evaluate models using LiteLLM as backend.
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
+ """
import yaml
from lighteval.logging.evaluation_tracker import EvaluationTracker
@@ -337,10 +342,11 @@ def inference_providers(
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate models using HuggingFace's inference providers as backend.
- """
+ """Evaluate models using HuggingFace's inference providers as backend.
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
+ """
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.models.endpoints.inference_providers_model import (
InferenceProvidersModelConfig,
diff --git a/src/lighteval/main_nanotron.py b/src/lighteval/main_nanotron.py
index 06935e69c..1131aea33 100644
--- a/src/lighteval/main_nanotron.py
+++ b/src/lighteval/main_nanotron.py
@@ -45,9 +45,7 @@ def nanotron(
remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default,
reasoning_tags: reasoning_tags.type = reasoning_tags.default,
):
- """
- Evaluate models using nanotron as backend.
- """
+ """Evaluate models using nanotron as backend."""
from lighteval.utils.imports import NO_NANOTRON_ERROR_MSG, is_nanotron_available
if not is_nanotron_available():
diff --git a/src/lighteval/main_sglang.py b/src/lighteval/main_sglang.py
index 135396263..0b506988e 100644
--- a/src/lighteval/main_sglang.py
+++ b/src/lighteval/main_sglang.py
@@ -66,8 +66,10 @@ def sglang(
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate models using sglang as backend.
+ """Evaluate models using sglang as backend.
+
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
"""
import yaml
diff --git a/src/lighteval/main_tasks.py b/src/lighteval/main_tasks.py
index 0f3cd3df1..62f1129f4 100644
--- a/src/lighteval/main_tasks.py
+++ b/src/lighteval/main_tasks.py
@@ -38,9 +38,7 @@ def inspect(
num_samples: Annotated[int, Option(help="Number of samples to display")] = 10,
show_config: Annotated[bool, Option(help="Will display the full task config")] = False,
):
- """
- Inspect a tasks
- """
+ """Inspect a tasks"""
from dataclasses import asdict
from pprint import pformat
@@ -74,9 +72,7 @@ def list(
),
] = None,
):
- """
- List all tasks
- """
+ """List all tasks"""
from lighteval.tasks.registry import Registry
registry = Registry(custom_tasks=custom_tasks, load_community=True, load_extended=True, load_multilingual=True)
@@ -85,9 +81,7 @@ def list(
@app.command()
def create(template: str, task_name: str, dataset_name: str):
- """
- Create a new task
- """
+ """Create a new task"""
logger = logging.getLogger(__name__)
logger.info(f"Creating task for dataset {dataset_name}")
diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py
index 45e40fd70..581e9e5f5 100644
--- a/src/lighteval/main_vllm.py
+++ b/src/lighteval/main_vllm.py
@@ -75,8 +75,10 @@ def vllm(
max_samples: max_samples.type = max_samples.default,
job_id: job_id.type = job_id.default,
):
- """
- Evaluate models using vllm as backend.
+ """Evaluate models using vllm as backend.
+
+ Returns:
+ dict: Evaluation results containing metrics and scores for all tasks
"""
import yaml
diff --git a/src/lighteval/metrics/dynamic_metrics.py b/src/lighteval/metrics/dynamic_metrics.py
index 39f1010bc..66ed91c3a 100644
--- a/src/lighteval/metrics/dynamic_metrics.py
+++ b/src/lighteval/metrics/dynamic_metrics.py
@@ -57,9 +57,7 @@
class LogLikelihoodAccMetric(SampleLevelMetric):
def __init__(self, normalization: LogProbNormalization | None = None):
- """
- Creates an accuracy (loglikelihood) metric, which returns accuracy given normalization.
- """
+ """Creates an accuracy (loglikelihood) metric, which returns accuracy given normalization."""
super().__init__(
metric_name="acc" + (f"_{normalization.name}" if normalization else ""),
sample_level_fn=LoglikelihoodAcc(logprob_normalization=normalization),
@@ -75,9 +73,7 @@ def __init__(
normalization: LogProbNormalization | None = None,
aggregation_function: Callable[[np.ndarray], float] = np.max,
):
- """
- Creates a normalized multi-choice probability metric, which returns the probability of the gold choice / sum of probabilities of all choices (after logprobs are normalized).
- """
+ """Creates a normalized multi-choice probability metric, which returns the probability of the gold choice / sum of probabilities of all choices (after logprobs are normalized)."""
super().__init__(
metric_name="normalized_mc_prob" + (f"_{normalization.name}" if normalization else ""),
sample_level_fn=NormalizedMultiChoiceProbability(
@@ -95,9 +91,7 @@ def __init__(
normalization: LogProbTokenNorm | None = None,
aggregation_function: Callable[[np.ndarray], float] = np.max,
):
- """
- Creates a probability metric, which returns the probability of the gold choice given normalization.
- """
+ """Creates a probability metric, which returns the probability of the gold choice given normalization."""
super().__init__(
metric_name="prob" + (f"_{normalization.name}" if normalization else ""),
sample_level_fn=Probability(normalization=normalization, aggregation_function=aggregation_function),
@@ -109,8 +103,7 @@ def __init__(
class MultilingualQuasiF1ScoreMetric(SampleLevelMetric):
def __init__(self, language: Language, aggregation_function: Callable[[list[float]], float] = max):
- """
- Creates a language-aware F1 score metric, which returns the F1 score.
+ """Creates a language-aware F1 score metric, which returns the F1 score.
Args:
language: The language of the samples.
@@ -136,8 +129,7 @@ def __init__(
match_type: Literal["prefix", "suffix", "full"] = "full",
aggregation_function: Callable[[list[float]], float] = max,
):
- """
- Creates a language-aware exact match metric, which returns the exact match score
+ """Creates a language-aware exact match metric, which returns the exact match score
Args:
language: The language of the samples.
match_type: The type of match to use
@@ -145,8 +137,6 @@ def __init__(
- "suffix": Suffixes must match
- "full": Full strings must match
aggregation_function: Aggregation samples to use when multiple golds are present.
- Returns:
- Exact match metric.
"""
super().__init__(
metric_name=f"exact_match_{language.value}_{match_type}",
@@ -205,9 +195,6 @@ def __init__(
timeout_seconds: int
Timeout for the extraction (each attempt) and comparison. Defaults to 5.
- Returns:
- A sample level metric that extracts and compares mathematical expressions.
-
"""
self.language = language
self.gold_extraction_target = gold_extraction_target
diff --git a/src/lighteval/metrics/harness_compatibility/drop.py b/src/lighteval/metrics/harness_compatibility/drop.py
index 382d9ad08..93ff0028b 100644
--- a/src/lighteval/metrics/harness_compatibility/drop.py
+++ b/src/lighteval/metrics/harness_compatibility/drop.py
@@ -48,6 +48,9 @@ def compute(self, doc: Doc, model_response: ModelResponse): # noqa: C901
For more information, please refer to the section 5 of the DROP paper (https://aclanthology.org/N19-1246/).
Todo: this code is really hard to follow, simplify when possible
+
+ Returns:
+ dict: Dictionary containing 'em' (exact match) and 'f1' (F1 score) metrics
"""
max_em = 0
max_f1 = 0
@@ -74,12 +77,14 @@ def _answer_to_bags(self, answer: List[str]) -> Tuple[List[str], List[Set[str]]]
return normalized_spans, token_bags
def _get_metrics(self, predicted: List[str], gold: List[str]):
- """
- Takes a predicted answer and a gold answer (that are both either a string or a list of
+ """Takes a predicted answer and a gold answer (that are both either a string or a list of
strings), and returns exact match and the DROP F1 metric for the prediction. If you are
writing a script for evaluating objects in memory (say, the output of predictions during
validation, or while training), this is the function you want to call, after using
:func:`answer_json_to_strings` when reading the gold answer from the released data file.
+
+ Returns:
+ tuple: A tuple containing (exact_match, f1_score) where exact_match is a float and f1_score is a float
"""
pred_normalized_spans, pred_bags = self._answer_to_bags(predicted)
gold_normalized_spans, gold_bags = self._answer_to_bags(gold)
@@ -117,9 +122,11 @@ def _match_numbers_if_present(self, gold_bag: Set[str], predicted_bag: Set[str])
return False
def _align_bags(self, predicted: List[Set[str]], gold: List[Set[str]]) -> np.array:
- """
- Takes gold and predicted answer sets and first finds the optimal 1-1 alignment
+ """Takes gold and predicted answer sets and first finds the optimal 1-1 alignment
between them and gets maximum metric values over all the answers.
+
+ Returns:
+ np.array: Array of maximum F1 scores for each aligned bag pair
"""
scores = np.zeros([len(gold), len(predicted)])
for gold_index, gold_item in enumerate(gold):
diff --git a/src/lighteval/metrics/imports/bert_scorer.py b/src/lighteval/metrics/imports/bert_scorer.py
index 11d66eeca..b8025bf3f 100644
--- a/src/lighteval/metrics/imports/bert_scorer.py
+++ b/src/lighteval/metrics/imports/bert_scorer.py
@@ -49,7 +49,7 @@ def padding(arr, pad_token, dtype=torch.long):
def sent_encode(tokenizer, sent):
- "Encoding as sentence based on the tokenizer"
+ """Encoding as sentence based on the tokenizer"""
sent = sent.strip()
if sent == "":
return tokenizer.build_inputs_with_special_tokens([])
@@ -73,20 +73,14 @@ def bert_encode(model, x, attention_mask, all_layers=False):
def collate_idf(arr, tokenizer, idf_dict, device="cuda:0"):
- """
- Helper function that pads a list of sentences to have the same length and
+ """Helper function that pads a list of sentences to have the same length and
loads idf score for words in the sentences.
Args:
- - :param: `arr` (list of str): sentences to process.
- - :param: `tokenize` : a function that takes a string and return list
- of tokens.
- - :param: `numericalize` : a function that takes a list of tokens and
- return list of token indexes.
- - :param: `idf_dict` (dict): mapping a word piece index to its
- inverse document frequency
- - :param: `pad` (str): the padding token.
- - :param: `device` (str): device to use, e.g. 'cpu' or 'cuda'
+ arr (list of str): sentences to process.
+ tokenizer: a tokenizer that takes a string and returns tokens.
+ idf_dict (dict): mapping a word piece index to its inverse document frequency.
+ device (str): device to use, e.g. 'cpu' or 'cuda'.
"""
arr = [sent_encode(tokenizer, a) for a in arr]
@@ -112,18 +106,20 @@ def get_bert_embedding(
device="cuda:0",
all_layers=False,
):
- """
- Compute BERT embedding in batches.
+ """Compute BERT embedding in batches.
Args:
- - :param: `all_sens` (list of str) : sentences to encode.
- - :param: `model` : a BERT model from `pytorch_pretrained_bert`.
- - :param: `tokenizer` : a BERT tokenizer corresponds to `model`.
- - :param: `idf_dict` (dict) : mapping a word piece index to its
- inverse document frequency
- - :param: `device` (str): device to use, e.g. 'cpu' or 'cuda'
+ all_sens (list of str): sentences to encode.
+ model: a BERT model from `pytorch_pretrained_bert`.
+ tokenizer: a BERT tokenizer corresponds to `model`.
+ idf_dict (dict): mapping a word piece index to its inverse document frequency.
+ batch_size (int): batch size for processing, -1 for all sentences.
+ device (str): device to use, e.g. 'cpu' or 'cuda'.
+ all_layers (bool): whether to return all layers or just the last layer.
+
+ Returns:
+ tuple: A tuple containing (total_embedding, mask, padded_idf)
"""
-
padded_sens, padded_idf, _, mask = collate_idf(all_sens, tokenizer, idf_dict, device=device)
if batch_size == -1:
@@ -155,26 +151,18 @@ def greedy_cos_idf(
hyp_idf,
all_layers=False,
):
- """
- Compute greedy matching based on cosine similarity.
+ """Compute greedy matching based on cosine similarity.
Args:
- - :param: `ref_embedding` (torch.Tensor):
- embeddings of reference sentences, BxKxd,
- B: batch size, K: longest length, d: bert dimension
- - :param: `ref_lens` (list of int): list of reference sentence length.
- - :param: `ref_masks` (torch.LongTensor): BxKxK, BERT attention mask for
- reference sentences.
- - :param: `ref_idf` (torch.Tensor): BxK, idf score of each word
- piece in the reference sentence
- - :param: `hyp_embedding` (torch.Tensor):
- embeddings of candidate sentences, BxKxd,
- B: batch size, K: longest length, d: bert dimension
- - :param: `hyp_lens` (list of int): list of candidate sentence length.
- - :param: `hyp_masks` (torch.LongTensor): BxKxK, BERT attention mask for
- candidate sentences.
- - :param: `hyp_idf` (torch.Tensor): BxK, idf score of each word
- piece in the candidate sentence
+ ref_embedding (torch.Tensor): embeddings of reference sentences, BxKxd,
+ B: batch size, K: longest length, d: bert dimension.
+ ref_masks (torch.LongTensor): BxKxK, BERT attention mask for reference sentences.
+ ref_idf (torch.Tensor): BxK, idf score of each word piece in the reference sentence.
+ hyp_embedding (torch.Tensor): embeddings of candidate sentences, BxKxd,
+ B: batch size, K: longest length, d: bert dimension.
+ hyp_masks (torch.LongTensor): BxKxK, BERT attention mask for candidate sentences.
+ hyp_idf (torch.Tensor): BxK, idf score of each word piece in the candidate sentence.
+ all_layers (bool): whether to use all layers or just the last layer.
"""
ref_embedding.div_(torch.norm(ref_embedding, dim=-1).unsqueeze(-1))
hyp_embedding.div_(torch.norm(hyp_embedding, dim=-1).unsqueeze(-1))
@@ -248,19 +236,18 @@ def bert_cos_score_idf(
device="cuda:0",
all_layers=False,
):
- """
- Compute BERTScore.
+ """Compute BERTScore.
Args:
- - :param: `model` : a BERT model in `pytorch_pretrained_bert`
- - :param: `refs` (list of str): reference sentences
- - :param: `hyps` (list of str): candidate sentences
- - :param: `tokenizer` : a BERT tokenizer corresponds to `model`
- - :param: `idf_dict` : a dictionary mapping a word piece index to its
- inverse document frequency
- - :param: `verbose` (bool): turn on intermediate status update
- - :param: `batch_size` (int): bert score processing batch size
- - :param: `device` (str): device to use, e.g. 'cpu' or 'cuda'
+ model: a BERT model in `pytorch_pretrained_bert`.
+ refs (list of str): reference sentences.
+ hyps (list of str): candidate sentences.
+ tokenizer: a BERT tokenizer corresponds to `model`.
+ idf_dict: a dictionary mapping a word piece index to its inverse document frequency.
+ verbose (bool): turn on intermediate status update.
+ batch_size (int): bert score processing batch size.
+ device (str): device to use, e.g. 'cpu' or 'cuda'.
+ all_layers (bool): whether to use all layers or just the last layer.
"""
preds = []
@@ -320,9 +307,7 @@ def length_to_mask(lens):
class BERTScorer:
- """
- BERTScore Scorer Object.
- """
+ """BERTScore Scorer Object."""
def __init__(
self,
@@ -337,27 +322,28 @@ def __init__(
rescale_with_baseline=False,
baseline_path=None,
):
- """
+ """Initialize BERTScorer.
+
Args:
- - :param: `model_type` (str): contextual embedding model specification, default using the suggested
- model for the target langauge; has to specify at least one of
- `model_type` or `lang`
- - :param: `num_layers` (int): the layer of representation to use.
- default using the number of layer tuned on WMT16 correlation data
- - :param: `verbose` (bool): turn on intermediate status update
- - :param: `idf` (bool): a booling to specify whether to use idf or not (this should be True even if `idf_sents` is given)
- - :param: `device` (str): on which the contextual embedding model will be allocated on.
- If this argument is None, the model lives on cuda:0 if cuda is available.
- - :param: `batch_size` (int): bert score processing batch size
- - :param: `nthreads` (int): number of threads
- - :param: `lang` (str): language of the sentences; has to specify
- at least one of `model_type` or `lang`. `lang` needs to be
- specified when `rescale_with_baseline` is True.
- - :param: `return_hash` (bool): return hash code of the setting
- - :param: `rescale_with_baseline` (bool): rescale bertscore with pre-computed baseline
- - :param: `baseline_path` (str): customized baseline file
+ model_type (str): Contextual embedding model specification, default using the suggested
+ model for the target language; has to specify at least one of
+ `model_type` or `lang`.
+ num_layers (int): The layer of representation to use.
+ Default using the number of layer tuned on WMT16 correlation data.
+ verbose (bool): Turn on intermediate status update.
+ idf (bool): A boolean to specify whether to use idf or not (this should be True even if `idf_sents` is given).
+ device (str): On which the contextual embedding model will be allocated on.
+ If this argument is None, the model lives on cuda:0 if cuda is available.
+ batch_size (int): BERT score processing batch size.
+ nthreads (int): Number of threads.
+ all_layers (bool): Whether to use all layers or just the last layer.
+ lang (str): Language of the sentences; has to specify
+ at least one of `model_type` or `lang`. `lang` needs to be
+ specified when `rescale_with_baseline` is True.
+ return_hash (bool): Return hash code of the setting.
+ rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline.
+ baseline_path (str): Customized baseline file.
"""
-
assert lang is not None or model_type is not None, "Either lang or model_type should be specified"
if rescale_with_baseline:
@@ -430,8 +416,7 @@ def baseline_vals(self):
return self._baseline_vals
def score(self, cands, refs, verbose=False, batch_size=64, return_hash=False):
- """
- Args:
+ """Args:
- :param: `cands` (list of str): candidate sentences
- :param: `refs` (list of str or list of list of str): reference sentences
@@ -442,7 +427,6 @@ def score(self, cands, refs, verbose=False, batch_size=64, return_hash=False):
multiple references, the returned score of this candidate is
the *best* score among all references.
"""
-
if self._model is None:
logger.info(f"Loading BERTScorer model `{self._model_type}`")
self._tokenizer = AutoTokenizer.from_pretrained(self._model_type)
diff --git a/src/lighteval/metrics/imports/data_stats_metric.py b/src/lighteval/metrics/imports/data_stats_metric.py
index b5ba1e5fa..bc2abf175 100644
--- a/src/lighteval/metrics/imports/data_stats_metric.py
+++ b/src/lighteval/metrics/imports/data_stats_metric.py
@@ -52,8 +52,7 @@ def find_ngrams(input_list, n):
class DataStatsMetric(Metric):
def __init__(self, n_gram=3, n_workers=24, case=False, tokenize=True):
- """
- Data Statistics metric
+ """Data Statistics metric
Makes use of Newsroom code: \
https://github.com/lil-lab/newsroom/blob/master/newsroom/analyze/fragments.py
Calculates extractive statistics such as coverage, density, compression as
@@ -65,11 +64,11 @@ def __init__(self, n_gram=3, n_workers=24, case=False, tokenize=True):
(e.g. news article) as opposed to the reference.
Args:
- :param n_gram: compute statistics for n-grams up to and including this length
- :param n_workers: number of processes to use if using multiprocessing
- :param case: whether to lowercase input before calculating statistics
- :param tokenize: whether to tokenize the input; otherwise assumes that the input
- is a string of space-separated tokens
+ n_gram (int): compute statistics for n-grams up to and including this length.
+ n_workers (int): number of processes to use if using multiprocessing.
+ case (bool): whether to lowercase input before calculating statistics.
+ tokenize (bool): whether to tokenize the input; otherwise assumes that the input
+ is a string of space-separated tokens.
"""
if not is_spacy_available():
raise ImportError(NO_SPACY_ERROR_MSG)
diff --git a/src/lighteval/metrics/imports/data_stats_utils.py b/src/lighteval/metrics/imports/data_stats_utils.py
index 708edee42..c5ae00725 100644
--- a/src/lighteval/metrics/imports/data_stats_utils.py
+++ b/src/lighteval/metrics/imports/data_stats_utils.py
@@ -6,12 +6,11 @@
def normalize(tokens, case=False):
- """
-
- Lowercases and turns tokens into distinct words.
+ """Lowercases and turns tokens into distinct words.
+ Returns:
+ list[str]: List of normalized tokens
"""
-
return [str(t).lower() if not case else str(t) for t in tokens]
@@ -39,9 +38,7 @@ def __init__(self, summary, text, case=False):
self._match(self._norm_summary, self._norm_text)
def overlaps(self):
- """
-
- Return a list of Fragments.Match objects between summary and text.
+ """Return a list of Fragments.Match objects between summary and text.
This is a list of named tuples of the form (summary, text, length):
- summary (int): the start index of the match in the summary
@@ -49,31 +46,22 @@ def overlaps(self):
- length (int): the length of the extractive fragment
"""
-
return self._matches
def strings(self, min_length=0, summary_base=True):
- """
-
- Return a list of explicit match strings between the summary and reference.
+ """Return a list of explicit match strings between the summary and reference.
Note that this will be in the same format as the strings are input. This is
important to remember if tokenization is done manually. If tokenization is
specified automatically on the raw strings, raw strings will automatically
be returned rather than SpaCy tokenized sequences.
- Arguments:
-
- - min_length (int): filter out overlaps shorter than this (default = 0)
- - raw (bool): return raw input rather than stringified
- - (default = False if automatic tokenization, True otherwise)
- - summary_base (true): strings are based of summary text (default = True)
+ Args:
+ min_length (int): filter out overlaps shorter than this (default = 0).
+ summary_base (bool): strings are based of summary text (default = True).
Returns:
-
- - list of overlaps, where overlaps are strings or token sequences
-
+ list: list of overlaps, where overlaps are strings or token sequences.
"""
-
# Compute the strings against the summary or the text?
base = self.summary if summary_base else self.text
@@ -96,18 +84,14 @@ def strings(self, min_length=0, summary_base=True):
return strings
def coverage(self, summary_base=True):
- """
- Return the COVERAGE score of the summary and text.
+ """Return the COVERAGE score of the summary and text.
- Arguments:
-
- - summary_base (bool): use summary as numerator (default = True)
+ Args:
+ summary_base (bool): use summary as numerator (default = True).
Returns:
-
- - decimal COVERAGE score within [0, 1]
+ float: decimal COVERAGE score within [0, 1].
"""
-
numerator = sum(o.length for o in self.overlaps())
if summary_base:
@@ -121,20 +105,14 @@ def coverage(self, summary_base=True):
return numerator / denominator
def density(self, summary_base=True):
- """
+ """Return the DENSITY score of summary and text.
- Return the DENSITY score of summary and text.
-
- Arguments:
-
- - summary_base (bool): use summary as numerator (default = True)
+ Args:
+ summary_base (bool): use summary as numerator (default = True).
Returns:
-
- - decimal DENSITY score within [0, ...]
-
+ float: decimal DENSITY score within [0, ...].
"""
-
numerator = sum(o.length**2 for o in self.overlaps())
if summary_base:
@@ -148,20 +126,14 @@ def density(self, summary_base=True):
return numerator / denominator
def compression(self, text_to_summary=True):
- """
-
- Return compression ratio between summary and text.
-
- Arguments:
+ """Return compression ratio between summary and text.
- - text_to_summary (bool): compute text/summary ratio (default = True)
+ Args:
+ text_to_summary (bool): compute text/summary ratio (default = True).
Returns:
-
- - decimal compression score within [0, ...]
-
+ float: decimal compression score within [0, ...].
"""
-
ratio = [len(self.text), len(self.summary)]
try:
@@ -174,12 +146,7 @@ def compression(self, text_to_summary=True):
return 0
def _match(self, a, b):
- """
-
- Raw procedure for matching summary in text, described in paper.
-
- """
-
+ """Raw procedure for matching summary in text, described in paper."""
self._matches = []
a_start = b_start = 0
diff --git a/src/lighteval/metrics/metrics_corpus.py b/src/lighteval/metrics/metrics_corpus.py
index 09018bf70..28a4b90c2 100644
--- a/src/lighteval/metrics/metrics_corpus.py
+++ b/src/lighteval/metrics/metrics_corpus.py
@@ -104,6 +104,7 @@ def __init__(self, metric_type: str, lang: Literal["zh", "ja", "ko", ""] = ""):
Args:
metric_type (str): Can be any of bleu, chrf, or ter depending on the metric to use.
+ lang (str): Language code for the translation metric.
"""
self.metric_type = metric_type
self.lang = lang
diff --git a/src/lighteval/metrics/metrics_sample.py b/src/lighteval/metrics/metrics_sample.py
index a0d250b0f..635f9d9de 100644
--- a/src/lighteval/metrics/metrics_sample.py
+++ b/src/lighteval/metrics/metrics_sample.py
@@ -108,8 +108,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
"""Computes the metric over a list of golds and predictions for one single sample.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): Predicted strings
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
float: Aggregated score over the current sample's items.
@@ -186,8 +187,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
"""Computes the metric over a list of golds and predictions for one single sample.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): Predicted strings
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
float: Aggregated score over the current sample's items.
@@ -233,7 +235,7 @@ def __init__(self, logprob_normalization: LogProbNormalization | None = None):
is actually in the gold ones.
Args:
- normalization (Normalization): The normalization to apply.
+ logprob_normalization (LogProbNormalization | None): The normalization to apply.
"""
self.logprob_normalization = logprob_normalization
@@ -248,12 +250,9 @@ def compute(
in the `gold_ixs`?
Args:
- gold_ixs (list[int]): All the gold choices indices
- choices_logprob (list[float]): Summed log-probabilities of all the possible choices for the model, ordered as the choices.
- unconditioned_logprob (list[float] | None): Unconditioned log-probabilities for PMI normalization, ordered as the choices.
- choices_tokens (list[list[int]] | None): Tokenized choices for token normalization, ordered as the choices.
- formatted_doc (Doc): Original document for the sample.
- Used to get the original choices' length for possible normalization
+ doc (Doc): The document containing choices and gold indices.
+ model_response (ModelResponse): The model's response containing logprobs.
+ **kwargs: Additional keyword arguments.
Returns:
int: The eval score: 1 if the best log-prob choice is in gold, 0 otherwise.
@@ -294,8 +293,8 @@ def __init__(
it returns the aggregated probability (default is max).
Args:
- normalization (Normalization | None): The normalization to apply.
- aggregation_function (Callable[[list[float]], float]): The function to use to aggregate gold probabilities in case of multiple golds.
+ log_prob_normalization (LogProbNormalization | None): The normalization to apply.
+ aggregation_function (Callable[[np.ndarray], float]): The function to use to aggregate gold probabilities in case of multiple golds.
"""
self.log_prob_normalization = log_prob_normalization
self.aggregation_function = aggregation_function
@@ -309,12 +308,9 @@ def compute(
"""Computes the log likelihood probability: chance of choosing the best choice.
Args:
- gold_ixs (list[int]): All the gold choices indices
- choices_logprob (list[float]): Summed log-probabilities of all the possible choices for the model, ordered as the choices.
- unconditioned_logprob (list[float] | None): Unconditioned log-probabilities for PMI normalization, ordered as the choices.
- choices_tokens (list[list[int]] | None): Tokenized choices for token normalization, ordered as the choices.
- formatted_doc (Doc): Original document for the sample.
- Used to get the original choices' length for possible normalization
+ doc (Doc): The document containing choices and gold indices.
+ model_response (ModelResponse): The model's response containing logprobs.
+ **kwargs: Additional keyword arguments.
Returns:
float: The probability of the best log-prob choice being a gold choice.
@@ -372,11 +368,9 @@ def compute(
"""Computes the log likelihood probability: chance of choosing the best choice.
Args:
- gold_ixs (list[int]): All the gold choices indices
- choices_logprob (list[float]): Summed log-probabilities of all the possible choices for the model, ordered as the choices.
- unconditioned_logprob (list[float] | None): Unconditioned log-probabilities for PMI normalization, ordered as the choices.
- choices_tokens (list[list[int]] | None): Tokenized choices for token normalization, ordered as the choices.
- reference_texts (list[str] | None): Reference texts for token normalization, ordered as the choices.
+ doc (Doc): The document containing choices and gold indices.
+ model_response (ModelResponse): The model's response containing logprobs.
+ **kwargs: Additional keyword arguments.
Returns:
float: The probability of the best log-prob choice being a gold choice.
@@ -405,7 +399,7 @@ def __init__(self, k: int) -> None:
"""Recall metric class. It checks if the top `k` best choices include one of the golds or not.
Args:
- at (int): Depth level of the recall.
+ k (int): Depth level of the recall.
Recall at 1 is equivalent to a logprob accuracy without normalization.
"""
self.recall_depth = k
@@ -415,8 +409,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> int:
highest log probabilities) and see if there is an actual gold among them.
Args:
- gold_ixs (list[int]): All the gold choices indices
- choices_logprob (list[float]): Summed log-probabilities of all the possible choices for the model, ordered as the choices.
+ doc (Doc): The document containing choices and gold indices.
+ model_response (ModelResponse): The model's response containing logprobs.
+ **kwargs: Additional keyword arguments.
Returns:
int: Score: 1 if one of the top level predicted choices was correct, 0 otherwise.
@@ -442,10 +437,9 @@ def compute(self, model_response: ModelResponse, doc: Doc, **kwargs) -> float:
"""Mean reciprocal rank. Measures the quality of a ranking of choices (ordered by correctness).
Args:
- gold_ixs (list[int]): All the gold choices indices
- choices_logprob (list[float]): Summed log-probabilities of all the possible choices for the model, ordered as the choices.
- formatted_doc (Doc): Original document for the sample.
- Used to get the original choices' length for possible normalization
+ model_response (ModelResponse): The model's response containing logprobs.
+ doc (Doc): The document containing choices and gold indices.
+ **kwargs: Additional keyword arguments.
Returns:
float: MRR score.
@@ -468,10 +462,12 @@ def compute(self, doc, model_response, **kwargs) -> int:
"""Tests if at least one of predicted gold targets' argmax of logits equals the gold.
Args:
- argmax_logits_eq_gold_list (list[int]): List of scores 1/0 indicating whether the argmax of logits equals the gold
+ doc: The document containing gold references.
+ model_response: The model's response containing argmax logits.
+ **kwargs: Additional keyword arguments.
Returns:
- int: 1 if at least one of the possible golds has argmax of logits == gold, 0 otherwise
+ int: 1 if at least one of the possible golds has argmax of logits == gold, 0 otherwise.
"""
return int(any(model_response.argmax_logits_eq_gold))
@@ -527,8 +523,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float |
"""Computes the metric(s) over a list of golds and predictions for one single sample.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): Predicted strings
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
float or dict: Aggregated score over the current sample's items.
@@ -621,8 +618,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str
"""Computes the prediction, recall and f1 score using the bert scorer.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): Predicted strings
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
dict: Scores over the current sample's items.
@@ -656,8 +654,7 @@ def __init__(
normalize_pred: callable = remove_braces_and_strip,
input_column: str = "text",
):
- """
- Extractiveness metric class.
+ """Extractiveness metric class.
Args:
normalize_input (callable, optional): Function to normalize the input strings.
@@ -672,15 +669,15 @@ def __init__(
self.input_column = input_column
def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]:
- """
- Compute the extractiveness of the predictions.
+ """Compute the extractiveness of the predictions.
This method calculates coverage, density, and compression scores for a single
prediction against the input text.
Args:
- predictions (list[str]): Predicted strings, a list of length 1.
- formatted_doc (Doc): The formatted document.
+ doc (Doc): The document containing input text.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
dict[str, float]: The extractiveness scores.
@@ -710,8 +707,7 @@ def __init__(
normalize_pred: Callable = remove_braces_and_strip,
input_column: str = "text",
):
- """
- Faithfulness metric class.
+ """Faithfulness metric class.
Args:
normalize_input (callable, optional): Function to normalize the input strings.
@@ -726,14 +722,14 @@ def __init__(
self.input_column = input_column
def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]:
- """
- Compute the faithfulness of the predictions.
+ """Compute the faithfulness of the predictions.
The SummaCZS (Summary Content Zero-Shot) model is used with configurable granularity and model variation.
Args:
- predictions (list[str]): Predicted strings, a list of length 1.
- formatted_doc (Doc): The formatted document.
+ doc (Doc): The document containing input text.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
dict[str, float]: The faithfulness scores.
@@ -777,8 +773,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
"""Uses the stored BLEURT scorer to compute the score on the current sample.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): Predicted strings
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
float: Score over the current sample's items.
@@ -805,8 +802,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
"""Computes the sentence level BLEU between the golds and each prediction, then takes the average.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): Predicted strings
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
float: Score over the current sample's items.
@@ -819,8 +817,8 @@ def _bleu_score(self, gold: list[str], pred: str):
"""Computes the BLEU score between a list of golds and the current prediction.
Args:
- golds (list[str]): Reference targets
- predictions (str): One of the predicted strings
+ gold (list[str]): Reference targets.
+ pred (str): One of the predicted strings.
Returns:
float: Score over the current prediction.
@@ -855,8 +853,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
"""Computes all the requested metrics on the golds and prediction.
Args:
- golds (list[str]): A list of possible golds. If it contains more than one item, only the first one is kept.
- predictions (list[str]): Predicted strings.
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
dict: The different scores computed
@@ -915,6 +914,9 @@ def edit_similarity(self, s1, s2):
Lee, Katherine, et al.
"Deduplicating training data makes language models better."
arXiv preprint arXiv:2107.06499 (2021).
+
+ Returns:
+ float: Edit similarity score between 0 and 1
"""
edist = edit_distance(s1, s2)
return 1.0 - edist / max(len(s1), len(s2)) if len(s1) > 0 and len(s2) > 0 else 0
@@ -997,8 +999,7 @@ def __init__(self):
)
def compute(self, responses: list[ModelResponse], docs: list[Doc], **kwargs) -> list:
- """
- Compute the score of a generative task using a llm as a judge.
+ """Compute the score of a generative task using a llm as a judge.
The generative task can be multiturn with 2 turns max, in that case, we
return scores for turn 1 and 2. Also returns user_prompt and judgement
which are ignored later by the aggregator.
@@ -1025,8 +1026,7 @@ def compute(self, responses: list[ModelResponse], docs: list[Doc], **kwargs) ->
class JudgeLLMMTBench(JudgeLLM):
def compute(self, model_response: list[ModelResponse], docs: list[Doc], **kwargs):
- """
- Compute the score of a generative task using a llm as a judge.
+ """Compute the score of a generative task using a llm as a judge.
The generative task can be multiturn with 2 turns max, in that case, we
return scores for turn 1 and 2. Also returns user_prompt and judgement
which are ignored later by the aggregator.
@@ -1058,8 +1058,7 @@ def compute(self, model_response: list[ModelResponse], docs: list[Doc], **kwargs
class JudgeLLMMixEval(JudgeLLM):
def compute(self, model_responses: list[ModelResponse], docs: list[Doc], **kwargs):
- """
- Compute the score of a generative task using a llm as a judge.
+ """Compute the score of a generative task using a llm as a judge.
The generative task can be multiturn with 2 turns max, in that case, we
return scores for turn 1 and 2. Also returns user_prompt and judgement
which are ignored later by the aggregator.
@@ -1156,13 +1155,8 @@ def __init__(self, k: int | None = None, **kwargs):
"""Sample score averages all the individual k predictions scores.
Args:
- normalize_gold (callable, optional): Function to use to normalize the reference strings.
- Defaults to None if no normalization is applied.
- normalize_pred (callable, optional): Function to use to normalize the predicted strings.
- Defaults to None if no normalization is applied.
- strip_strings (bool, optional): Whether to strip both reference and predictions. Defaults to False.
- sample_scoring_function (callable | str, optional): Function to use to compute the score for each sample.
- If None, uses the default scoring function which is a simple exact match.
+ k (int | None): The number of top choices to consider.
+ **kwargs: Additional keyword arguments.
"""
super().__init__(**kwargs)
self.k = k
@@ -1174,8 +1168,9 @@ def compute(self, model_response: ModelResponse, doc: Doc, **kwargs):
then compares it to the gold.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): k predicted strings
+ model_response (ModelResponse): The model's response containing predictions.
+ doc (Doc): The document containing gold references.
+ **kwargs: Additional keyword arguments.
Returns:
float: Aggregated score over the current sample's items.
@@ -1188,13 +1183,23 @@ def compute(self, model_response: ModelResponse, doc: Doc, **kwargs):
return avg_score
def num_samples(self):
- return self.k
+ """Get the number of samples for this metric.
+
+ Returns:
+ int: The number of samples (n if specified, otherwise k)
+ """
+ return self.n if self.n is not None else self.k
class MajAtK(SamplingMetric, SampleLevelComputation):
def __init__(self, k: int | None = None, **kwargs):
- """An exact match class."""
- super().__init__(**kwargs)
+ """An exact match class.
+
+ Args:
+ k (int): The number of top choices to consider.
+ **kwargs: Additional keyword arguments.
+ """
+ super().__init__(kwargs)
self.k = k
self.attribute_must_be_set = ["k"]
@@ -1205,8 +1210,9 @@ def compute(self, model_response: ModelResponse, docs: Doc, **kwargs):
then compares it to the gold.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): k predicted strings
+ model_response (ModelResponse): The model's response containing predictions.
+ docs (Doc): The document containing gold references.
+ **kwargs: Additional keyword arguments.
Returns:
float: Aggregated score over the current sample's items.
@@ -1241,8 +1247,9 @@ def __init__(self, k: int | None = None, n: int | None = None, **kwargs):
"""Computing pass at k
Args:
- k (int): Threshold for the number of successful attempts.
- n (int): Number of samples to generate
+ k (int | None): Threshold for the number of successful attempts.
+ n (int | None): Number of samples to generate.
+ **kwargs: Additional keyword arguments.
"""
super().__init__(**kwargs)
self.k = k
@@ -1255,8 +1262,9 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
then aggregates the scores over the samples using a pass@k.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): k predicted strings
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
float: Aggregated score over the current sample's items.
@@ -1313,9 +1321,11 @@ def __init__(
"""Computing G-Pass@k from http://arxiv.org/abs/2412.13147
Args:
- k (int, list): The number of successful attempts to be considered.
- n (int): Number of samples to generate.
- thresholds (list): Thresholds to control successful attempts in k generate.
+ k (Union[int, list[int]] | None): The number of successful attempts to be considered.
+ n (int | None): Number of samples to generate.
+ thresholds (list[float]): Thresholds to control successful attempts in k generate.
+ name_prefix (str | None): Prefix for the metric name.
+ **kwargs: Additional keyword arguments.
"""
super().__init__(**kwargs)
self._k = k
@@ -1339,8 +1349,9 @@ def compute(self, model_response: ModelResponse, doc: Doc, **kwargs) -> float:
then aggregates the scores over the samples using a pass@k.
Args:
- golds (list[str]): Reference targets
- predictions (list[str]): k predicted strings
+ model_response (ModelResponse): The model's response containing predictions.
+ doc (Doc): The document containing gold references.
+ **kwargs: Additional keyword arguments.
Returns:
float: Aggregated score over the current sample's items.
diff --git a/src/lighteval/metrics/normalizations.py b/src/lighteval/metrics/normalizations.py
index 97c1bd384..925448f17 100644
--- a/src/lighteval/metrics/normalizations.py
+++ b/src/lighteval/metrics/normalizations.py
@@ -35,7 +35,11 @@
def helm_normalizer(text: str) -> str:
"""Lower text and remove punctuation, articles and extra whitespace.
Copied from the [QuAC](http://quac.ai/) evaluation script found at
- https://s3.amazonaws.com/my89public/quac/scorer.py"""
+ https://s3.amazonaws.com/my89public/quac/scorer.py
+
+ Returns:
+ str: Normalized text with punctuation, articles removed and whitespace fixed
+ """
def remove_articles(text: str) -> str:
return re.sub(r"\b(a|an|the)\b", " ", text)
@@ -68,14 +72,29 @@ def _tokenize(text):
def harness_triviaqa_normalizer(text: str) -> str:
+ """Normalize text for TriviaQA evaluation.
+
+ Returns:
+ str: Lowercase text with punctuation removed
+ """
return text.lower().translate(str.maketrans("", "", string.punctuation))
def bigbench_normalizer(text: str):
+ """Normalize text for BigBench evaluation.
+
+ Returns:
+ str: Text with " . " replaced by ".\n"
+ """
return text.replace(" . ", ".\n")
def remove_braces(text: str) -> str:
+ """Remove opening and closing braces from text.
+
+ Returns:
+ str: Text with braces removed from start and end
+ """
if text.startswith("{"):
text = text[1:]
if text.endswith("}"):
@@ -84,6 +103,11 @@ def remove_braces(text: str) -> str:
def remove_braces_and_strip(text: str) -> str:
+ """Remove braces and strip whitespace from text.
+
+ Returns:
+ str: Text with braces removed and whitespace stripped
+ """
text = text.strip()
if text.startswith("{"):
text = text[1:]
@@ -96,11 +120,14 @@ def math_normalizer(text: str) -> str: # noqa C901
"""Source: https://github.com/hendrycks/math"""
def _remove_boxed(text: str | None) -> str:
- """
- Extract the text within a \\boxed{...} environment.
+ """Extract the text within a \\boxed{...} environment.
+
Example:
>>> _remove_boxed(\\boxed{\\frac{2}{3}})
\\frac{2}{3}
+
+ Returns:
+ str: The text within the boxed environment, or empty string if extraction fails
"""
if text is None:
return ""
@@ -120,7 +147,11 @@ def _remove_boxed(text: str | None) -> str:
return ""
def _last_boxed_only_string(text: str) -> str | None:
- """Extract the last \\boxed{...} or \\fbox{...} element from a string."""
+ """Extract the last \\boxed{...} or \\fbox{...} element from a string.
+
+ Returns:
+ str | None: The last boxed string element, or None if not found
+ """
idx = text.rfind("\\boxed")
if idx < 0:
idx = text.rfind("\\fbox")
@@ -148,8 +179,7 @@ def _last_boxed_only_string(text: str) -> str | None:
return retval
def _fix_fracs(text: str) -> str:
- """
- Fix the formatting of fractions in the given text.
+ """Fix the formatting of fractions in the given text.
Copied from: https://github.com/hendrycks/math/blob/357963a7f5501a6c1708cf3f3fb0cdf525642761/modeling/math_equivalence.py#L1
Args:
@@ -199,9 +229,13 @@ def _fix_fracs(text: str) -> str:
def _fix_a_slash_b(text: str) -> str:
"""Source: https://github.com/hendrycks/math
Reformat fractions formatted as a/b to \\frac{a}{b}.
+
Example:
>>> _fix_a_slash_b("2/3")
\frac{2}{3}
+
+ Returns:
+ str: Text with a/b fractions reformatted to LaTeX \\frac{a}{b} format
"""
if len(text.split("/")) != 2:
return text
@@ -217,8 +251,7 @@ def _fix_a_slash_b(text: str) -> str:
return text
def _remove_right_units(text: str) -> str:
- """
- Removes unit descriptions from LaTeX-formatted text, where units are indicated by "\\text{ }".
+ """Removes unit descriptions from LaTeX-formatted text, where units are indicated by "\\text{ }".
This function splits the text at each "\\text{ " and returns the part before the first occurrence,
effectively discarding any units and additional text following this pattern. This function also
trims any trailing whitespace left after removing units.
@@ -258,9 +291,13 @@ def _remove_right_units(text: str) -> str:
def _fix_sqrt(text: str) -> str:
"""Source: https://github.com/hendrycks/math
Reformat square roots.
+
Example:
>>> _fix_sqrt("\\sqrt3")
\\sqrt{3}
+
+ Returns:
+ str: Text with square roots reformatted to proper LaTeX format
"""
if "\\sqrt" not in text:
return text
@@ -339,8 +376,7 @@ def _fix_sqrt(text: str) -> str:
def gsm8k_normalizer(text: str) -> str:
- """
- from https://github.com/openai/grade-school-math/blob/3101c7d5072418e28b9008a6636bde82a006892c/grade_school_math/dataset.py#L28
+ """From https://github.com/openai/grade-school-math/blob/3101c7d5072418e28b9008a6636bde82a006892c/grade_school_math/dataset.py#L28
Args:
text (str): input text
@@ -386,22 +422,34 @@ def gsm8k_normalizer(text: str) -> str:
def remove_articles(text: str, lang: Language) -> str:
- """
- Removes definite and indefinite articles from the text.
+ """Removes definite and indefinite articles from the text.
Generated using LLM then manually checked by non-expert.
We currently only support languages that don't blend articles.
If you are a native speaker of a language where articles are blended,
we would appreciate your contribution!
+
+ Returns:
+ str: Text with articles removed for the specified language
"""
pattern = _ARTICLE_PATTERNS.get(lang)
return re.sub(pattern, " ", text) if pattern else text
def remove_punc(text: str) -> str:
+ """Remove punctuation from text.
+
+ Returns:
+ str: Text with punctuation removed
+ """
return "".join(ch for ch in text if ch not in PUNCT)
def get_multilingual_normalizer(lang: Language, lower: bool = True) -> Callable[[str], str]:
+ """Get a normalizer function for the specified language.
+
+ Returns:
+ Callable[[str], str]: A function that normalizes text for the specified language
+ """
tokenizer = get_word_tokenizer(lang)
def _inner_normalizer(text: str) -> str:
@@ -419,8 +467,7 @@ def _inner_normalizer(text: str) -> str:
# Loglikelihood normalization
@dataclass
class LogProbPMINorm:
- """
- Performs Pointwise mutual information normalization. log_likelihood_conditioned - log_likelihood_unconditioned.
+ """Performs Pointwise mutual information normalization. log_likelihood_conditioned - log_likelihood_unconditioned.
Useful when answer contains generally unlikely tokens.
"""
@@ -429,8 +476,7 @@ class LogProbPMINorm:
@dataclass
class LogProbTokenNorm:
- """
- Performs token level normalization. log_likelihood/token_length.
+ """Performs token level normalization. log_likelihood/token_length.
Useful for non-english languages.
"""
@@ -439,8 +485,7 @@ class LogProbTokenNorm:
@dataclass
class LogProbCharNorm:
- """
- Performs character level normalization. log_likelihood/char_length
+ """Performs character level normalization. log_likelihood/char_length
ignore_first_space (bool, optional): Whether to ignore the first token's log prob (if it's a space only). Defaults to False.
The only case when it should be True is when the possible choices (for example `A`,`B` ...) have an extra
space added in front of them to manage tokenization issues (` A`, ` B`, ...) for some models.
diff --git a/src/lighteval/metrics/sample_preparator.py b/src/lighteval/metrics/sample_preparator.py
index ad9338e76..725ba6fc6 100644
--- a/src/lighteval/metrics/sample_preparator.py
+++ b/src/lighteval/metrics/sample_preparator.py
@@ -70,8 +70,9 @@ def prepare(doc: Doc, model_response: ModelResponse, **kwargs):
"""Prepares an individual generative example to the format expected by metrics computed at the corpus level (aggregated).
Args:
- golds (list[str]): List of allowed targets for the current example
- predictions (list[str]): List of generated predictions for the current example.
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing predictions.
+ **kwargs: Additional keyword arguments.
Returns:
GenerativeCorpusMetricInput: Stores the golds and predictions as such
@@ -95,8 +96,9 @@ def prepare(self, doc: Doc, model_response: ModelResponse, **kwargs) -> LogprobC
"""Prepares an individual loglikelihood example to the format expected by metrics computed at the corpus level (aggregated).
Args:
- golds_ixs (list[int]): List of the gold indices among the possible choices
- choices_logprob (list[float]): List of each choice's aggregated logprobs (usually with an average or weighted average).
+ doc (Doc): The document containing gold indices and choices.
+ model_response (ModelResponse): The model's response containing logprobs.
+ **kwargs: Additional keyword arguments.
Returns:
LogprobCorpusMetricInput: Stores the golds indices and the model's choice (choice with the highest logprob)
@@ -147,13 +149,13 @@ def prepare(self, doc: Doc, model_response: ModelResponse, **kwargs):
"""Prepares an individual perplexity example to the format expected by metrics computed at the corpus level (aggregated).
Args:
- logprobs (list[float]): List of the log-probabilities computed for each item of the sequence or single aggregated logprob over the sequence
- reference_text (str): Current reference text for which to compute the length in self.units_type
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing logprobs.
+ **kwargs: Additional keyword arguments.
Returns:
PerplexityCorpusMetricInput: Stores the measured logprobs and associated text lengths, counted in the reference unit.
"""
-
logprobs_flat = np.sum(model_response.logprobs)
reference_text_flat = " ".join(doc.get_golds())
return PerplexityCorpusMetricInput(logprobs=logprobs_flat, weights=self.count_units(reference_text_flat))
@@ -192,13 +194,13 @@ def prepare(self, doc: Doc, model_response: ModelResponse, **kwargs):
"""Prepares an individual perplexity example to the format expected by metrics computed at the corpus level (aggregated).
Args:
- logprobs (list[float]): List of the log-probabilities computed for each item of the sequence or single aggregated logprob over the sequence
- reference_text (str): Current reference text for which to compute the length in self.units_type
+ doc (Doc): The document containing gold references.
+ model_response (ModelResponse): The model's response containing logprobs.
+ **kwargs: Additional keyword arguments.
Returns:
PerplexityCorpusMetricInput: Stores the measured logprobs and associated text lengths, counted in the reference unit.
"""
-
logprobs_flat = np.sum(model_response.logprobs)
if doc.original_query is not None:
diff --git a/src/lighteval/metrics/utils/llm_as_judge.py b/src/lighteval/metrics/utils/llm_as_judge.py
index 3b7e4d537..dcf0a5a88 100644
--- a/src/lighteval/metrics/utils/llm_as_judge.py
+++ b/src/lighteval/metrics/utils/llm_as_judge.py
@@ -46,8 +46,7 @@
class JudgeLM:
- """
- A class representing a judge for evaluating answers using either the OpenAI or Transformers library.
+ """A class representing a judge for evaluating answers using either the OpenAI or Transformers library.
Args:
model (str): The name of the model.
@@ -186,8 +185,7 @@ def __lazy_load_client(self): # noqa: C901
raise ValueError(f"Unsupported backend: {self.backend}")
def dict_of_lists_to_list_of_dicts(self, dict_of_lists):
- """
- Transform a dictionary of lists into a list of dictionaries.
+ """Transform a dictionary of lists into a list of dictionaries.
Each dictionary in the output list will contain one element from each list in the input dictionary,
with the same keys as the input dictionary.
@@ -255,13 +253,13 @@ def evaluate_answer_batch(
return scores, prompts, responses
def evaluate_answer(self, question: str, answer: str, options: list[str] | None = None, gold: str | None = None):
- """
- Evaluates an answer using either Transformers or OpenAI API.
+ """Evaluates an answer using either Transformers or OpenAI API.
Args:
- questions (list[str]): The prompt asked to the evaluated model
- answers (list[str]): Answer given by the evaluated model
- references (list[str]): A list of reference answers
+ question (str): The prompt asked to the evaluated model.
+ answer (str): Answer given by the evaluated model.
+ options (list[str] | None): Optional list of answer options.
+ gold (str | None): Optional reference answer.
Returns:
A tuple containing the score, prompts, and judgment.
diff --git a/src/lighteval/metrics/utils/math_comparison.py b/src/lighteval/metrics/utils/math_comparison.py
index 6592bb8d8..2650ee335 100644
--- a/src/lighteval/metrics/utils/math_comparison.py
+++ b/src/lighteval/metrics/utils/math_comparison.py
@@ -170,12 +170,12 @@ def sympy_deep_compare_set_and_tuple(gold: FiniteSet | Tuple, pred: FiniteSet |
"""Compare two finite sets by comparing each element with given precision.
Args:
- a: First finite set
- b: Second finite set
- precision: Number of decimal places to compare
+ gold (FiniteSet | Tuple): First finite set or tuple.
+ pred (FiniteSet | Tuple): Second finite set or tuple.
+ precision (int): Number of decimal places to compare.
Returns:
- True if sets contain equal elements within precision, False otherwise
+ True if sets contain equal elements within precision, False otherwise.
Note: in order to fully support finite sets, we should ideally do kartesian product comparison
but this is not implemented yet. We kinda hope sympy will order the elements.
@@ -571,10 +571,7 @@ def sympy_expr_eq(gold: Basic | MatrixBase, pred: Basic | MatrixBase, precision:
def should_treat_as_complex(latex_str: str) -> bool:
- """
- Returns True if the latex string likely contains complex numbers, matrices, or vectors.
- """
-
+ """Returns True if the latex string likely contains complex numbers, matrices, or vectors."""
return bool(complex_number_pattern.search(latex_str))
diff --git a/src/lighteval/metrics/utils/metric_utils.py b/src/lighteval/metrics/utils/metric_utils.py
index 85b1e2bc6..c23a7854c 100644
--- a/src/lighteval/metrics/utils/metric_utils.py
+++ b/src/lighteval/metrics/utils/metric_utils.py
@@ -50,6 +50,7 @@ def compute_sample(
elif isinstance(self.sample_level_fn, Preparator):
sample_level_fn = self.sample_level_fn.prepare
else:
+ breakpoint()
raise ValueError(
f"Incorrect type for {self.sample_level_fn}, should be a SampleLevelComputation or Preparator"
)
diff --git a/src/lighteval/models/abstract_model.py b/src/lighteval/models/abstract_model.py
index ab4ce43e2..46487656e 100644
--- a/src/lighteval/models/abstract_model.py
+++ b/src/lighteval/models/abstract_model.py
@@ -39,8 +39,7 @@
class ModelConfig(BaseModel, extra="forbid"):
- """
- Base configuration class for all model types in Lighteval.
+ """Base configuration class for all model types in Lighteval.
This is the foundation class that all specific model configurations inherit from.
It provides common functionality for parsing configuration from files and command-line arguments,
@@ -53,6 +52,8 @@ class ModelConfig(BaseModel, extra="forbid"):
system_prompt (str | None):
Optional system prompt to be used with chat models. This prompt sets the
behavior and context for the model during evaluation.
+ cache_dir (str):
+ Directory to cache the model. Defaults to "~/.cache/huggingface/lighteval".
Methods:
from_path(path: str):
@@ -185,8 +186,7 @@ def greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
docs (list[Doc]): List of documents containing the context for generation.
@@ -200,12 +200,19 @@ def greedy_until(
def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]:
"""Tokenize the context and continuation and compute the log likelihood of those
tokenized sequences.
+
+ Returns:
+ list[ModelResponse]: List of model responses containing log likelihood scores
"""
return NotImplemented
@abstractmethod
def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]:
- """This function is used to compute the log likelihood of the context for perplexity metrics."""
+ """This function is used to compute the log likelihood of the context for perplexity metrics.
+
+ Returns:
+ list[ModelResponse]: List of model responses containing log likelihood scores
+ """
return NotImplemented
# Tokenization utils
@@ -223,9 +230,10 @@ def tok_encode(self, str_to_encode: str | list[str], add_special_tokens: Optiona
def tok_encode_pair(self, context, continuations: list[str], pairwise: bool = False):
"""Encodes a context with a list of continuations by taking care of the spaces in between.
+
Args:
context (str): The context string to be encoded.
- continuation (list[str]): List of continuation strings to be encoded.
+ continuations (list[str]): List of continuation strings to be encoded.
pairwise (bool):
If True, encode context and continuations separately.
If False, encode them together and then split.
diff --git a/src/lighteval/models/custom/custom_model.py b/src/lighteval/models/custom/custom_model.py
index fb654acf6..ea11fe3c6 100644
--- a/src/lighteval/models/custom/custom_model.py
+++ b/src/lighteval/models/custom/custom_model.py
@@ -24,8 +24,7 @@
class CustomModelConfig(ModelConfig):
- """
- Configuration class for loading custom model implementations in Lighteval.
+ """Configuration class for loading custom model implementations in Lighteval.
This config allows users to define and load their own model implementations by specifying
a Python file containing a custom model class that inherits from LightevalModel.
diff --git a/src/lighteval/models/dummy/dummy_model.py b/src/lighteval/models/dummy/dummy_model.py
index 6971b6daa..157996150 100644
--- a/src/lighteval/models/dummy/dummy_model.py
+++ b/src/lighteval/models/dummy/dummy_model.py
@@ -33,8 +33,7 @@
class DummyModelConfig(ModelConfig):
- """
- Configuration class for dummy models used for testing and baselines.
+ """Configuration class for dummy models used for testing and baselines.
This configuration is used to create dummy models that generate random responses
or baselines for evaluation purposes. Useful for testing evaluation pipelines
diff --git a/src/lighteval/models/endpoints/endpoint_model.py b/src/lighteval/models/endpoints/endpoint_model.py
index f2dc2c03b..7bede3851 100644
--- a/src/lighteval/models/endpoints/endpoint_model.py
+++ b/src/lighteval/models/endpoints/endpoint_model.py
@@ -69,8 +69,7 @@
class ServerlessEndpointModelConfig(ModelConfig):
- """
- Configuration class for HuggingFace Inference API (inference endpoints).
+ """Configuration class for HuggingFace Inference API (inference endpoints).
https://huggingface.co/inference-endpoints/dedicated
@@ -82,6 +81,12 @@ class ServerlessEndpointModelConfig(ModelConfig):
Whether to add special tokens during tokenization. Defaults to True.
batch_size (int):
Batch size for requests. Defaults to 1 (serverless API limitation).
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Example:
```python
@@ -101,8 +106,7 @@ class ServerlessEndpointModelConfig(ModelConfig):
class InferenceEndpointModelConfig(ModelConfig):
- """
- Configuration class for HuggingFace Inference Endpoints (dedicated infrastructure).
+ """Configuration class for HuggingFace Inference Endpoints (dedicated infrastructure).
This configuration is used to create and manage dedicated inference endpoints
on HuggingFace's infrastructure. These endpoints provide dedicated compute
@@ -143,6 +147,12 @@ class InferenceEndpointModelConfig(ModelConfig):
Additional environment variables for the endpoint.
batch_size (int):
Batch size for requests. Defaults to 1.
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Methods:
model_post_init():
diff --git a/src/lighteval/models/endpoints/inference_providers_model.py b/src/lighteval/models/endpoints/inference_providers_model.py
index 3c0d025fe..20d3c94e4 100644
--- a/src/lighteval/models/endpoints/inference_providers_model.py
+++ b/src/lighteval/models/endpoints/inference_providers_model.py
@@ -43,8 +43,7 @@
class InferenceProvidersModelConfig(ModelConfig):
- """
- Configuration class for HuggingFace's inference providers (like Together AI, Anyscale, etc.).
+ """Configuration class for HuggingFace's inference providers (like Together AI, Anyscale, etc.).
inference providers doc: https://huggingface.co/docs/inference-providers/en/index
@@ -62,6 +61,12 @@ class InferenceProvidersModelConfig(ModelConfig):
parallel_calls_count (NonNegativeInt):
Number of parallel API calls to make. Defaults to 10.
Higher values increase throughput but may hit rate limits.
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Example:
```python
@@ -196,12 +201,10 @@ def greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
- override_bs (int, optional): Override the batch size for generation. Defaults to None.
+ docs (list[Doc]): List of documents containing the context for generation.
Returns:
list[ModelResponse]: list of generated responses.
diff --git a/src/lighteval/models/endpoints/litellm_model.py b/src/lighteval/models/endpoints/litellm_model.py
index 20fb87d04..544123e00 100644
--- a/src/lighteval/models/endpoints/litellm_model.py
+++ b/src/lighteval/models/endpoints/litellm_model.py
@@ -56,8 +56,7 @@
class LiteLLMModelConfig(ModelConfig):
- """
- Configuration class for LiteLLM unified API client.
+ """Configuration class for LiteLLM unified API client.
This configuration is used to connect to various LLM providers through the LiteLLM
unified API. LiteLLM provides a consistent interface to multiple providers including
@@ -82,6 +81,12 @@ class LiteLLMModelConfig(ModelConfig):
Maximum number of concurrent API requests to execute in parallel.
Higher values can improve throughput for batch processing but may hit rate limits
or exhaust API quotas faster. Default is 10.
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Example:
```python
@@ -109,8 +114,7 @@ class LiteLLMClient(LightevalModel):
_DEFAULT_MAX_LENGTH: int = 4096
def __init__(self, config: LiteLLMModelConfig) -> None:
- """
- IMPORTANT: Your API keys should be set in the environment variables.
+ """IMPORTANT: Your API keys should be set in the environment variables.
If a base_url is not set, it will default to the public API.
"""
self.config = config
@@ -259,12 +263,10 @@ def greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
- override_bs (int, optional): Override the batch size for generation. Defaults to None.
+ docs (list[Doc]): List of documents containing the context for generation.
Returns:
list[ModelResponse]: list of generated responses.
diff --git a/src/lighteval/models/endpoints/tgi_model.py b/src/lighteval/models/endpoints/tgi_model.py
index 91940f88f..8130cba88 100644
--- a/src/lighteval/models/endpoints/tgi_model.py
+++ b/src/lighteval/models/endpoints/tgi_model.py
@@ -53,8 +53,7 @@ def divide_chunks(array, n):
class TGIModelConfig(ModelConfig):
- """
- Configuration class for Text Generation Inference (TGI) backend.
+ """Configuration class for Text Generation Inference (TGI) backend.
doc: https://huggingface.co/docs/text-generation-inference/en/index
@@ -70,6 +69,12 @@ class TGIModelConfig(ModelConfig):
Authentication token for the TGI server. If None, no authentication is used.
model_name (str | None):
Optional model name override. If None, uses the model name from server info.
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Example:
```python
diff --git a/src/lighteval/models/model_input.py b/src/lighteval/models/model_input.py
index d6e5fe193..ad41c23eb 100644
--- a/src/lighteval/models/model_input.py
+++ b/src/lighteval/models/model_input.py
@@ -66,6 +66,9 @@ def from_dict(cls, config_dict: dict):
"truncate_prompt": value
}
}
+
+ Returns:
+ GenerationParameters: A GenerationParameters object created from the config dictionary
"""
return GenerationParameters(**config_dict.get("generation", {}))
@@ -80,6 +83,9 @@ def from_model_args(cls, model_args: str):
Args:
model_args (str): A string like the following:
"pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,dtype=float16,max_model_length=32768,generation_parameters={temperature:0.7,top_p:5}"
+
+ Returns:
+ GenerationParameters: A GenerationParameters object created from the model args string
"""
def parse_model_args(model_args):
diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index 01995ddef..ccae20a5f 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -61,14 +61,11 @@
def load_model( # noqa: C901
config: ModelConfig,
) -> LightevalModel:
- """
- Load a model from a checkpoint, depending on the config type.
+ """Load a model from a checkpoint, depending on the config type.
Args:
config (ModelConfig): configuration of the model to load
- Raises:
- ValueError: If you try to have both the multichoice continuations start with a space and not to start with a space
Returns:
LightevalModel: The model that will be evaluated
"""
diff --git a/src/lighteval/models/model_output.py b/src/lighteval/models/model_output.py
index fcb96099e..db72cb7df 100644
--- a/src/lighteval/models/model_output.py
+++ b/src/lighteval/models/model_output.py
@@ -27,8 +27,7 @@
@dataclass
class ModelResponse:
- """
- A class to represent the response from a model during evaluation.
+ """A class to represent the response from a model during evaluation.
This dataclass contains all the information returned by a model during inference,
including generated text, log probabilities, token information, and metadata.
diff --git a/src/lighteval/models/nanotron/nanotron_model.py b/src/lighteval/models/nanotron/nanotron_model.py
index 7c80aebf9..686111e04 100644
--- a/src/lighteval/models/nanotron/nanotron_model.py
+++ b/src/lighteval/models/nanotron/nanotron_model.py
@@ -178,7 +178,18 @@ def __init__(
model_class: Optional[Type] = None,
):
"""Initializes a nanotron model for evaluation.
+
Args:
+ checkpoint_path (str): Path to the model checkpoint.
+ nanotron_config (FullNanotronConfig): Configuration for the nanotron model.
+ parallel_context (ParallelContext): Parallel context for distributed training.
+ max_gen_toks (Optional[int]): Maximum number of tokens to generate.
+ max_length (Optional[int]): Maximum sequence length.
+ add_special_tokens (Optional[bool]): Whether to add special tokens.
+ dtype (Optional[Union[str, torch.dtype]]): Data type for the model.
+ trust_remote_code (bool): Whether to trust remote code.
+ debug_one_layer_model (bool): Whether to use a single layer for debugging.
+ model_class (Optional[Type]): Custom model class to use.
"""
self.config = nanotron_config
model_args = nanotron_config.nanotron_model
@@ -307,7 +318,6 @@ def _create_auto_tokenizer(
trust_remote_code: bool = False,
) -> transformers.PreTrainedTokenizer:
"""Returns a pre-trained tokenizer from a pre-trained tokenizer configuration."""
-
try:
tokenizer = AutoTokenizer.from_pretrained(
pretrained if tokenizer is None else tokenizer,
diff --git a/src/lighteval/models/sglang/sglang_model.py b/src/lighteval/models/sglang/sglang_model.py
index 227200de8..3c39315a8 100644
--- a/src/lighteval/models/sglang/sglang_model.py
+++ b/src/lighteval/models/sglang/sglang_model.py
@@ -52,8 +52,7 @@
class SGLangModelConfig(ModelConfig):
- """
- Configuration class for SGLang inference engine.
+ """Configuration class for SGLang inference engine.
This configuration is used to load and configure models using the SGLang inference engine,
which provides high-performance inference.
@@ -98,6 +97,12 @@ class SGLangModelConfig(ModelConfig):
override_chat_template (bool):
If True, we force the model to use a chat template. If alse, we prevent the model from using
a chat template. If None, we use the default (true if present in the tokenizer, false otherwise)
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Example:
```python
@@ -221,15 +226,13 @@ def greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
- override_bs (int, optional): Override the batch size for generation. Defaults to None.
+ docs (list[Doc]): List of documents containing the context for generation.
Returns:
- list[GenerateReturn]: list of generated responses.
+ list[ModelResponse]: list of generated responses.
"""
return self._greedy_until(docs)
@@ -319,7 +322,6 @@ def _generate(
generate: bool = True,
) -> list:
"""Contains the actual logic of the generation."""
-
logprob_start_len = None
top_logprobs_num = None
if generate:
diff --git a/src/lighteval/models/transformers/adapter_model.py b/src/lighteval/models/transformers/adapter_model.py
index e0d0d6318..a868ad20f 100644
--- a/src/lighteval/models/transformers/adapter_model.py
+++ b/src/lighteval/models/transformers/adapter_model.py
@@ -40,8 +40,7 @@
class AdapterModelConfig(TransformersModelConfig):
- """
- Configuration class for PEFT (Parameter-Efficient Fine-Tuning) adapter models.
+ """Configuration class for PEFT (Parameter-Efficient Fine-Tuning) adapter models.
This configuration is used to load models that have been fine-tuned using PEFT adapters,
such as LoRA, AdaLoRA, or other parameter-efficient fine-tuning methods. The adapter
diff --git a/src/lighteval/models/transformers/delta_model.py b/src/lighteval/models/transformers/delta_model.py
index bb3d1b06e..3fb166c1c 100644
--- a/src/lighteval/models/transformers/delta_model.py
+++ b/src/lighteval/models/transformers/delta_model.py
@@ -36,8 +36,7 @@
class DeltaModelConfig(TransformersModelConfig):
- """
- Configuration class for delta models (weight difference models).
+ """Configuration class for delta models (weight difference models).
This configuration is used to load models that represent the difference between a
fine-tuned model and its base model. The delta weights are added to the base model
diff --git a/src/lighteval/models/transformers/transformers_model.py b/src/lighteval/models/transformers/transformers_model.py
index 9645904fa..3d0d7c12b 100644
--- a/src/lighteval/models/transformers/transformers_model.py
+++ b/src/lighteval/models/transformers/transformers_model.py
@@ -68,8 +68,7 @@
class TransformersModelConfig(ModelConfig):
- """
- Configuration class for HuggingFace Transformers models.
+ """Configuration class for HuggingFace Transformers models.
This configuration is used to load and configure models from the HuggingFace Transformers library.
@@ -116,6 +115,12 @@ class TransformersModelConfig(ModelConfig):
override_chat_template (bool):
If True, we force the model to use a chat template. If alse, we prevent the model from using
a chat template. If None, we use the default (true if present in the tokenizer, false otherwise)
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Example:
```python
@@ -368,8 +373,7 @@ def init_model_parallel(self, model_parallel: bool | None = None) -> Tuple[bool,
return model_parallel, max_mem_this_process, device_map
def _create_auto_model(self) -> transformers.PreTrainedModel:
- """
- Creates an instance of the pretrained HF model.
+ """Creates an instance of the pretrained HF model.
Returns:
transformers.PreTrainedModel: The created auto model instance.
@@ -425,8 +429,7 @@ def _create_auto_model(self) -> transformers.PreTrainedModel:
def _create_auto_tokenizer(
self,
) -> transformers.PreTrainedTokenizer:
- """
- Create a Hugging Face AutoTokenizer for language model.
+ """Create a Hugging Face AutoTokenizer for language model.
Returns:
transformers.PreTrainedTokenizer: The created tokenizer.
@@ -461,7 +464,6 @@ def _init_max_length(self) -> int:
Returns:
int: Max length to use depending on the available args and config
"""
-
if self.config.max_length is not None:
return self.config.max_length
@@ -518,15 +520,13 @@ def _continuous_greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
- override_bs (int, optional): Override the batch size for generation. Defaults to None.
+ docs (list[Doc]): List of documents containing the context for generation.
Returns:
- list[GenerateReturn]: list of generated responses.
+ list[ModelResponse]: list of generated responses.
"""
dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS)
results = []
@@ -623,12 +623,10 @@ def _padded_greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
- override_bs (int, optional): Override the batch size for generation. Defaults to None.
+ docs (list[Doc]): List of documents containing the context for generation.
Returns:
list[ModelResponse]: list of generated responses.
@@ -878,10 +876,10 @@ def loglikelihood(
tokenized sequences.
Args:
- requests (list[Tuple[str, dict]]): _description_
+ docs (list[Doc]): List of documents containing the context and choices.
Returns:
- list[Tuple[float, bool]]: _description_
+ list[ModelResponse]: List of model responses with logprobs.
"""
return self._loglikelihood_tokens(docs)
@@ -891,7 +889,6 @@ def loglikelihood_rolling(
docs: list[Doc],
) -> list[ModelResponse]:
"""This function is used to compute the log likelihood of the context for perplexity metrics."""
-
return self._loglikelihood_tokens(
docs,
rolling=True,
@@ -1193,14 +1190,14 @@ def prepare_batch_logprob(
def pad_and_gather(
self, output_tensor: torch.Tensor, drop_last_samples: bool = True, num_samples: int = None
) -> torch.Tensor:
- """
- Pads the `output_tensor` to the maximum length and gathers the lengths across processes.
+ """Pads the `output_tensor` to the maximum length and gathers the lengths across processes.
Args:
output_tensor (torch.Tensor): The output tensor to be padded.
drop_last_samples (bool, optional): Whether to drop the last samples during gathering.
- Last samples are dropped when the number of samples is not divisible by the number of processes.
+ Last samples are dropped when the number of samples is not divisible by the number of processes.
Defaults to True.
+ num_samples (int, optional): Number of samples per batch item.
Returns:
torch.Tensor: The padded output tensor and the gathered length tensor.
diff --git a/src/lighteval/models/transformers/vlm_transformers_model.py b/src/lighteval/models/transformers/vlm_transformers_model.py
index f640d15d9..61f9d58ab 100644
--- a/src/lighteval/models/transformers/vlm_transformers_model.py
+++ b/src/lighteval/models/transformers/vlm_transformers_model.py
@@ -70,8 +70,7 @@ def __call__(self, requests: list[Doc]) -> Tuple[dict[str, torch.Tensor], list[D
class VLMTransformersModelConfig(ModelConfig):
- """
- Configuration class for VLM (image-text-to-text) models.
+ """Configuration class for VLM (image-text-to-text) models.
Attributes:
model_name (str):
@@ -103,6 +102,14 @@ class VLMTransformersModelConfig(ModelConfig):
model at a quantized precision. Needed for 4-bit and 8-bit precision.
trust_remote_code (bool): Whether to trust remote code during model
loading.
+ compile (bool, optional, defaults to False): Whether to compile the model for faster inference.
+ device_map (str | None, optional, defaults to None): Device mapping strategy for model loading.
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
"""
model_name: str
@@ -140,7 +147,6 @@ def __init__(
config: VLMTransformersModelConfig,
):
"""Initializes a HuggingFace `AutoModel` and `AutoTokenizer` for evaluation."""
-
self.accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))])
self.device = self.accelerator.device
self.torch_dtype = _get_dtype(config.dtype)
@@ -287,8 +293,7 @@ def _create_auto_model(self):
return model
def _create_auto_processor(self):
- """
- Create a transformers `Processor` for VLM (image-text-to-text) model.
+ """Create a transformers `Processor` for VLM (image-text-to-text) model.
Returns:
transformers.ProcessorMixin: The created processor.
@@ -338,8 +343,7 @@ def greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
docs (list[Docs]): list of docs containing the context and ending conditions.
diff --git a/src/lighteval/models/utils.py b/src/lighteval/models/utils.py
index 22b2ab27d..f615019ea 100644
--- a/src/lighteval/models/utils.py
+++ b/src/lighteval/models/utils.py
@@ -35,8 +35,7 @@
def _get_dtype(dtype: Union[str, torch.dtype, None], config: Optional[AutoConfig] = None) -> Optional[torch.dtype]:
- """
- Get the torch dtype based on the input arguments.
+ """Get the torch dtype based on the input arguments.
Args:
dtype (Union[str, torch.dtype]): The desired dtype. Can be a string or a torch dtype.
@@ -45,7 +44,6 @@ def _get_dtype(dtype: Union[str, torch.dtype, None], config: Optional[AutoConfig
Returns:
torch.dtype: The torch dtype based on the input arguments.
"""
-
if config is not None and hasattr(config, "quantization_config"):
# must be infered
return None
@@ -64,8 +62,7 @@ def _get_dtype(dtype: Union[str, torch.dtype, None], config: Optional[AutoConfig
def _simplify_name(name_or_path: str) -> str:
- """
- If the model is loaded from disk, then the name will have the following format:
+ """If the model is loaded from disk, then the name will have the following format:
/p/a/t/h/models--org--model_name/revision/model_files
This function return the model_name as if it was loaded from the hub:
org/model_name
@@ -116,7 +113,9 @@ def uses_chat_template(
a chat template or not
Args:
- model_name (str): Model name on HF
+ model_name (str, optional): Model name on HF.
+ tokenizer (AutoTokenizer, optional): The tokenizer to check.
+ override_chat_template (bool, optional): Override the chat template detection.
Returns:
bool: True if Tokenizer config contains a chat template, False otherwise
diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py
index b9f438e81..eac77e640 100644
--- a/src/lighteval/models/vllm/vllm_model.py
+++ b/src/lighteval/models/vllm/vllm_model.py
@@ -74,8 +74,7 @@
class VLLMModelConfig(ModelConfig):
- """
- Configuration class for VLLM inference engine.
+ """Configuration class for VLLM inference engine.
This configuration is used to load and configure models using the VLLM inference engine,
which provides high-performance inference for large language models with features like
@@ -128,6 +127,12 @@ class VLLMModelConfig(ModelConfig):
override_chat_template (bool):
If True, we force the model to use a chat template. If alse, we prevent the model from using
a chat template. If None, we use the default (true if present in the tokenizer, false otherwise)
+ generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
+ Configuration parameters that control text generation behavior, including
+ temperature, top_p, max_new_tokens, etc.
+ system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models.
+ This prompt sets the behavior and context for the model during evaluation.
+ cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model.
Example:
```python
@@ -229,22 +234,13 @@ def max_length(self) -> int:
return self._max_length
def _create_auto_model(self, config: VLLMModelConfig) -> Optional[LLM]:
- """
- Creates an instance of the pretrained HF model.
+ """Creates an instance of the pretrained HF model.
Args:
- pretrained (str): The name or path of the pretrained model.
- revision (str): The revision of the model.
- subfolder (Optional[str], optional): The subfolder within the model. Defaults to None.
- max_memory (Optional[dict], optional): The maximum memory to allocate for the model per GPU. Defaults to None.
- device_map (Optional[dict], optional): The device mapping for the model. Defaults to None.
- torch_dtype (Optional[Union[str, torch.dtype]], optional): The torch data type for the model. Defaults to None.
- quantization_config (Optional[Union[BitsAndBytesConfig, GPTQConfig]], optional): The quantization configuration for the model. Defaults to None.
- trust_remote_code (bool, optional): Whether to trust remote code. Defaults to False.
- cache_dir (str, optional): The cache directory for the model. Defaults to "/scratch".
+ config (VLLMModelConfig): The VLLM model configuration.
Returns:
- transformers.PreTrainedModel: The created auto model instance.
+ Optional[LLM]: The created auto model instance.
"""
self.model_args = {
"model": config.model_name,
@@ -305,15 +301,13 @@ def greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
- override_bs (int, optional): Override the batch size for generation. Defaults to None.
+ docs (list[Doc]): List of documents containing the context for generation.
Returns:
- list[GenerateReturn]: list of generated responses.
+ list[ModelResponse]: list of generated responses.
"""
return self._greedy_until(docs)
@@ -544,8 +538,7 @@ def cleanup(self):
torch.cuda.empty_cache()
def _create_auto_model(self, config: VLLMModelConfig):
- """
- Creates an instance of the async vllm model loaded from HF. Requires using the v1 of VLLM.
+ """Creates an instance of the async vllm model loaded from HF. Requires using the v1 of VLLM.
Returns:
AsyncLLM: The created async VLLM instance
@@ -628,14 +621,13 @@ async def greedy_until(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met.
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
+ docs (list[Doc]): List of documents containing the context for generation.
Returns:
- list[GenerateReturn]: list of generated responses.
+ list[ModelResponse]: list of generated responses.
"""
results = []
@@ -663,12 +655,11 @@ async def loglikelihood(
self,
docs: list[Doc],
) -> list[ModelResponse]:
- """
- Generates responses using a greedy decoding strategy until certain ending conditions are met and
+ """Generates responses using a greedy decoding strategy until certain ending conditions are met and
stores the logprobs.
Args:
- requests (list[Request]): list of requests containing the context and ending conditions.
+ docs (list[Doc]): List of documents containing the context and choices.
Returns:
list[ModelResponse]: list of generated responses.
diff --git a/src/lighteval/tasks/default_prompts.py b/src/lighteval/tasks/default_prompts.py
index 7f1544e98..b1d766155 100644
--- a/src/lighteval/tasks/default_prompts.py
+++ b/src/lighteval/tasks/default_prompts.py
@@ -180,10 +180,7 @@ def apps(line, task_name: str = None):
def arc_agi_2(line, task_name: str = None):
# query from: https://github.com/arcprize/model_baseline/blob/main/src/prompts/system_prompt.txt
def convert_2d_list_to_string(list_of_lists: list[list[int]]) -> str:
- """
- Convert a list of lists to a string
- """
-
+ """Convert a list of lists to a string"""
string_list = ""
for row in list_of_lists:
@@ -746,6 +743,9 @@ def _flatten_validated_answers(validated_answers):
"""Flattens a dict of lists of validated answers.
{"number": ['1', '8'], ...}
-> [{"number": ['1'], ...}, {"number": ['8'], ...}]
+
+ Returns:
+ list: List of dictionaries with flattened validated answers
"""
valid_answers = []
for i in range(len(validated_answers["number"])):
diff --git a/src/lighteval/tasks/default_tasks.py b/src/lighteval/tasks/default_tasks.py
index 12d1d3a15..8e3f286be 100644
--- a/src/lighteval/tasks/default_tasks.py
+++ b/src/lighteval/tasks/default_tasks.py
@@ -8590,7 +8590,7 @@
name="gsm8k",
suite=["lighteval"],
prompt_function=prompt.gsm8k,
- hf_repo="openai/gsm8k",
+ hf_repo="lighteval/openai/gsm8k",
hf_subset="main",
hf_avail_splits=["train", "test"],
evaluation_splits=["test"],
diff --git a/src/lighteval/tasks/extended/ifeval/instructions.py b/src/lighteval/tasks/extended/ifeval/instructions.py
index 6e84b6ef4..4022e640f 100644
--- a/src/lighteval/tasks/extended/ifeval/instructions.py
+++ b/src/lighteval/tasks/extended/ifeval/instructions.py
@@ -690,7 +690,6 @@ def check_following(self, value):
True if `value` and `instruction_args` only differ by the words/sentences
in between two asterisks such as *change me*; otherwise, False.
"""
-
if not self.is_change(value):
raise ValueError(f"value {value} does not contain changes in the form of *change me*.")
@@ -721,7 +720,6 @@ def build_description(self, *, keywords=None):
Returns:
A string representing the instruction description.
"""
-
if not keywords:
self._keywords = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS)
else:
@@ -741,7 +739,11 @@ def get_instruction_args_keys(self):
return ["keywords"]
def check_following(self, value):
- """Check if the response contain the expected keywords."""
+ """Check if the response contain the expected keywords.
+
+ Returns:
+ bool: True if the response contains all expected keywords, False otherwise
+ """
for keyword in self._keywords:
if not re.search(keyword, value, flags=re.IGNORECASE):
return False
@@ -835,7 +837,6 @@ def build_description(self, *, num_words=None, relation=None):
Returns:
A string representing the instruction description.
"""
-
self._num_words = num_words
if self._num_words is None or self._num_words < 0:
self._num_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT)
@@ -972,7 +973,6 @@ def check_following(self, value):
True if the number of paragraphs is the same as required and the first
word of the specified paragraph is the same as required. Otherwise, false.
"""
-
paragraphs = re.split(r"\n\n", value)
num_paragraphs = len(paragraphs)
@@ -1021,7 +1021,6 @@ def build_description(self, key_sentences=None, num_sentences=None):
Returns:
A string representing the instruction description.
"""
-
if not key_sentences:
# TODO(jeffrey) make a generate sentences function? wonderwords package
self._key_sentences = set("For now, this is fine.")
@@ -1072,7 +1071,6 @@ def build_description(self, forbidden_words=None):
Returns:
A string representing the instruction description.
"""
-
if not forbidden_words:
self._forbidden_words = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS)
else:
diff --git a/src/lighteval/tasks/extended/ifeval/instructions_utils.py b/src/lighteval/tasks/extended/ifeval/instructions_utils.py
index 63e7a9231..02d6ee2f9 100644
--- a/src/lighteval/tasks/extended/ifeval/instructions_utils.py
+++ b/src/lighteval/tasks/extended/ifeval/instructions_utils.py
@@ -1657,7 +1657,11 @@ def split_into_sentences(text):
def count_words(text):
- """Counts the number of words."""
+ """Counts the number of words (split on white spaces).
+
+ Returns:
+ int: The number of words in the text
+ """
tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+")
tokens = tokenizer.tokenize(text)
num_words = len(tokens)
@@ -1680,12 +1684,20 @@ def count_stopwords(text):
def count_sentences(text):
- """Count the number of sentences."""
+ """Count the number of sentences (split on ``).
+
+ Returns:
+ int: The number of sentences in the text
+ """
tokenizer = _get_sentence_tokenizer()
tokenized_sentences = tokenizer.tokenize(text)
return len(tokenized_sentences)
def generate_keywords(num_keywords):
- """Randomly generates a few keywords."""
+ """Randomly generates a few keywords.
+
+ Returns:
+ list[str]: List of randomly selected keywords
+ """
return random.sample(WORD_LIST, k=num_keywords)
diff --git a/src/lighteval/tasks/extended/ifeval/main.py b/src/lighteval/tasks/extended/ifeval/main.py
index 78bbe6e3f..1f63f91f1 100644
--- a/src/lighteval/tasks/extended/ifeval/main.py
+++ b/src/lighteval/tasks/extended/ifeval/main.py
@@ -22,10 +22,9 @@
import numpy as np
-from aenum import extend_enum
import lighteval.tasks.extended.ifeval.instructions_registry as instructions_registry
-from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.metrics_sample import SampleLevelComputation
from lighteval.metrics.utils.metric_utils import (
SampleLevelMetricGrouping,
)
@@ -58,69 +57,70 @@ def ifeval_prompt(line, task_name: str = ""):
]
-def ifeval_metric(doc: Doc, model_response: ModelResponse, **kwargs) -> dict:
- response = model_response.final_text[0]
-
- # Strict instructions
- instruction_list = doc.specific["instructions_id_list"]
- all_kwargs = doc.specific["kwargs"]
- prompt = doc.query
-
- # Loose instructions
- r = response.split("\n")
- response_remove_first = "\n".join(r[1:]).strip()
- response_remove_last = "\n".join(r[:-1]).strip()
- response_remove_both = "\n".join(r[1:-1]).strip()
- revised_response = response.replace("*", "")
- revised_response_remove_first = response_remove_first.replace("*", "")
- revised_response_remove_last = response_remove_last.replace("*", "")
- revised_response_remove_both = response_remove_both.replace("*", "")
- all_responses = [
- response,
- revised_response,
- response_remove_first,
- response_remove_last,
- response_remove_both,
- revised_response_remove_first,
- revised_response_remove_last,
- revised_response_remove_both,
- ]
-
- is_following_list_strict = []
- is_following_list_loose = []
-
- for index, instruction_id in enumerate(instruction_list):
- instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
- instruction = instruction_cls(instruction_id)
-
- # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
- task_kwargs = {k: v for k, v in all_kwargs[index].items() if v}
- instruction.build_description(**task_kwargs)
- args = instruction.get_instruction_args()
- if args and "prompt" in args:
- instruction.build_description(prompt=prompt)
-
- # Strict
- if response.strip() and instruction.check_following(response):
- is_following_list_strict.append(True)
- else:
- is_following_list_strict.append(False)
-
- # Loose
- is_following = False
- for r in all_responses:
- if r.strip() and instruction.check_following(r):
- is_following = True
- break
-
- is_following_list_loose.append(is_following)
-
- return {
- "prompt_level_strict_acc": int(all(is_following_list_strict)),
- "inst_level_strict_acc": is_following_list_strict,
- "prompt_level_loose_acc": int(all(is_following_list_loose)),
- "inst_level_loose_acc": is_following_list_loose,
- }
+class IFEvalMetrics(SampleLevelComputation):
+ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict:
+ response = model_response.final_text[0]
+
+ # Strict instructions
+ instruction_list = doc.specific["instructions_id_list"]
+ all_kwargs = doc.specific["kwargs"]
+ prompt = doc.query
+
+ # Loose instructions
+ r = response.split("\n")
+ response_remove_first = "\n".join(r[1:]).strip()
+ response_remove_last = "\n".join(r[:-1]).strip()
+ response_remove_both = "\n".join(r[1:-1]).strip()
+ revised_response = response.replace("*", "")
+ revised_response_remove_first = response_remove_first.replace("*", "")
+ revised_response_remove_last = response_remove_last.replace("*", "")
+ revised_response_remove_both = response_remove_both.replace("*", "")
+ all_responses = [
+ response,
+ revised_response,
+ response_remove_first,
+ response_remove_last,
+ response_remove_both,
+ revised_response_remove_first,
+ revised_response_remove_last,
+ revised_response_remove_both,
+ ]
+
+ is_following_list_strict = []
+ is_following_list_loose = []
+
+ for index, instruction_id in enumerate(instruction_list):
+ instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
+ instruction = instruction_cls(instruction_id)
+
+ # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
+ task_kwargs = {k: v for k, v in all_kwargs[index].items() if v}
+ instruction.build_description(**task_kwargs)
+ args = instruction.get_instruction_args()
+ if args and "prompt" in args:
+ instruction.build_description(prompt=prompt)
+
+ # Strict
+ if response.strip() and instruction.check_following(response):
+ is_following_list_strict.append(True)
+ else:
+ is_following_list_strict.append(False)
+
+ # Loose
+ is_following = False
+ for r in all_responses:
+ if r.strip() and instruction.check_following(r):
+ is_following = True
+ break
+
+ is_following_list_loose.append(is_following)
+
+ return {
+ "prompt_level_strict_acc": int(all(is_following_list_strict)),
+ "inst_level_strict_acc": is_following_list_strict,
+ "prompt_level_loose_acc": int(all(is_following_list_loose)),
+ "inst_level_loose_acc": is_following_list_loose,
+ }
def agg_inst_level_acc(items):
@@ -133,7 +133,7 @@ def agg_inst_level_acc(items):
metric_name=submetric_names,
higher_is_better=dict.fromkeys(submetric_names, True),
category=SamplingMethod.GENERATIVE,
- sample_level_fn=ifeval_metric,
+ sample_level_fn=IFEvalMetrics(),
corpus_level_fn={
"prompt_level_strict_acc": np.mean,
"inst_level_strict_acc": agg_inst_level_acc,
@@ -161,5 +161,3 @@ def agg_inst_level_acc(items):
TASKS_TABLE = [ifeval]
-
-extend_enum(Metrics, "ifeval_metric", ifeval_metrics)
diff --git a/src/lighteval/tasks/extended/lcb/codegen_metrics.py b/src/lighteval/tasks/extended/lcb/codegen_metrics.py
index 821a03247..98fad8858 100644
--- a/src/lighteval/tasks/extended/lcb/codegen_metrics.py
+++ b/src/lighteval/tasks/extended/lcb/codegen_metrics.py
@@ -425,17 +425,16 @@ def run_test(sample: dict[str, str], test=None, timeout: int = 6) -> list[int |
def reliability_guard(maximum_memory_bytes: Optional[int] = None):
- """
- This disables various destructive functions and prevents the generated code
+ """This disables various destructive functions and prevents the generated code
from interfering with the test (e.g. fork bomb, killing other processes,
removing filesystem files, etc.)
- WARNING
+
+ Warning:
This function is NOT a security sandbox. Untrusted code, including, model-
generated code, should not be blindly executed outside of one. See the
Codex paper for more information about OpenAI's code sandbox, and proceed
with caution.
"""
-
if maximum_memory_bytes is not None:
import resource
@@ -568,14 +567,15 @@ def evaluate_generations(
and the run their corresponding unit tests which are retrieved from the APPS dataset.
Args:
- generations: list of code generations (same order as samples in APPS dataset)
- level: difficulty level used in the generation, can be "all", "introductory", "interview" or "competition"
+ samples_list (list): List of sample problems from the APPS dataset.
+ generations_list (list[list[str]]): List of code generations for each sample.
+ num_process_evaluate (int): Number of processes to use for evaluation.
+ timeout (int): Timeout for each evaluation in seconds.
Returns:
- results: dictionary of results, key is the problem index, value is a list of results for each generation
+ dict[int, list[int | bool]]: Dictionary of results, key is the problem index, value is a list of results for each generation
[-2] = compile error, [-1] = runtime error [False] = failed test case [True] = passed test case
"""
-
# generations are code generations in the same order of the dataset
inputs = [
diff --git a/src/lighteval/tasks/extended/tiny_benchmarks/main.py b/src/lighteval/tasks/extended/tiny_benchmarks/main.py
index 88dcfb95e..44e05d0cc 100644
--- a/src/lighteval/tasks/extended/tiny_benchmarks/main.py
+++ b/src/lighteval/tasks/extended/tiny_benchmarks/main.py
@@ -21,8 +21,7 @@
# SOFTWARE.
# ruff: noqa: F405, F403, F401
-"""
-See https://github.com/felipemaiapolo/tinyBenchmarks/ for the original code.
+"""See https://github.com/felipemaiapolo/tinyBenchmarks/ for the original code.
Test with `python run_evals_accelerate.py --model_args "pretrained=EleutherAI/pythia-70m" --tasks "extended|tiny:winogrande|0,extended|tiny:gsm8k|0,extended|tiny:hellaswag|0,extended|tiny:arc|0,extended|tiny:truthfulqa|0" --extended_tasks extended_tasks --output_dir "./evals"`
"""
diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py
index 8488755b9..73956c835 100644
--- a/src/lighteval/tasks/lighteval_task.py
+++ b/src/lighteval/tasks/lighteval_task.py
@@ -46,26 +46,65 @@
@dataclass
class LightevalTaskConfig:
- """Stored configuration of a given [`LightevalTask`].
+ """Configuration dataclass for a LightevalTask.
- Arguments:
+ This class stores all the configuration parameters needed to define and run
+ an evaluation task, including dataset information, prompt formatting,
+ evaluation metrics, and generation parameters.
+
+ Args:
name (str): Short name of the evaluation task.
- suite (list[str]): Evaluation suites to which the task belongs.
- prompt_function (Callable[[dict, str], Doc]): Function used to create the [`Doc`] samples from each line of the evaluation dataset.
- hf_repo (str): Path of the hub dataset repository containing the evaluation information.
- hf_subset (str): Subset used for the current task, will be default if none is selected.
- hf_avail_splits (list[str]): All the available splits in the evaluation dataset
- evaluation_splits (list[str]): List of the splits actually used for this evaluation
- few_shots_split (str): Name of the split from which to sample few-shot examples
- few_shots_select (str): Method with which to sample few-shot examples
- generation_size (int): Maximum allowed size of the generation
- generation_grammar (TextGenerationInputGrammarType): The grammar to generate completion according to. Currently only available for TGI and Inference Endpoint models.
- metric (list[str]): List of all the metrics for the current task.
- stop_sequence (list[str]): Stop sequence which interrupts the generation for generative metrics.
- original_num_docs (int): Number of documents in the task
- effective_num_docs (int): Number of documents used in a specific evaluation
- truncated_num_docs (bool): Whether less than the total number of documents were used
- version (int): The version of the task. Defaults to 0. Can be increased if the underlying dataset or the prompt changes.
+ prompt_function (Callable[[dict, str], Doc]): Function that converts dataset
+ row to Doc objects for evaluation. Takes a dataset row dict and task
+ name as input.
+ hf_repo (str): HuggingFace Hub repository path containing the evaluation dataset.
+ hf_subset (str): Dataset subset/configuration name to use for this task.
+ metrics (ListLike[Metric]): List of metrics to compute for this task.
+
+ Dataset Configuration:
+ hf_revision (str | None, optional): Specific dataset revision to use.
+ Defaults to None (latest).
+ hf_filter (Callable[[dict], bool] | None, optional): Filter function to
+ apply to dataset items. Defaults to None.
+ hf_avail_splits (ListLike[str], optional): Available dataset splits.
+ Defaults to ["train", "validation", "test"].
+
+ Evaluation Splits:
+ evaluation_splits (ListLike[str], optional): Dataset splits to use for
+ evaluation. Defaults to ["validation"].
+ few_shots_split (str | None, optional): Split to sample few-shot examples
+ from. Defaults to None.
+ few_shots_select (str | None, optional): Method for selecting few-shot
+ examples. Defaults to None.
+
+ Generation Parameters:
+ generation_size (int | None, optional): Maximum token length for generated
+ text. Defaults to None.
+ generation_grammar (TextGenerationInputGrammarType | None, optional): Grammar
+ for structured text generation. Only available for TGI and Inference
+ Endpoint models. Defaults to None.
+ stop_sequence (ListLike[str] | None, optional): Sequences that stop text
+ generation. Defaults to None.
+ num_samples (list[int] | None, optional): Number of samples to generate
+ per input. Defaults to None.
+
+ Task Configuration:
+ suite (ListLike[str], optional): Evaluation suites this task belongs to.
+ Defaults to ["custom"].
+ version (int, optional): Task version number. Increment when dataset or
+ prompt changes. Defaults to 0.
+ num_fewshots (int, optional): Number of few-shot examples to include.
+ Defaults to 0.
+ truncate_fewshots (bool, optional): Whether to truncate few-shot examples.
+ Defaults to False.
+ must_remove_duplicate_docs (bool, optional): Whether to remove duplicate
+ documents. Defaults to False.
+
+ Document Tracking:
+ original_num_docs (int, optional): Total number of documents in the task.
+ Defaults to -1.
+ effective_num_docs (int, optional): Number of documents actually used
+ in evaluation. Defaults to -1.
"""
name: str
@@ -146,8 +185,7 @@ def __init__(
self,
config: LightevalTaskConfig,
):
- """
- Initialize a LightEval task.
+ """Initialize a LightEval task.
Args:
config (dict): configuration dictionary containing
@@ -197,8 +235,7 @@ def __init__(
self.num_samples.append(metric.sample_level_fn.num_samples())
def get_first_possible_fewshot_splits(self, available_splits: ListLike[str]) -> str | None:
- """
- Parses the possible fewshot split keys in order: train, then validation
+ """Parses the possible fewshot split keys in order: train, then validation
keys and matches them with the available keys. Returns the first
available.
@@ -223,13 +260,12 @@ def get_first_possible_fewshot_splits(self, available_splits: ListLike[str]) ->
return None
def _get_docs_from_split(self, splits: list[str], few_shots=False) -> list[Doc]:
- """
- Get the documents from the dataset for the given keys (splits).
+ """Get the documents from the dataset for the given keys (splits).
Args:
- splits (list[str]): List of splits, (e.g. ["train", "dev"])
- few_shots (bool, optional): Whether the documents are used for few
- shot examples. Defaults to False.
+ splits (list[str]): List of dataset splits to process (e.g. ["train", "dev"])
+ few_shots (bool, optional): Whether the documents are used for few-shot
+ examples. This affects how the formatter processes the items. Defaults to False.
Returns:
list[Doc]: List of documents.
@@ -269,8 +305,7 @@ def remove_duplicate_docs(self, docs: list[Doc]) -> list[Doc]:
return res
def fewshot_docs(self) -> list[Doc]:
- """
- Returns the few shot documents. If the few shot documents are not
+ """Returns the few shot documents. If the few shot documents are not
available, it gets them from the few shot split or the evaluation split.
Returns:
@@ -289,8 +324,7 @@ def fewshot_docs(self) -> list[Doc]:
return self._fewshot_docs
def eval_docs(self) -> list[Doc]:
- """
- Returns the evaluation documents.
+ """Returns the evaluation documents.
Returns:
list[Doc]: Evaluation documents.
@@ -302,6 +336,23 @@ def eval_docs(self) -> list[Doc]:
return self._docs
def get_docs(self, max_samples: int | None = None) -> list[Doc]:
+ """Get evaluation documents with few-shot examples and generation parameters configured.
+
+ Retrieves evaluation documents, optionally limits the number of samples,
+ shuffles them for reproducibility, and configures each document with
+ few-shot examples and generation parameters for evaluation.
+
+ Args:
+ max_samples (int | None, optional): Maximum number of documents to return.
+ If None, returns all available documents. Defaults to None.
+
+ Returns:
+ list[Doc]: List of documents ready for evaluation with few-shot examples
+ and generation parameters configured.
+
+ Raises:
+ ValueError: If no documents are available for evaluation.
+ """
eval_docs = self.eval_docs()
if len(eval_docs) == 0:
@@ -330,8 +381,7 @@ def get_docs(self, max_samples: int | None = None) -> list[Doc]:
return docs
def aggregation(self):
- """
- Return a dict with metric name and its aggregation function for all
+ """Return a dict with metric name and its aggregation function for all
metrics
"""
aggregations = {}
@@ -341,17 +391,13 @@ def aggregation(self):
@staticmethod
def load_datasets(tasks: dict[str, "LightevalTask"], dataset_loading_processes: int = 1) -> None:
- """
- Load datasets from the HuggingFace Hub for the given tasks.
+ """Load datasets from the HuggingFace Hub for the given tasks.
Args:
- tasks (list): A list of tasks.
- dataset_loading_processes (int, optional): number of processes to use for dataset loading. Defaults to 1.
-
- Returns:
- None
+ tasks (dict[str, LightevalTask]): Dictionary mapping task names to task objects.
+ dataset_loading_processes (int, optional): Number of processes to use for
+ parallel dataset loading. Defaults to 1 (sequential loading).
"""
-
if dataset_loading_processes <= 1:
# Useful for the test suite: we can mock loading tasks by overwriting the
# individual download_dataset_worker functions
@@ -370,9 +416,17 @@ def load_datasets(tasks: dict[str, "LightevalTask"], dataset_loading_processes:
def download_dataset_worker(
task: "LightevalTask",
) -> DatasetDict:
- """
- Worker function to download a dataset from the HuggingFace Hub.
- Used for parallel dataset loading.
+ """Worker function to download a dataset from the HuggingFace Hub.
+
+ Downloads the dataset specified in the task configuration, optionally
+ applies a filter if configured, and returns the dataset dictionary.
+ This method is designed to be used for parallel dataset loading.
+
+ Args:
+ task (LightevalTask): The task object containing dataset configuration.
+
+ Returns:
+ DatasetDict: The loaded dataset dictionary containing all splits.
"""
dataset = load_dataset(
path=task.dataset_path,
diff --git a/src/lighteval/tasks/multilingual/utils/adapters_utils.py b/src/lighteval/tasks/multilingual/utils/adapters_utils.py
index 2e7f27dda..b82e9c434 100644
--- a/src/lighteval/tasks/multilingual/utils/adapters_utils.py
+++ b/src/lighteval/tasks/multilingual/utils/adapters_utils.py
@@ -31,8 +31,10 @@
def multichoice_join(choices: list[str], variant: MULTICHOICE_JOIN_VARIANT, translation_literals: TranslationLiterals):
- """
- Joins the choices with the appropriate separator.
+ """Joins the choices with the appropriate separator.
+
+ Returns:
+ str: The joined choices string with appropriate separators
"""
separator: str
if variant == "COMMA":
@@ -49,13 +51,14 @@ def multichoice_to_single_choice(
join_variant: MULTICHOICE_JOIN_VARIANT,
translation_literals: TranslationLiterals,
):
- """
- Converts from multi-choice format to single-choice format, by joining the correct choices with the appropriate separator.
+ """Converts from multi-choice format to single-choice format, by joining the correct choices with the appropriate separator.
+
Args:
choices (list[str]): List of choices.
gold_idx (list[int]): List of indices of the correct choices.
join_variant (MULTICHOICE_JOIN_VARIANT): Variant of the separator to join the choices.
translation_literals (TranslationLiterals): Translation literals.
+
Returns:
tuple[list[str], list[int]]: Tuple of the new choices and the new gold index.
"""
@@ -78,8 +81,7 @@ def multichoice_to_single_choice(
def extract_answers_from_string(answer_string: str, answer_prefixes: list[str]) -> tuple[int, dict[str, str]] | None:
- """
- Attempts to extract answers from the answer_string. The answers are identified by being prefixed with answer prefixes.
+ """Attempts to extract answers from the answer_string. The answers are identified by being prefixed with answer prefixes.
The extraction is done from the end to the beginning and all answer prefixes must be found in the answer_string.
Example:
@@ -94,16 +96,18 @@ def extract_answers_from_string(answer_string: str, answer_prefixes: list[str])
Args:
answer_string (str): String possibly containing answers.
answer_prefixes (list[str]): List of answer prefixes.
+
Returns:
Optional[tuple[int, dict[str, str]]]: A tuple containing the start index of the answer and dictionary mapping the prefix to the answer.
"""
def extract_answer(acc: tuple[str, int, list[str]], symbol: str) -> tuple[str, int, list[str]]:
- """
- Extracts an answer from the text until the next symbol is found.
+ """Extracts an answer from the text until the next symbol is found.
+
Args:
acc (tuple[str, int, list[str]]): Tuple containing the text, the right index (where to start searching) and the list of found answers.
symbol (str): Symbol to extract the answer from.
+
Returns:
tuple[str, int, list[str]]: Tuple containing the text, the right index and the list of answers.
"""
diff --git a/src/lighteval/tasks/multilingual/utils/task_utils.py b/src/lighteval/tasks/multilingual/utils/task_utils.py
index d439eed16..5a116f6a3 100644
--- a/src/lighteval/tasks/multilingual/utils/task_utils.py
+++ b/src/lighteval/tasks/multilingual/utils/task_utils.py
@@ -31,9 +31,7 @@ def normalize_subset(subset: str) -> str:
def get_metrics_for_formulation(formulation: Formulation, metrics: list[Metric]) -> list[Metric]:
- """
- Choose the appropriate metrics for the given formulation otherwise fallback to the original metrics.
- """
+ """Choose the appropriate metrics for the given formulation otherwise fallback to the original metrics."""
match formulation:
#
case MCFFormulation(choice_prefix="Letters"):
diff --git a/src/lighteval/tasks/prompt_manager.py b/src/lighteval/tasks/prompt_manager.py
index 6b7068bd8..2c854281d 100644
--- a/src/lighteval/tasks/prompt_manager.py
+++ b/src/lighteval/tasks/prompt_manager.py
@@ -46,7 +46,11 @@ def __init__(self, use_chat_template: bool = False, tokenizer=None, system_promp
self.system_prompt = system_prompt # System prompt to be used in chat templates
def prepare_prompt(self, doc: Doc) -> str:
- """Prepare a prompt from a document, either using chat template or plain text format."""
+ """Prepare a prompt from a document, either using chat template or plain text format.
+
+ Returns:
+ str: The formatted prompt string
+ """
if self.use_chat_template:
return self._prepare_chat_template(doc)
else:
@@ -82,14 +86,20 @@ def prepare_prompt_multimodal(self, doc: Doc) -> str:
)
def prepare_prompt_api(self, doc: Doc) -> list[dict[str, str]]:
- """
- Prepare a prompt for API calls, using a chat-like format.
+ """Prepare a prompt for API calls, using a chat-like format.
Will not tokenize the message because APIs will usually handle this.
+
+ Returns:
+ list[dict[str, str]]: List of message dictionaries for API calls
"""
return self._prepare_chat_template(doc, tokenize=False)
def _prepare_chat_template(self, doc: Doc, tokenize: bool = True) -> str:
- """Prepare prompt using chat template format."""
+ """Prepare prompt using chat template format.
+
+ Returns:
+ str | list[dict[str, str]]: Formatted chat template string or list of messages
+ """
messages = []
instruction_used = False # Flag to check if instruction is used in the first few-shot example
@@ -129,7 +139,11 @@ def _prepare_chat_template(self, doc: Doc, tokenize: bool = True) -> str:
return messages
def _prepare_plain_text(self, doc: Doc) -> str:
- """Prepare prompt using plain text format."""
+ """Prepare prompt using plain text format.
+
+ Returns:
+ str: The formatted plain text prompt
+ """
parts = []
# Add system prompt if available
@@ -151,7 +165,11 @@ def _prepare_plain_text(self, doc: Doc) -> str:
return "\n\n".join(parts)
def _extract_query(self, query: str, instruction: str | None) -> str:
- """Extract query content, removing instruction prefix if appropriate."""
+ """Extract query content, removing instruction prefix if appropriate.
+
+ Returns:
+ str: The extracted query content without instruction prefix if it was present
+ """
if instruction is not None:
if query.startswith(instruction):
return query[len(instruction) :].strip()
diff --git a/src/lighteval/tasks/registry.py b/src/lighteval/tasks/registry.py
index 13ba3d00b..f64aa443b 100644
--- a/src/lighteval/tasks/registry.py
+++ b/src/lighteval/tasks/registry.py
@@ -49,7 +49,11 @@
def load_community_tasks():
- """Dynamically load community tasks, handling errors gracefully."""
+ """Dynamically load community tasks, handling errors gracefully.
+
+ Returns:
+ list: List of successfully loaded community task modules
+ """
modules = []
try:
# Community tasks are in the lighteval directory, not under src
@@ -111,9 +115,7 @@ def load_community_tasks():
class Registry:
- """
- The Registry class is used to manage the task registry and get task classes.
- """
+ """The Registry class is used to manage the task registry and get task classes."""
def __init__(
self,
@@ -128,16 +130,27 @@ def __init__(
Registry is responsible for holding a dict of task and their config, initializing a LightevalTask instance when asked.
Args:
- custom_tasks (Optional[Union[str, Path, ModuleType]]): Custom tasks to be included in registry. Can be a string path, Path object, or a module.
- Each custom task should be a module with a TASKS_TABLE exposing a list of LightevalTaskConfig.
- E.g:
- TASKS_TABLE = [
- LightevalTaskConfig(
- name="custom_task",
- suite="custom",
- ...
- )
- ]
+ tasks: Task specification string or path to file containing task list.
+ custom_tasks: Custom tasks to be included in the registry. Can be:
+ - A string path to a Python file containing custom tasks
+ - A Path object pointing to a custom tasks file
+ - A module object containing custom task configurations
+ - None for default behavior (no custom tasks)
+ load_community: Whether to load community-contributed tasks.
+ load_extended: Whether to load extended tasks with custom logic.
+ load_multilingual: Whether to load multilingual tasks.
+
+ Each custom task module should contain a TASKS_TABLE exposing
+ a list of LightevalTaskConfig objects.
+
+ Example:
+ TASKS_TABLE = [
+ LightevalTaskConfig(
+ name="custom_task",
+ suite="custom",
+ ...
+ )
+ ]
"""
self._custom_tasks = custom_tasks
@@ -342,8 +355,7 @@ def load_tasks(self) -> dict[str, LightevalTask]:
@property
@lru_cache
def _task_superset_dict(self):
- """
- Returns:
+ """Returns:
dict[str, list[str]]: A dictionary where keys are task super set names (suite|task) and values are lists of task subset names (suite|task).
Example:
@@ -363,10 +375,10 @@ def _expand_task_definition(self, task_definition: str):
task_definition (str): Task definition to expand. In format:
- suite|task
- suite|task_superset (e.g lighteval|mmlu, which runs all the mmlu subtasks)
+
Returns:
list[str]: List of task names (suite|task)
"""
-
# Try if it's a task superset
tasks = self._task_superset_dict.get(task_definition, None)
if tasks is not None:
@@ -402,8 +414,7 @@ def create_custom_tasks_module(custom_tasks: str | Path | ModuleType) -> ModuleT
@staticmethod
def create_task_config_dict(meta_table: list[LightevalTaskConfig] | None = None) -> dict[str, LightevalTaskConfig]:
- """
- Create configuration tasks based on the provided meta_table.
+ """Create configuration tasks based on the provided meta_table.
Args:
meta_table: meta_table containing tasks
@@ -412,7 +423,6 @@ def create_task_config_dict(meta_table: list[LightevalTaskConfig] | None = None)
Returns:
Dict[str, LightevalTaskConfig]: A dictionary of task names mapped to their corresponding LightevalTaskConfig.
"""
-
if meta_table is None:
meta_table = [config for config in vars(default_tasks).values() if isinstance(config, LightevalTaskConfig)]
diff --git a/src/lighteval/tasks/requests.py b/src/lighteval/tasks/requests.py
index 30da2adea..35b04f5c9 100644
--- a/src/lighteval/tasks/requests.py
+++ b/src/lighteval/tasks/requests.py
@@ -33,9 +33,7 @@
class SamplingMethod(str, Enum):
- """
- Enum representing different sampling methods for text generation.
- """
+ """Enum representing different sampling methods for text generation."""
GENERATIVE = "GENERATIVE"
LOGPROBS = "LOGPROBS" # computes logprobs of choices
@@ -44,8 +42,7 @@ class SamplingMethod(str, Enum):
@dataclass(slots=True)
class Doc:
- """
- Dataclass representing a single evaluation sample for a benchmark.
+ """Dataclass representing a single evaluation sample for a benchmark.
This class encapsulates all the information needed to evaluate a model on a single
task instance. It contains the input query, expected outputs, metadata, and
diff --git a/src/lighteval/tasks/templates/boolq.py b/src/lighteval/tasks/templates/boolq.py
index 09959e874..5f2349af9 100644
--- a/src/lighteval/tasks/templates/boolq.py
+++ b/src/lighteval/tasks/templates/boolq.py
@@ -33,8 +33,8 @@
# Defined for type hinting only
class BoolQInput(TypedDict):
- """
- Input for the BoolQ task.
+ """Input for the BoolQ task.
+
Args:
question: The question of the BoolQ task (e.g. What is the capital of Germany?)
answer: The answer of the BoolQ task (True = yes, False = no)
@@ -49,8 +49,8 @@ class BoolQInput(TypedDict):
class BoolQAdapter(TypedDict):
- """
- Adapter for mapping from the dataset row into the BoolQInput format.
+ """Adapter for mapping from the dataset row into the BoolQInput format.
+
Args:
question: Column name in the row that contains the question of the BoolQ task (e.g. What is the capital of Germany?)
answer: Column name in the row that contains the answer of the BoolQ task (True = yes, False = no)
@@ -69,8 +69,7 @@ def get_boolq_prompt_function(
adapter: Callable[[dict], BoolQInput | None] | BoolQAdapter,
formulation: Formulation = MCFFormulation(),
):
- """
- Create a templated prompt function for a BoolQ task.
+ """Create a templated prompt function for a BoolQ task.
It leverages the translation literals (yes/no) for the choices. All other logic is the same as the mcq prompt function.
Example tasks:
- boolQ
@@ -78,6 +77,9 @@ def get_boolq_prompt_function(
Format:
See mcq prompt function for the format.
+
+ Returns:
+ Callable: A function that generates BoolQ prompts based on the given parameters
"""
translation_literals = TRANSLATION_LITERALS[language]
diff --git a/src/lighteval/tasks/templates/continuation.py b/src/lighteval/tasks/templates/continuation.py
index c9cd5d1bc..179cc551b 100644
--- a/src/lighteval/tasks/templates/continuation.py
+++ b/src/lighteval/tasks/templates/continuation.py
@@ -51,8 +51,8 @@
# Defined for type hinting only
class ContinuationInput(TypedDict):
- """
- Input for the continuation task.
+ """Input for the continuation task.
+
Args:
context: The contextualization of choices (e.g. If I ask you a question, you should answer it)
continuations: Possible continuations of the context (e.g. [you should answer it, you should leave])
@@ -67,8 +67,8 @@ class ContinuationInput(TypedDict):
class ContinuationDictAdapter(TypedDict):
- """
- Adapter for mapping from the dataset row into the ContinuationInput format.
+ """Adapter for mapping from the dataset row into the ContinuationInput format.
+
Args:
context: Column name in the row that contains the contextualization of choices (e.g. If I ask you a question, you should answer it)
continuations: Column name in the row that contains the possible continuations of the context (e.g. [you should answer it, you should leave])
@@ -88,8 +88,7 @@ def get_continuation_prompt_function(
formulation: Formulation = MCFFormulation(),
fix_formatting: bool = True,
):
- """
- Create a templated prompt function for a Continuation task.
+ """Create a templated prompt function for a Continuation task.
Example tasks:
- Hellaswag
- XStoryCloze
@@ -120,6 +119,7 @@ def get_continuation_prompt_function(
Note: Both ContinuationDictAdapter and ContinuationInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class!
formulation (Formulation, optional): The formulation (MCF/Hybrid/CF) to use for the task. Defaults to MCFFormulation().
fix_formatting (bool, optional): Whether to fix the formatting of the text by capitalizing and fixing punctuation based on language. If False, the text will be used as-is. Defaults to True.
+
Returns:
Callable: A function that generates Continuation prompt based on the given parameters.
"""
diff --git a/src/lighteval/tasks/templates/copa.py b/src/lighteval/tasks/templates/copa.py
index a4d82c4de..b4040aa69 100644
--- a/src/lighteval/tasks/templates/copa.py
+++ b/src/lighteval/tasks/templates/copa.py
@@ -38,8 +38,8 @@
# Defined for type hinting only
class COPAInput(TypedDict):
- """
- Input for the COPA task.
+ """Input for the COPA task.
+
Args:
cause_effect: The type of the COPA task (X therefore Y / X because Y)
context: The contextualization of choices (e.g. You are young)
@@ -56,8 +56,8 @@ class COPAInput(TypedDict):
class COPAAdapter(TypedDict):
- """
- Adapter for mapping from the dataset row into the COPAInput format.
+ """Adapter for mapping from the dataset row into the COPAInput format.
+
Args:
cause_effect: Column name in the row that contains the type of the COPA task (X therefore Y / X because Y)
context: Column name in the row that contains the contextualization of choices (e.g. You are young)
@@ -78,8 +78,7 @@ def get_copa_prompt_function(
adapter: Callable[[dict], COPAInput | None] | COPAAdapter,
formulation: Formulation = MCFFormulation(),
):
- """
- Create a templated prompt function for a COPA task.
+ """Create a templated prompt function for a COPA task.
Example tasks:
- COPA
- PARUS
@@ -108,6 +107,7 @@ def get_copa_prompt_function(
Note: Both COPAAdapter and COPAInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class!
Note: The gold_idx must be an index or list of indices in the continuations list, indicating the correct continuation(s).
formulation (Formulation, optional): The formulation to use for the task. Defaults to MCFFormulation().
+
Returns:
Callable: A function that generates COPA prompts based on the given parameters.
"""
diff --git a/src/lighteval/tasks/templates/hellaswag.py b/src/lighteval/tasks/templates/hellaswag.py
index 79108a9cd..9970edd82 100644
--- a/src/lighteval/tasks/templates/hellaswag.py
+++ b/src/lighteval/tasks/templates/hellaswag.py
@@ -66,8 +66,7 @@ def get_hellaswag_prompt_function(
formulation: Formulation = MCFFormulation(),
wikihow_artifacts: list[str] = [" [title]"],
):
- """
- Create a templated prompt function for a Hellaswag task.
+ """Create a templated prompt function for a Hellaswag task.
Format:
Context Premise therefore/cause | (Continuation 1, Continuation 2, Continuation 3)
@@ -84,7 +83,6 @@ def get_hellaswag_prompt_function(
Returns:
Callable: A function that generates COPA prompts based on the given parameters.
"""
-
translation_literals = TRANSLATION_LITERALS[language]
def process_context(ctx):
diff --git a/src/lighteval/tasks/templates/multichoice.py b/src/lighteval/tasks/templates/multichoice.py
index 92808488e..2d597c29e 100644
--- a/src/lighteval/tasks/templates/multichoice.py
+++ b/src/lighteval/tasks/templates/multichoice.py
@@ -40,8 +40,8 @@
# Defined for type hinting only
class MCQInput(TypedDict):
- """
- Input for the multiple choice task.
+ """Input for the multiple choice task.
+
Args:
question: The question to be answered (e.g. What is the capital of France?)
choices: Possible choices for the question (e.g. [Paris, London, Berlin, Rome])
@@ -58,8 +58,8 @@ class MCQInput(TypedDict):
class MCQDictAdapter(TypedDict):
- """
- Adapter for mapping from the dataset row into the MCQInput format.
+ """Adapter for mapping from the dataset row into the MCQInput format.
+
Args:
question: Column name in the row that contains the question to be answered (e.g. What is the capital of France?)
choices: Column name in the row that contains the possible choices for the question (e.g. [Paris, London, Berlin, Rome])
@@ -83,8 +83,7 @@ def get_mcq_prompt_function(
adapter: Callable[[dict], MCQInput | None] | MCQDictAdapter,
formulation: Formulation = MCFFormulation(),
):
- """
- Create a templated prompt function for a Multiple Choice Question (MCQ) task.
+ """Create a templated prompt function for a Multiple Choice Question (MCQ) task.
Example tasks:
- ARC
- TruthfulQA
@@ -119,7 +118,6 @@ def get_mcq_prompt_function(
Returns:
Callable: A function that generates MCQ prompts based on the given parameters.
"""
-
adapter_fn = create_adapter_from_dict(adapter)
def prompt_fn(line, task_name: str):
diff --git a/src/lighteval/tasks/templates/nli.py b/src/lighteval/tasks/templates/nli.py
index e8809e17b..06908401f 100644
--- a/src/lighteval/tasks/templates/nli.py
+++ b/src/lighteval/tasks/templates/nli.py
@@ -38,8 +38,8 @@
class NLIInput(TypedDict):
- """
- Input for the Natural Language Inference (NLI) task.
+ """Input for the Natural Language Inference (NLI) task.
+
Args:
premise: The premise of the NLI task (e.g. He can speak French)
hypothesis: The hypothesis of the NLI task (e.g. He can speak anglo-saxon language)
@@ -54,8 +54,8 @@ class NLIInput(TypedDict):
class NLIAdapter(TypedDict):
- """
- Adapter for mapping from the dataset row into the NLIInput format.
+ """Adapter for mapping from the dataset row into the NLIInput format.
+
Args:
premise: Column name in the row that contains the premise of the NLI task (e.g. He can speak French)
hypothesis: Column name in the row that contains the hypothesis of the NLI task (e.g. He can speak anglo-saxon language)
@@ -77,8 +77,7 @@ def _nli_prompt_function_natural(
adapter: Callable[[dict], NLIInput | None] | NLIAdapter,
relations: list[RelationType],
):
- """
- Create a templated prompt function for a Natural Language Inference (NLI) task.
+ """Create a templated prompt function for a Natural Language Inference (NLI) task.
Example tasks:
- ANLI
- XNLI
@@ -166,8 +165,7 @@ def get_nli_prompt_function(
relations: list[RelationType],
formulation: Formulation = MCFFormulation(),
):
- """
- Create a templated prompt function for a Natural Language Inference (NLI) task.
+ """Create a templated prompt function for a Natural Language Inference (NLI) task.
Example tasks:
- ANLI
- XNLI
diff --git a/src/lighteval/tasks/templates/qa.py b/src/lighteval/tasks/templates/qa.py
index e798f820d..3c5074b7d 100644
--- a/src/lighteval/tasks/templates/qa.py
+++ b/src/lighteval/tasks/templates/qa.py
@@ -44,8 +44,7 @@ class QAAdapter(TypedDict):
def get_qa_prompt_function(language: Language, adapter: Callable[[dict], QAInput | None] | QAAdapter):
- """
- Create a templated prompt function for a QA task.
+ """Create a templated prompt function for a QA task.
Example tasks:
- XQuAD
- SQuAD
@@ -62,7 +61,6 @@ def get_qa_prompt_function(language: Language, adapter: Callable[[dict], QAInput
Returns:
Callable: A function that generates QA prompts based on the given parameters.
"""
-
adapter_fn = create_adapter_from_dict(adapter)
def adapter_for_mcq(line: dict) -> MCQInput | None:
diff --git a/src/lighteval/tasks/templates/translation.py b/src/lighteval/tasks/templates/translation.py
index c90b99e01..6b4c54a62 100644
--- a/src/lighteval/tasks/templates/translation.py
+++ b/src/lighteval/tasks/templates/translation.py
@@ -43,8 +43,8 @@
# Defined for type hinting only
class TranslationInput(TypedDict):
- """
- Input for the Translation task.
+ """Input for the Translation task.
+
Args:
source_text: The source text to be translated
target_text: The target text to be translated
@@ -58,8 +58,8 @@ class TranslationInput(TypedDict):
class TranslationAdapter(TypedDict):
- """
- Adapter for mapping from the dataset row into the TranslationInput format.
+ """Adapter for mapping from the dataset row into the TranslationInput format.
+
Args:
source_text: Column name in the row that contains the source text to be translated
target_text: Column name in the row that contains the target text to be translated
@@ -78,8 +78,7 @@ def get_translation_prompt_function(
adapter: Callable[[dict], TranslationInput | None] | TranslationAdapter,
formulation: Formulation = MCFFormulation(),
):
- """
- Create a templated prompt function for a Translation task.
+ """Create a templated prompt function for a Translation task.
Example tasks:
- WMT2016
- WMT2017
@@ -101,9 +100,12 @@ def get_translation_prompt_function(
Answer: | A/B
Args:
+ source_language (Language): The source language for translation.
+ target_language (Language): The target language for translation.
adapter (Callable[[dict], TranslationInput] | TranslationAdapter): Either a function that takes a dataset row and returns a TranslationInput, or a dictionary with keys corresponding to the field names in the dataset row.
Note: Both TranslationAdapter and TranslationInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class!
formulation (Formulation, optional): The formulation to use for the task. Defaults to MCFFormulation().
+
Returns:
Callable: A function that generates Translation prompts based on the given parameters.
"""
diff --git a/src/lighteval/tasks/templates/utils/adapter_utils.py b/src/lighteval/tasks/templates/utils/adapter_utils.py
index c260e5682..f4dcbe80c 100644
--- a/src/lighteval/tasks/templates/utils/adapter_utils.py
+++ b/src/lighteval/tasks/templates/utils/adapter_utils.py
@@ -30,13 +30,14 @@
def create_adapter_from_dict(
adapter: Mapping[str, Any] | Callable[[dict], AdapterReturnTypeVar],
) -> Callable[[dict], AdapterReturnTypeVar]:
- """
- Creates adapter function for the template input from a dict.
+ """Creates adapter function for the template input from a dict.
+
Args:
adapter: Dict of the form {key: value} where value is key in the input dict to get.
+ Returns:
+ Callable[[dict], AdapterReturnTypeVar]: A function that adapts dictionary input to the expected format
"""
-
if not isinstance(adapter, Mapping):
return adapter
diff --git a/src/lighteval/tasks/templates/utils/formatting_utils.py b/src/lighteval/tasks/templates/utils/formatting_utils.py
index 0ee3c3e0f..2d2daace0 100644
--- a/src/lighteval/tasks/templates/utils/formatting_utils.py
+++ b/src/lighteval/tasks/templates/utils/formatting_utils.py
@@ -28,8 +28,10 @@
def decapitalize(word: str):
- """
- Decapitalize the first letter of the string
+ """Decapitalize the first letter of the string
+
+ Returns:
+ str: The string with the first letter decapitalized
"""
if len(word) == 0:
return word
@@ -37,8 +39,10 @@ def decapitalize(word: str):
def capitalize(word: str):
- """
- Capitalize the first letter of the string
+ """Capitalize the first letter of the string
+
+ Returns:
+ str: The string with the first letter capitalized
"""
if len(word) == 0:
return word
@@ -46,9 +50,11 @@ def capitalize(word: str):
def fix_ending_punct(ctx: str, translation_literals: TranslationLiterals):
- """
- Fixes the ending punctuation so that it uses the correct punctuation for the language.
+ """Fixes the ending punctuation so that it uses the correct punctuation for the language.
E.g in Chinese the "?" is written as "？"
+
+ Returns:
+ str: The string with corrected punctuation for the language
"""
ctx = ctx.rstrip()
if len(ctx) == 0:
@@ -65,10 +71,12 @@ def fix_ending_punct(ctx: str, translation_literals: TranslationLiterals):
def punctuation_ends_sentence(text: str, translation_literals: TranslationLiterals):
- """
- Check if the string ends with a sentence-ending punctuation mark.
+ """Check if the string ends with a sentence-ending punctuation mark.
That's .?!:
It's not perfect method as some languages don't have full stops etc..
+
+ Returns:
+ bool: True if the string ends with sentence-ending punctuation, False otherwise
"""
return text.rstrip().endswith(
(
@@ -82,11 +90,13 @@ def punctuation_ends_sentence(text: str, translation_literals: TranslationLitera
def fix_capitalization(prefix: str, text: str, translation_literals: TranslationLiterals):
- """
- Fixes the capitalization of the text based on the prefix.
+ """Fixes the capitalization of the text based on the prefix.
It's based on simple heuristics:
- If the prefix ends with a sentence-ending punctuation mark, the text should be capitalized.
- If the prefix ends with a newline, the text should be capitalized.
+
+ Returns:
+ str: The text with appropriate capitalization based on the prefix
"""
if len(prefix) == 0:
return capitalize(text)
diff --git a/src/lighteval/tasks/templates/utils/formulation.py b/src/lighteval/tasks/templates/utils/formulation.py
index 6447e01dc..7e2c3af9e 100644
--- a/src/lighteval/tasks/templates/utils/formulation.py
+++ b/src/lighteval/tasks/templates/utils/formulation.py
@@ -32,8 +32,7 @@
@dataclass
class MCFFormulation:
- """
- MCF Formulation
+ """MCF Formulation
Presenting the choices as A. B. C.
The target is A, B, C
@@ -47,8 +46,7 @@ class MCFFormulation:
@dataclass
class HybridFormulation:
- """
- Hybrid Formulation
+ """Hybrid Formulation
Presenting the choices as A. B. C.
The target is then the answer itself not A, B, C
@@ -62,8 +60,7 @@ class HybridFormulation:
@dataclass
class CFFormulation:
- """
- CF Formulation
+ """CF Formulation
No choices are presented, the target is the answer itself
"""
@@ -88,8 +85,7 @@ def build_choices(
translation_literals: TranslationLiterals,
use_sentence_space: bool = True,
):
- """
- Builds a string version of the choices based on passed formulation for available options presentation.
+ """Builds a string version of the choices based on passed formulation for available options presentation.
For Hybrid/MCF, the choices are presented as A. OptionA B. OptionB C. OptionC etc.
For CF no choices are presented
@@ -101,6 +97,9 @@ def build_choices(
The same value should be passed to `build_answers` function to ensure consistent tokenization.
Defaults to True.
+
+ Returns:
+ str | None: Formatted choices string for Hybrid/MCF formulations, None for CF formulation
"""
if isinstance(formulation, CFFormulation):
return None
@@ -132,8 +131,7 @@ def build_answers(
translation_literals: TranslationLiterals,
use_sentence_space: bool = True,
) -> list[str]:
- """
- Builds a string version of the answers based on passed formulation.
+ """Builds a string version of the answers based on passed formulation.
For MCF, the answers are presented as A, B, C etc.
For Hybrid/CF, the answers are presented as the answer itself.
@@ -143,6 +141,9 @@ def build_answers(
translation_literals (TranslationLiterals): The translation literals scoped to required language.
use_sentence_space (bool, optional): Whether to use sentence or word space in front of the answer. Defaults to True.
The same value should be passed to `build_choices` function to ensure consistent tokenization.
+
+ Returns:
+ list[str]: List of formatted answer strings
"""
if isinstance(formulation, MCFFormulation):
prefixes = get_prefix(formulation.choice_prefix, translation_literals)
diff --git a/src/lighteval/utils/cache_management.py b/src/lighteval/utils/cache_management.py
index c1c144cdf..70435d715 100644
--- a/src/lighteval/utils/cache_management.py
+++ b/src/lighteval/utils/cache_management.py
@@ -48,8 +48,7 @@ class SampleType(Enum):
class SampleCache:
- """
- Disk-based cache for sample evaluation results using HuggingFace datasets.
+ """Disk-based cache for sample evaluation results using HuggingFace datasets.
The model hash is a hash of the model config, to make sure we rerun the eval if any parameter changes
(generation param, model version, etc).
@@ -62,8 +61,7 @@ class SampleCache:
"""
def __init__(self, model_config: ModelConfig):
- """
- Initialize the sample cache.
+ """Initialize the sample cache.
Args:
model_config: Configuration for the model being cached
@@ -116,7 +114,11 @@ def _get_cached_indices(self, sample_type: SampleType) -> dict:
return cached_indices
def get_model_hash(self, model_config: ModelConfig) -> str:
- """Create a hash for model configuration."""
+ """Create a hash for model configuration.
+
+ Returns:
+ str: A 16-character hexadecimal hash of the model configuration
+ """
# Use Pydantic's model_dump instead of asdict for BaseModel
config_dict = model_config.model_dump()
config_str = json.dumps(config_dict, sort_keys=True, default=str)
@@ -170,8 +172,7 @@ def _dump_sample(self, result: Union[dict, ModelResponse], sample_type: SampleTy
return asdict(result)
def get_notcached_samples(self, docs: List[Doc], sample_type: SampleType) -> Tuple[List[Doc], Set]:
- """
- Identify which docs need processing based on cached indices.
+ """Identify which docs need processing based on cached indices.
Returns:
Tuple of (docs_not_cached, tasks_with_cached_samples) where
@@ -195,8 +196,7 @@ def get_notcached_samples(self, docs: List[Doc], sample_type: SampleType) -> Tup
def get_samples_from_cache(
self, docs: List[Doc], task_names: list | set, sample_type: SampleType
) -> List[dict | ModelResponse]:
- """
- Get cached samples for the given docs.
+ """Get cached samples for the given docs.
Warning: Assumes all docs and task_names provided are stored in cache, will fail otherwise.
Returns:
@@ -277,8 +277,7 @@ def store_samples(
def cached(cache_type_name: str): # noqa C901
- """
- Decorator to cache method results based on Doc inputs.
+ """Decorator to cache method results based on Doc inputs.
Args:
cache_type_name: Type of cache ("tokenization" or "predictions")
@@ -291,6 +290,9 @@ def tok_encode_pair(self, docs: List[Doc], ...):
@cached("predictions")
def greedy_until(self, docs: List[Doc], ...):
# method implementation
+
+ Returns:
+ Callable: A decorator function that wraps the original function with caching functionality
"""
def decorator(func: Callable): # noqa C901
diff --git a/src/lighteval/utils/parallelism.py b/src/lighteval/utils/parallelism.py
index e9bee5f5a..2e73f4c73 100644
--- a/src/lighteval/utils/parallelism.py
+++ b/src/lighteval/utils/parallelism.py
@@ -39,12 +39,14 @@
def should_reduce_batch_size(exception: Exception) -> bool:
- """
- Checks if `exception` relates to CUDA out-of-memory, CUDNN not supported, or CPU out-of-memory
+ """Checks if `exception` relates to CUDA out-of-memory, CUDNN not supported, or CPU out-of-memory
Args:
exception (`Exception`):
An exception
+
+ Returns:
+ bool: True if the exception is related to memory issues that can be resolved by reducing batch size
"""
_statements = [
"CUDA out of memory.", # CUDA OOM
@@ -58,8 +60,7 @@ def should_reduce_batch_size(exception: Exception) -> bool:
def find_executable_batch_size(function: callable = None, starting_batch_size: int = 128):
- """
- A basic decorator that will try to execute `function`. If it fails from exceptions related to out-of-memory or
+ """A basic decorator that will try to execute `function`. If it fails from exceptions related to out-of-memory or
CUDNN, the batch size is cut in half and passed to `function`
`function` must take in a `batch_size` parameter as its first argument.
@@ -71,7 +72,6 @@ def find_executable_batch_size(function: callable = None, starting_batch_size: i
The batch size to try and fit into memory
Example:
-
```python
>>> from lighteval.utils_parallelism import find_executable_batch_size
@@ -83,6 +83,9 @@ def find_executable_batch_size(function: callable = None, starting_batch_size: i
>>> train(model, optimizer)
```
+
+ Returns:
+ Callable: A decorator function that automatically adjusts batch size to fit in memory
"""
if function is None:
return functools.partial(find_executable_batch_size, starting_batch_size=starting_batch_size)
@@ -118,8 +121,7 @@ def decorator(*args, **kwargs):
def test_all_gather(accelerator=None, parallel_context=None):
- """
- Test the gather operation in a parallel setup.
+ """Test the gather operation in a parallel setup.
Args:
accelerator (Optional): The accelerator object used for parallelism.
diff --git a/src/lighteval/utils/timeout.py b/src/lighteval/utils/timeout.py
index 91f247f9a..0b9e248c2 100644
--- a/src/lighteval/utils/timeout.py
+++ b/src/lighteval/utils/timeout.py
@@ -33,6 +33,9 @@ def timeout(timeout_seconds: int = 10): # noqa: C901
Notes:
On Unix systems, uses a signal-based alarm approach which is more efficient as it doesn't require spawning a new process.
On Windows systems, uses a multiprocessing-based approach since signal.alarm is not available. This will incur a huge performance penalty.
+
+ Returns:
+ Callable: A decorator function that wraps the original function with timeout functionality
"""
if os.name == "posix":
# Unix-like approach: signal.alarm
diff --git a/src/lighteval/utils/utils.py b/src/lighteval/utils/utils.py
index 115987ba7..3ab5976d8 100644
--- a/src/lighteval/utils/utils.py
+++ b/src/lighteval/utils/utils.py
@@ -18,8 +18,24 @@
from pytablewriter import MarkdownTableWriter
-def flatten_dict(nested: dict, sep="/") -> dict:
- """Flatten dictionary, list, tuple and concatenate nested keys with separator."""
+def flatten_dict(nested: dict, sep: str = "/") -> dict:
+ """Flattens a nested dictionary structure into a single-level dictionary.
+
+ Recursively traverses nested dictionaries, lists, and tuples, concatenating keys with a separator
+ to create unique keys for each value. Handles numpy arrays and sanitizes markdown-incompatible characters.
+
+ Args:
+ nested (dict): The nested dictionary structure to flatten.
+ sep (str, optional): The separator to use between nested keys. Defaults to "/".
+
+ Returns:
+ dict: A flattened dictionary where all values are at the root level and keys indicate
+ the original nesting structure using the separator.
+
+ Examples:
+ >>> flatten_dict({'a': {'b': 1, 'c': [{'d': 2}, {'e': 3}]}})
+ {'a/b': 1, 'a/c/0/d': 2, 'a/c/1/e': 3}
+ """
def clean_markdown(v: str) -> str:
return v.replace("|", "_").replace("\n", "_") if isinstance(v, str) else v # Need this for markdown
@@ -51,13 +67,21 @@ def rec(nest: dict, prefix: str, into: dict):
def clean_s3_links(value: str) -> str:
- """Cleans and formats s3 bucket links for better display in the result table (nanotron models)
+ """Converts S3 bucket links to clickable HTML links for better display in result tables.
+
+ Takes an S3 URI and converts it to a clickable HTML link that opens the AWS S3 console
+ at the specific bucket and prefix location. This is particularly useful for nanotron models
+ and other S3-hosted model artifacts.
Args:
- value (str): path to clean
+ value (str): The S3 URI to convert (e.g., "s3://my-bucket/path/to/model").
Returns:
- str : cleaned path
+ str: HTML anchor tag with the original S3 URI as display text and a link to the AWS S3 console.
+
+ Examples:
+ >>> clean_s3_links("s3://my-bucket/models/gpt2")
+ ' s3://my-bucket/models/gpt2 '
"""
s3_bucket, s3_prefix = str(value).replace("s3://", "").split("/", maxsplit=1)
if not s3_prefix.endswith("/"):
@@ -68,7 +92,28 @@ def clean_s3_links(value: str) -> str:
def obj_to_markdown(obj, convert_s3_links: bool = True) -> str:
- """Convert a (potentially nested) dataclass object or a dict in a readable markdown string for logging"""
+ """Converts a dataclass object or dictionary to a readable markdown table for logging.
+
+ Flattens the object structure and creates a markdown table with key-value pairs.
+ Optionally converts S3 links to clickable HTML links for better display.
+
+ Args:
+ obj: A dataclass object or dictionary to convert to markdown.
+ convert_s3_links (bool, optional): Whether to convert S3 URIs to clickable links.
+ Defaults to True.
+
+ Returns:
+ str: A markdown-formatted table string with the object's key-value pairs.
+
+ Examples:
+ >>> from dataclasses import dataclass
+ >>> @dataclass
+ ... class Config:
+ ... model_name: str = "gpt2"
+ ... batch_size: int = 32
+ >>> obj_to_markdown(Config())
+ '| Key | Value |\n|-----|-------|\n| model_name | gpt2 |\n| batch_size | 32 |'
+ """
from pytablewriter import MarkdownTableWriter
if is_dataclass(obj):
@@ -89,14 +134,22 @@ def obj_to_markdown(obj, convert_s3_links: bool = True) -> str:
def sanitize_numpy(example_dict: dict) -> dict:
- """
- Sanitizes a dictionary by converting any numpy generic types to their corresponding Python types.
+ """Sanitizes a dictionary by converting numpy generic types to their corresponding Python types.
+
+ Numpy generic types (like np.int64, np.float32, etc.) can cause issues with JSON serialization
+ and other operations. This function converts them to standard Python types.
Args:
example_dict (dict): The dictionary to be sanitized.
Returns:
dict: The sanitized dictionary with numpy generic types converted to Python types.
+
+ Examples:
+ >>> import numpy as np
+ >>> data = {'score': np.float32(0.95), 'count': np.int64(42)}
+ >>> sanitize_numpy(data)
+ {'score': 0.95, 'count': 42}
"""
output_dict = {}
for k, v in example_dict.items():
@@ -115,19 +168,26 @@ def sanitize_numpy(example_dict: dict) -> dict:
def as_list(item: ListLike[ElementType] | ElementType) -> list[ElementType]:
- """
- Convert the given item into a list.
+ """Converts the given item into a list.
- If the item is already a list, it is returned as is.
- If the item is a tuple, it is converted into a list.
- Otherwise, the item is wrapped in a list.
+ If the item is already a list, it is returned as is. If the item is a tuple,
+ it is converted into a list. Otherwise, the item is wrapped in a list.
Args:
- item (Union[list, tuple, Any]): The item to be converted.
+ item: The item to be converted. Can be a list, tuple, or any other type.
Returns:
list: The converted list.
+ Examples:
+ >>> as_list([1, 2, 3])
+ [1, 2, 3]
+ >>> as_list((1, 2, 3))
+ [1, 2, 3]
+ >>> as_list("single_item")
+ ['single_item']
+ >>> as_list(42)
+ [42]
"""
if isinstance(item, list):
return item
@@ -139,14 +199,22 @@ def as_list(item: ListLike[ElementType] | ElementType) -> list[ElementType]:
def flatten(item: list[Union[list, str]]) -> list[str]:
- """
- Flattens a nested list of strings into a single flat list.
+ """Flattens a nested list of strings into a single flat list.
+
+ Recursively flattens nested lists, extracting all string elements into a single-level list.
+ Non-list elements are treated as strings and added directly.
Args:
- item (list[Union[list, str]]): The nested list to be flattened.
+ item: The nested list to be flattened. Can contain strings and nested lists.
Returns:
list[str]: The flattened list of strings.
+
+ Examples:
+ >>> flatten([["a", "b"], "c", ["d", ["e", "f"]]])
+ ['a', 'b', 'c', 'd', 'e', 'f']
+ >>> flatten(["simple", "list"])
+ ['simple', 'list']
"""
flat_item = []
for sub_item in item:
@@ -155,7 +223,31 @@ def flatten(item: list[Union[list, str]]) -> list[str]:
def make_results_table(result_dict):
- """Generate table of results."""
+ """Generates a markdown table from evaluation results.
+
+ Creates a formatted markdown table displaying task results with metrics, values,
+ and standard errors. The table includes columns for task name, version, metric,
+ value, and standard error.
+
+ Args:
+ result_dict (dict): Dictionary containing evaluation results with the structure:
+ - 'results': Dict mapping task names to metric dictionaries
+ - 'versions': Dict mapping task names to version strings
+
+ Returns:
+ str: A markdown-formatted table string displaying the results.
+
+ Examples:
+ >>> results = {
+ ... 'results': {
+ ... 'squad': {'accuracy': 0.85, 'accuracy_stderr': 0.02},
+ ... 'glue': {'f1': 0.92, 'f1_stderr': 0.01}
+ ... },
+ ... 'versions': {'squad': 'v2.0', 'glue': 'v1.0'}
+ ... }
+ >>> table = make_results_table(results)
+ # Returns markdown table with task, version, metric, value, ±, stderr columns
+ """
md_writer = MarkdownTableWriter()
md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]
@@ -180,38 +272,36 @@ def make_results_table(result_dict):
return md_writer.dumps()
-def boolstring_to_bool(x: Union[str, bool, int]) -> Union[bool, None]:
- """Allows to manage string or bool to bool conversion, in case a configuration input is badly formatted.
-
- Args:
- x (str): A string (true, false, True, False, ...)
-
- Returns:
- Union[bool, None]: the corresponding boolean
- """
- if x in [None, "None", "null", ""]:
- return None
- if x in ["True", "true", True, 1]:
- return True
- if x in ["False", "false", False, 0]:
- return False
- raise ValueError(f"You tried to convert {x} to a boolean but it's not possible.")
-
-
def safe_divide(numerator: np.ndarray, denominator: float, default_value: float = 0.0) -> np.ndarray:
return np.where(denominator != 0, numerator / denominator, default_value)
def remove_reasoning_tags(text: str, tag_pairs: list[tuple[str, str]]) -> str:
- """Remove all instances of reasoning tag pairs from text.
+ """Removes all instances of reasoning tag pairs from text.
+
+ Iteratively removes content between specified start and end tag pairs.
+ This is useful for cleaning model outputs that contain reasoning sections
+ that should be excluded from evaluation.
See: https://github.com/huggingface/lighteval/issues/790
- Example:
- >>> text = " Reasoning section Answer section"
- >>> tag_pairs = [("", "")]
- >>> remove_reasoning_tags(text, tag_pairs)
- ' Answer section'
+ Args:
+ text (str): The input text containing reasoning tags to remove.
+ tag_pairs (list[tuple[str, str]]): List of (start_tag, end_tag) pairs to remove.
+
+ Returns:
+ str: The text with all reasoning tag content removed.
+
+ Examples:
+ >>> text = " Reasoning section Answer section"
+ >>> tag_pairs = [("", "")]
+ >>> remove_reasoning_tags(text, tag_pairs)
+ ' Answer section'
+
+ >>> text = "Step 1AnswerStep 2"
+ >>> tag_pairs = [("", "")]
+ >>> remove_reasoning_tags(text, tag_pairs)
+ 'Answer'
"""
result = text