Merge branch 'main' into mini-fixes

Author: clefourrier

.github/release.yml

@@ -5,7 +5,10 @@ changelog:
   categories:
     - title: New Features 🎉
       labels:
-        - feature/enhancement
+        - feature
+    - title: Enhancement ⚙️
+      labels:
+        - enhancement
     - title: Documentation 📚
       labels:
         - documentation

community_tasks/slr_bench_evals.py

@@ -0,0 +1,125 @@
+# MIT License
+
+# Copyright (c) 2025 Lukas Helff
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+SLR-Bench is a large-scale benchmark for scalable logical reasoning with language models, comprising 19,000 prompts organized into 20 curriculum levels.
+The tasks progressively increase in relational, arithmetic, and recursive complexity, requiring models to synthesize Prolog rules that classify train compositions.
+For more details see: https://huggingface.co/datasets/AIML-TUDA/SLR-Bench
+The paper can be found here: https://arxiv.org/abs/2506.15787
+Before using this task, please ensure that SWI-Prolog and evaluate are installed on your system, as they are required for symbolic verification of the generated Prolog programs.
+"""
+
+import logging
+import shutil
+
+import numpy as np
+from evaluate import load
+
+from lighteval.metrics.utils.metric_utils import SampleLevelComputation, SampleLevelMetric
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+from lighteval.tasks.requests import Doc, SamplingMethod
+
+
+logger = logging.getLogger(__name__)
+
+
+# Check for SWI-Prolog installation
+if shutil.which("swipl") is None:
+    raise ImportError(
+        "SWI-Prolog (swipl) is not installed or not in PATH. "
+        "Please install SWI-Prolog to use this task. "
+        "You can install required dependencies with: pip install -r community_tasks/slr_bench_requirements.txt"
+    )
+
+# Load the symbolic judge for evaluating Prolog programs
+symbolic_judge = load("AIML-TUDA/VerifiableRewardsForScalableLogicalReasoning")
+
+
+def prompt_fn(line: dict, task_name: str):
+    """Defines how to go from a dataset line to a doc object."""
+    return Doc(
+        task_name=task_name, query=line["prompt"], choices=[str(line.get("validation program", ""))], gold_index=0
+    )
+
+
+class VerifiableRewardMetric(SampleLevelComputation):
+    def compute(self, doc, model_response, **kwargs):
+        try:
+            prediction = model_response.final_text[0]
+            validation_program = doc.choices[0] if doc.choices else ""
+            ref_format = [
+                {
+                    "validation_program": validation_program,
+                    "evaluation_config": {"positive_predicate": "eastbound", "negative_predicate": "westbound"},
+                }
+            ]
+
+            results = symbolic_judge.compute(predictions=[prediction], references=ref_format)
+            return results["accuracy"]
+
+        except Exception as e:
+            logger.error("Error during the computation of the metric")
+            raise RuntimeError(f"Failed to compute verifiable reward metric: {e}")
+
+
+custom_metric = SampleLevelMetric(
+    metric_name="verifiable_reward",
+    higher_is_better=True,
+    category=SamplingMethod.GENERATIVE,
+    sample_level_fn=VerifiableRewardMetric(),
+    corpus_level_fn=np.mean,
+)
+
+# Define the subsets available in the SLR-Bench dataset
+CONFIGURATIONS = ["All", "Basic", "Easy", "Medium", "Hard"]
+
+
+class SLRBenchTask(LightevalTaskConfig):
+    """Task configuration for SLR-Bench evaluation."""
+
+    def __init__(
+        self,
+        config: str,
+    ):
+        name = f"slr_bench_{config.lower()}"
+        super().__init__(
+            name=name,
+            hf_subset=f"v1-{config}",
+            prompt_function=prompt_fn,
+            hf_repo="AIML-TUDA/SLR-Bench",
+            metrics=[custom_metric],
+            hf_avail_splits=["train", "validation", "test"],
+            evaluation_splits=["test"],
+            few_shots_split="train",
+            few_shots_select="random_sampling_from_train",
+            suite=["community"],
+            generation_size=4096,
+            stop_sequence=None,
+            version=1,
+        )
+
+
+# Create a single task instance for each configuration
+TASKS = [SLRBenchTask(config) for config in CONFIGURATIONS]
+
+# Export tasks table
+TASKS_TABLE = TASKS

community_tasks/slr_bench_requirements.txt

@@ -0,0 +1,2 @@
+evaluate
+swipl

docs/source/use-inference-providers-as-backend.mdx

@@ -66,4 +66,4 @@ model_parameters:
   org_to_bill: "my-organization"
 ```
 
-For more detailed error handling and provider-specific information, refer to the [Hugging Face Inference Providers documentation](https://huggingface.co/docs/inference-endpoints/guides/inference_providers).
+For more detailed error handling and provider-specific information, refer to the [Hugging Face Inference Providers documentation](https://huggingface.co/docs/inference-providers/en/index).

pyproject.toml

@@ -88,7 +88,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-litellm = ["litellm[caching]", "diskcache"]
+litellm = ["litellm[caching]>=1.66.0", "diskcache"]
 tgi = ["text-generation>=0.7.0"]
 optimum = ["optimum==1.12.0"]
 quantization = ["bitsandbytes>=0.41.0", "auto-gptq>=0.4.2"]

src/lighteval/metrics/metrics.py

@@ -526,8 +526,12 @@ class Metrics(Enum):
         metric_name="extractive_match",
         sample_level_fn=MultilingualExtractiveMatchMetric(
             language=Language.ENGLISH,
-            gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")],
-            pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")],
+            gold_extraction_target=[
+                IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
+            ],
+            pred_extraction_target=[
+                IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
+            ],
             precision=6,
         ),
         category=SamplingMethod.GENERATIVE,
@@ -539,8 +543,12 @@ class Metrics(Enum):
         sample_level_fn=PassAtK(
             sample_scoring_function=MultilingualExtractiveMatchMetric(
                 language=Language.ENGLISH,
-                gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")],
-                pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")],
+                gold_extraction_target=[
+                    IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
+                ],
+                pred_extraction_target=[
+                    IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
+                ],
                 precision=6,
             ),
         ),

src/lighteval/models/endpoints/inference_providers_model.py

@@ -116,7 +116,7 @@ def __init__(self, config: InferenceProvidersModelConfig) -> None:
         self.API_RETRY_SLEEP = 3
         self.API_RETRY_MULTIPLIER = 2
         self.pairwise_tokenization = False
-        self.semaphore = asyncio.Semaphore(config.parallel_calls_count)  # Limit concurrent API calls
+        self.parallel_calls_count = config.parallel_calls_count
 
         self.client = AsyncInferenceClient(
             provider=self.provider,
@@ -179,13 +179,16 @@ async def __call_api_parallel(
     ):
         results = []
 
+        # Initialize semaphore for the current event loop
+        semaphore = asyncio.Semaphore(self.parallel_calls_count)
+
         num_sampless = [num_samples for _ in prompts] if not isinstance(num_samples, list) else num_samples
         assert len(prompts) == len(num_sampless), (
             f"Length of prompts and max_new_tokenss should be the same but are {len(prompts)}, {len(num_sampless)}"
         )
 
         async def bounded_api_call(prompt, num_samples):
-            async with self.semaphore:
+            async with semaphore:
                 return await self.__call_api(prompt, num_samples)
 
         tasks = [bounded_api_call(prompt, num_samples) for prompt, num_samples in zip(prompts, num_sampless)]