Merge upstream huggingface/lighteval main into merge_hf_main

Upstream refactor splits src/lighteval/tasks into per-task files under
src/lighteval/tasks/tasks/ and src/lighteval/tasks/multilingual/tasks/,
drops default_tasks.py / default_prompts.py / multilingual/tasks.py, and
removes the suite field from LightevalTaskConfig.

Port our edits to the new structure:
- tasks/gsm_plus.py: generation_size 16384
- tasks/gsm8k.py: generation_size 2048
- tasks/mgsm.py: hf_revision, suffix exact_match + expr_gold_metric,
  language-specific stop sequences for all 11 subsets
- tasks/piqa.py: switch to lighteval/piqa mirror
- tasks/siqa.py: pin hf_revision
- tasks/mmlu_pro.py: fix upstream's hardcoded ABCD letters so the prompt
  uses dynamic letters based on the number of options; add a parallel
  mmlu_pro_raw task exposing the handmade prompt (no inspect_ai)
- tasks/ruler.py: new home for the ruler prompt helper
- tasks/advbench.py: move here from community_tasks/
- multilingual/tasks/mathalea.py: move here from community_tasks/
- multilingual/tasks/french.py: keep jzhang86/fr_ifeval fallback and the
  generative GPQA-fr-diamond variant with prompt_gpqa_fr_instruct

Other conflict resolutions:
- pyproject.toml: take upstream unpinned transformers, vllm>=0.11.0,
  new inspect-ai and openai deps
- vllm_model.py: keep max_seq_len_to_capture fallback, Mistral eos_token
  guard, prefix-cache None-skip in logprob loop, and
  skip_reading_prefix_cache via guarded attribute assignment; adopt
  upstream's build_vllm_token_prompts helper
- llm_as_judge.py: keep max_model_len=65536, adopt upstream's
  api_key/base_url litellm pass-through
- lighteval_task.py: preserve name/data_dir fallback in load_dataset
  while picking up upstream's data_files support; keep partial args
  detail in __str__ for deterministic cache hashing
- cache_management.py: adopt name-only task_to_configs lookup; keep
  regex that strips function memory addresses for hash determinism

Author: Jeronymous

.github/release.yml

@@ -5,7 +5,10 @@ changelog:
   categories:
     - title: New Features 🎉
       labels:
-        - feature/enhancement
+        - feature
+    - title: Enhancement ⚙️
+      labels:
+        - enhancement
     - title: Documentation 📚
       labels:
         - documentation

.github/workflows/doc-build.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   build:
-    uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main
+    uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
     with:
       commit_sha: ${{ github.sha }}
       package: lighteval

.github/workflows/doc-pr-build.yml

@@ -9,7 +9,7 @@ concurrency:
 
 jobs:
   build:
-    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main
+    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
     with:
       commit_sha: ${{ github.event.pull_request.head.sha }}
       pr_number: ${{ github.event.number }}

.github/workflows/doc-pr-upload.yml

@@ -8,9 +8,10 @@ on:
 
 jobs:
   build:
-    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main
+    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@9ad2de8582b56c017cb530c1165116d40433f1c6  # main
     with:
       package_name: lighteval
     secrets:
       hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
-      comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}
+      comment_bot_app_id: ${{ secrets.COMMENT_BOT_APP_ID }}
+      comment_bot_secret_pem: ${{ secrets.COMMENT_BOT_SECRET_PEM }}

.github/workflows/pr_style_bot.yaml

@@ -0,0 +1,16 @@
+name: PR Style Bot
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  style:
+    uses: huggingface/huggingface_hub/.github/workflows/style-bot-action.yml@e000c1c89c65aee188041723456ac3a479416d4c  # main
+    with:
+      python_quality_dependencies: "[quality]"
+    secrets:
+      bot_token: ${{ secrets.HF_STYLE_BOT_ACTION }}

.github/workflows/quality.yaml

@@ -16,9 +16,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Setup Python environment
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e  # v2
         with:
           python-version: '3.10'
       - name: Install dependencies

.github/workflows/slow_tests.yaml

@@ -25,21 +25,57 @@ jobs:
           fi
 
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           lfs: true
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86  # v5
         with:
           enable-cache: true
 
       - name: Install the project
-        run: uv sync --extra dev
+        run: uv sync --extra dev-gpu
 
+      - name: Install Python development headers
+        run: sudo apt-get update && sudo apt-get install -y python3.12-dev
+
+      - name: Cache CUDA Toolkit
+        id: cache-cuda
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830  # v4
+        with:
+          path: /usr/local/cuda-12.8
+          key: cuda-toolkit-12-8-${{ runner.os }}
+
+      - name: Install CUDA Toolkit
+        if: steps.cache-cuda.outputs.cache-hit != 'true'
+        run: |
+          # Add NVIDIA package repositories
+          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+          sudo dpkg -i cuda-keyring_1.1-1_all.deb
+          sudo apt-get update
+          # Install CUDA toolkit 12.8 to match nvidia-cuda-runtime-cu12==12.8.90
+          sudo apt-get install -y cuda-toolkit-12-8
+
+      - name: Verify CUDA installation
+        run: |
+          ls -la /usr/local/cuda-12.8/bin/nvcc || echo "WARNING: nvcc not found at /usr/local/cuda-12.8/bin/nvcc"
+          if [ -f /usr/local/cuda-12.8/bin/nvcc ]; then
+            /usr/local/cuda-12.8/bin/nvcc --version
+          fi
+
+      - name: Setup CUDA environment
+        run: |
+          export CUDA_HOME=/usr/local/cuda-12.8
+          export PATH="/usr/local/cuda-12.8/bin:$PATH"
+          echo "CUDA_HOME=/usr/local/cuda-12.8" >> $GITHUB_ENV
+          echo "/usr/local/cuda-12.8/bin" >> $GITHUB_PATH
 
       - name: run nvidia-smi
         run: nvidia-smi
 
       - name: Run tests
-        run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/
+        run: |
+          export CUDA_HOME=/usr/local/cuda-12.8
+          export PATH="/usr/local/cuda-12.8/bin:$PATH"
+          uv run pytest --disable-pytest-warnings --runslow -v -s tests/slow_tests/

.github/workflows/tests.yaml

@@ -46,7 +46,7 @@ jobs:
           enable-cache: true
 
       - name: Install the project
-        run: uv sync --extra dev
+        run: uv sync --extra dev-gpu
 
       - name: Ensure cache directories exist
         run: mkdir -p cache/models cache/datasets

.github/workflows/trufflehog.yml

@@ -11,10 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v4
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       with:
         fetch-depth: 0
     - name: Secret Scanning
-      uses: trufflesecurity/trufflehog@main
+      uses: trufflesecurity/trufflehog@6bd2d14f7a4bc1e569fa3550efa7ec632a4fa67b  # main
       with:
         extra_args: --only-verified

.github/workflows/vllm_main_tests.yaml

@@ -0,0 +1,79 @@
+name: vLLM Main Branch Tests
+
+on:
+  schedule:
+    - cron: '0 2 * * 1'  # Every Monday at 2 AM UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test_vllm_main:
+    name: Test with vLLM main branch
+    runs-on: 'aws-g4dn-2xlarge-use1-public-80'
+    continue-on-error: true
+
+    steps:
+      - name: Install Git LFS
+        run: |
+          if ! command -v git-lfs &> /dev/null; then
+            sudo apt-get update && sudo apt-get install -y git-lfs
+            git lfs install
+          fi
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install the project
+        run: uv sync --extra dev-gpu
+
+      - name: Install Python development headers
+        run: sudo apt-get update && sudo apt-get install -y python3.12-dev
+
+      - name: Cache CUDA Toolkit
+        id: cache-cuda
+        uses: actions/cache@v4
+        with:
+          path: /usr/local/cuda-12.8
+          key: cuda-toolkit-12-8-${{ runner.os }}
+
+      - name: Install CUDA Toolkit
+        if: steps.cache-cuda.outputs.cache-hit != 'true'
+        run: |
+          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+          sudo dpkg -i cuda-keyring_1.1-1_all.deb
+          sudo apt-get update
+          sudo apt-get install -y cuda-toolkit-12-8
+
+      - name: Setup CUDA environment
+        run: |
+          echo "CUDA_HOME=/usr/local/cuda-12.8" >> $GITHUB_ENV
+          echo "/usr/local/cuda-12.8/bin" >> $GITHUB_PATH
+
+      - name: Verify CUDA
+        run: |
+          nvidia-smi
+          nvcc --version
+
+      - name: Install vLLM from main branch
+        run: |
+          uv pip uninstall -y vllm || true
+          uv pip install git+https://github.com/vllm-project/vllm.git@main
+
+      - name: Get vLLM version
+        id: vllm-info
+        run: |
+          VERSION=$(uv run python -c "import vllm; print(vllm.__version__)")
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Testing vLLM version: $VERSION"
+
+      - name: Run tests
+        run: uv run pytest --disable-pytest-warnings --runslow -v -s tests/slow_tests/test_vllm_model.py