Feature/add leaderboard

.github/workflows/deploy-site.yml

@@ -0,0 +1,111 @@
+name: Deploy Site
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "leaderboard/site/**"
+      - ".github/workflows/deploy-site.yml"
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Docs version to deploy (e.g., 1.0.0 or dev)"
+        required: false
+        default: "dev"
+      manifest_url_override:
+        description: "Optional leaderboard manifest URL override"
+        required: false
+        default: ""
+
+permissions:
+  contents: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: Deploy docs and leaderboard
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.github/actions/setup
+        with:
+          sync-args: --group dev
+          configure-git-identity: "true"
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: leaderboard/site/package-lock.json
+
+      - name: Install leaderboard site dependencies
+        run: npm ci
+        working-directory: leaderboard/site
+
+      - name: Fetch gh-pages branch
+        run: |
+          git fetch origin gh-pages --depth=1 || echo "gh-pages branch does not exist yet"
+
+      # Step 1: mike deploy (no --push) updates local gh-pages ref
+      - name: Deploy docs to local gh-pages
+        run: uv run mike deploy ${{ inputs.version || 'dev' }}
+
+      # Step 2: Build leaderboard Astro app
+      - name: Resolve manifest URL
+        id: cfg
+        env:
+          MANIFEST_URL_OVERRIDE: ${{ inputs.manifest_url_override || '' }}
+          CONFIGURED_MANIFEST_URL: ${{ vars.PUBLIC_LEADERBOARD_MANIFEST_URL }}
+        run: |
+          set -euo pipefail
+          MANIFEST_URL="$(uv run inv dev.leaderboard.site-resolve-manifest-url \
+            --manifest-url-override "$MANIFEST_URL_OVERRIDE" \
+            --configured-manifest-url "$CONFIGURED_MANIFEST_URL")"
+          echo "manifest_url=$MANIFEST_URL" >> "$GITHUB_OUTPUT"
+
+      - name: Build leaderboard site
+        env:
+          PUBLIC_LEADERBOARD_MANIFEST_URL: ${{ steps.cfg.outputs.manifest_url }}
+        run: npm run build
+        working-directory: leaderboard/site
+
+      # Step 3: Inject leaderboard dist into gh-pages and push
+      - name: Inject leaderboard and push gh-pages
+        run: |
+          set -euo pipefail
+
+          # Checkout gh-pages into a worktree
+          git worktree add _gh-pages gh-pages
+
+          # Remove old leaderboard dir (if any) and copy fresh build
+          rm -rf _gh-pages/leaderboard
+          cp -r leaderboard/site/dist _gh-pages/leaderboard
+
+          # Commit and push
+          cd _gh-pages
+          git add -A
+          if git diff --cached --quiet; then
+            echo "No changes to deploy" >> "$GITHUB_STEP_SUMMARY"
+          else
+            git commit -m "Deploy docs and leaderboard"
+            git push origin gh-pages
+            echo "Deployed docs and leaderboard to gh-pages" >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Deploy summary
+        run: |
+          echo "docs_version=${{ inputs.version || 'dev' }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "manifest_url=${{ steps.cfg.outputs.manifest_url }}" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/leaderboard-hf-ingest.yml

@@ -0,0 +1,51 @@
+name: Leaderboard HF Ingest
+
+on:
+  schedule:
+    - cron: "*/30 * * * *"
+
+permissions:
+  contents: write
+
+jobs:
+  sync_submissions:
+    name: Sync submissions from HF schedule trigger
+    runs-on: ubuntu-latest
+    concurrency:
+      group: leaderboard-data-writer
+      cancel-in-progress: false
+    steps:
+      - name: Checkout leaderboard-submissions
+        uses: actions/checkout@v4
+        with:
+          ref: leaderboard-submissions
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.github/actions/setup
+        with:
+          sync-args: --group dev
+          configure-git-identity: "true"
+
+      - name: Sync pending submissions from HF
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          WEBARENA_VERIFIED_LEADERBOARD_SUBMISSION_HF_REPO: ${{ vars.WEBARENA_VERIFIED_LEADERBOARD_SUBMISSION_HF_REPO || vars.LEADERBOARD_HF_REPO || 'AmineHA/WebArena-Verified-Submissions-dev' }}
+        run: |
+          set -euo pipefail
+          uv run inv dev.leaderboard.hf-sync-submissions \
+            --repo-root "." \
+            --hf-repo "$WEBARENA_VERIFIED_LEADERBOARD_SUBMISSION_HF_REPO" \
+            --hf-token "$HF_TOKEN"
+
+      - name: Commit and push leaderboard updates
+        run: |
+          set -euo pipefail
+          git add -A leaderboard
+          if git diff --cached --quiet; then
+            echo "No submission sync changes" >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          git commit -m "Sync HF submissions and rebuild leaderboard"
+          git push origin HEAD:leaderboard-submissions

.github/workflows/leaderboard-site-build.yml

@@ -0,0 +1,67 @@
+name: Leaderboard Site Build
+
+on:
+  pull_request:
+    paths:
+      - "leaderboard/site/**"
+      - ".github/workflows/leaderboard-site-build.yml"
+  workflow_dispatch:
+    inputs:
+      manifest_url_override:
+        description: "Optional manifest URL override for this run"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+concurrency:
+  group: leaderboard-site-build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build leaderboard site with production manifest source
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: leaderboard/site/package-lock.json
+
+      - name: Install site dependencies
+        run: npm ci
+        working-directory: leaderboard/site
+
+      - name: Resolve manifest URL
+        id: cfg
+        env:
+          MANIFEST_URL_OVERRIDE: ${{ github.event.inputs.manifest_url_override }}
+          CONFIGURED_MANIFEST_URL: ${{ vars.PUBLIC_LEADERBOARD_MANIFEST_URL }}
+        run: |
+          set -euo pipefail
+          MANIFEST_URL="$(uv run inv dev.leaderboard.site-resolve-manifest-url \
+            --manifest-url-override "$MANIFEST_URL_OVERRIDE" \
+            --configured-manifest-url "$CONFIGURED_MANIFEST_URL")"
+          echo "$MANIFEST_URL"
+          echo "manifest_url=$MANIFEST_URL" >> "$GITHUB_OUTPUT"
+
+      - name: Run site tests
+        run: npm test
+        working-directory: leaderboard/site
+
+      - name: Build site
+        env:
+          PUBLIC_LEADERBOARD_MANIFEST_URL: ${{ steps.cfg.outputs.manifest_url }}
+        run: npm run build
+        working-directory: leaderboard/site
+
+      - name: Build summary
+        run: |
+          echo "manifest_url=${{ steps.cfg.outputs.manifest_url }}" >> "$GITHUB_STEP_SUMMARY"

.gitignore

@@ -225,6 +225,8 @@ __marimo__/
 output/
 scratch/
 *.wip.json
+.sisyphus/
+.wip/
 *.json.backup
 *.json.bkp
 *.tmp.json

docs/leaderboard/index.md

@@ -0,0 +1,33 @@
+# Leaderboard
+
+The WebArena-Verified leaderboard is the public results table for benchmark submissions.
+Each entry is produced from deterministic offline evaluation and linked to a submission record.
+
+- Live leaderboard: https://servicenow.github.io/webarena-verified/leaderboard/
+- Boards:
+  - `WebArena-Verified` (full benchmark, 812 tasks)
+  - `WebArena-Verified-Hard` (hard subset, 258 tasks)
+
+## What You Can Inspect
+
+Each row includes:
+
+- Rank
+- Name
+- Overall Score
+- Per-site scores: Shopping, Shopping Admin, Reddit, GitLab, Wikipedia, Map
+- Submission ID
+- Evaluator Version
+
+## UI Features
+
+- Search and filter by submission name or submission ID
+- Toggle between Full and Hard boards
+- Export leaderboard rows as CSV
+
+!!! info
+    Evaluation is deterministic and offline. WebArena-Verified does not use LLM-as-judge scoring for leaderboard entries.
+
+## Next Step
+
+To publish results, follow the submission walkthrough: [Submitting Results](submission.md).