benchmark.yml

name: Benchmark Contract

on:
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: benchmark-contract-${{ github.sha }}
  cancel-in-progress: false

jobs:
  benchmark:
    name: Live MCP benchmark (200/500/200)
    runs-on: ubuntu-latest
    timeout-minutes: 300
    env:
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
      # The benchmark server runs under an isolated HOME, so patchright cannot
      # see the default ~/.cache/ms-playwright browsers. Pin the browser path
      # via env: it flows through wet_server_env's `**os.environ` into the
      # spawned server regardless of the isolated home.
      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.ms-playwright
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
        with:
          egress-policy: audit

      - name: Checkout exact dispatch SHA
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.sha }}
          fetch-depth: 0

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0

      - name: Setup Python 3.13
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.13"

      - name: Install dependencies
        run: uv sync --group dev

      - name: Install headless browser binaries
        # The extract suite's `headless` fallback (patchright/chromium) is
        # exercised by anti-bot fixture URLs; browser binaries are not part
        # of `uv sync` and must be fetched once per job.
        run: uv run python -m patchright install chromium

      - name: Require a real provider credential
        shell: bash
        run: |
          if [ -z "${OPENROUTER_API_KEY}" ]; then
            echo "::error::No real wet provider credential is projected into this run. Sync it from skret; the benchmark never mocks credentials."
            exit 1
          fi

      - name: Resolve benchmark model
        shell: bash
        env:
          VAR_MODEL: ${{ vars.OPENROUTER_MODEL }}
        run: |
          # Precedence: explicit repo variable first, then the model-sync pin
          # file. Both unset stays fail-closed — the benchmark never picks a
          # paid model silently.
          model="${VAR_MODEL}"
          src="vars.OPENROUTER_MODEL"
          if [ -z "${model}" ] && [ -f bench/model.txt ]; then
            pin="$(tr -d ' \t\r\n' < bench/model.txt)"
            if [ -n "${pin}" ]; then
              model="${pin}"
              src="bench/model.txt"
            fi
          fi
          if [ -z "${model}" ]; then
            echo "::error::No benchmark model resolved: vars.OPENROUTER_MODEL is unset and bench/model.txt is empty. The benchmark never picks a paid model silently — set the repo variable or merge a [model-sync] pin PR before dispatching."
            exit 1
          fi
          echo "OPENROUTER_MODEL=${model}" >> "${GITHUB_ENV}"
          echo "Resolved benchmark model ${model} from ${src}"

      - name: Pre-flight provider spend gate
        shell: bash
        env:
          OPENROUTER_SPEND_CAP_USD: ${{ vars.OPENROUTER_SPEND_CAP_USD }}
        run: |
          cap="${OPENROUTER_SPEND_CAP_USD:-5}"
          if [ -z "${OPENROUTER_SPEND_CAP_USD}" ]; then
            echo "NOTE: no vars.OPENROUTER_SPEND_CAP_USD set — using DEFAULT cap ${cap} USD. Set the variable to make the bound an explicit user decision."
          fi
          resp="$(curl -sf -H "Authorization: Bearer ${OPENROUTER_API_KEY}" https://openrouter.ai/api/v1/credits)" || {
            echo "::error::Cannot read OpenRouter credits endpoint — failing closed before spending."
            exit 1
          }
          avail="$(printf '%s' "$resp" | python3 -c 'import json,sys;d=json.load(sys.stdin)["data"];print(round(d["total_credits"]-d["total_usage"],2))')"
          echo "Pre-flight: available credit ${avail} USD; run cap ${cap} USD; model ${OPENROUTER_MODEL}"
          awk -v a="$avail" -v c="$cap" 'BEGIN{exit (a>=c)?0:1}' || {
            echo "::error::Available OpenRouter credit (${avail} USD) is below this run's cap (${cap} USD). Top up or raise vars.OPENROUTER_SPEND_CAP_USD consciously."
            exit 1
          }

      - name: Validate immutable fixtures
        run: uv run python tests/search_quality/run_contract.py validate-fixtures

      - name: Execute live MCP benchmark
        env:
          BENCHMARK_ARTIFACT: ${{ runner.temp }}/wet-benchmark-contract.json
        run: uv run python tests/search_quality/run_contract.py run --output "${BENCHMARK_ARTIFACT}"

      - name: Upload exact-SHA benchmark artifact
        # Runs before the gate (and unconditionally) so a failed gate still
        # leaves the measured artifact downloadable — diagnosing the next
        # failure must not require scraping job logs.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: wet-benchmark-contract-${{ github.sha }}
          path: ${{ runner.temp }}/wet-benchmark-contract.json
          # ignore, not error: with `if: always()` this step also runs on a
          # pre-flight abort (no model / no credit) where no artifact exists —
          # a second failure line there would mask the real error.
          if-no-files-found: ignore
          retention-days: 90

      - name: Gate measured artifact against exact SHA
        env:
          BENCHMARK_ARTIFACT: ${{ runner.temp }}/wet-benchmark-contract.json
          GIT_SHA: ${{ github.sha }}
        run: uv run python tests/search_quality/run_contract.py gate --artifact "${BENCHMARK_ARTIFACT}" --expected-commit "${GIT_SHA}"