Repository navigation
Benchmark Contract #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Contract | |
| on: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: benchmark-contract-${{ github.sha }} | |
| cancel-in-progress: false | |
| jobs: | |
| benchmark: | |
| name: Live MCP benchmark (200/500/200) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| env: | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| # The benchmark server runs under an isolated HOME, so patchright cannot | |
| # see the default ~/.cache/ms-playwright browsers. Pin the browser path | |
| # via env: it flows through wet_server_env's `**os.environ` into the | |
| # spawned server regardless of the isolated home. | |
| PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.ms-playwright | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout exact dispatch SHA | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 | |
| - name: Setup Python 3.13 | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.13" | |
| - name: Install dependencies | |
| run: uv sync --group dev | |
| - name: Install headless browser binaries | |
| # The extract suite's `headless` fallback (patchright/chromium) is | |
| # exercised by anti-bot fixture URLs; browser binaries are not part | |
| # of `uv sync` and must be fetched once per job. | |
| run: uv run python -m patchright install chromium | |
| - name: Require a real provider credential | |
| shell: bash | |
| run: | | |
| if [ -z "${OPENROUTER_API_KEY}" ]; then | |
| echo "::error::No real wet provider credential is projected into this run. Sync it from skret; the benchmark never mocks credentials." | |
| exit 1 | |
| fi | |
| - name: Resolve benchmark model | |
| shell: bash | |
| env: | |
| VAR_MODEL: ${{ vars.OPENROUTER_MODEL }} | |
| run: | | |
| # Precedence: explicit repo variable first, then the model-sync pin | |
| # file. Both unset stays fail-closed — the benchmark never picks a | |
| # paid model silently. | |
| model="${VAR_MODEL}" | |
| src="vars.OPENROUTER_MODEL" | |
| if [ -z "${model}" ] && [ -f bench/model.txt ]; then | |
| pin="$(tr -d ' \t\r\n' < bench/model.txt)" | |
| if [ -n "${pin}" ]; then | |
| model="${pin}" | |
| src="bench/model.txt" | |
| fi | |
| fi | |
| if [ -z "${model}" ]; then | |
| echo "::error::No benchmark model resolved: vars.OPENROUTER_MODEL is unset and bench/model.txt is empty. The benchmark never picks a paid model silently — set the repo variable or merge a [model-sync] pin PR before dispatching." | |
| exit 1 | |
| fi | |
| echo "OPENROUTER_MODEL=${model}" >> "${GITHUB_ENV}" | |
| echo "Resolved benchmark model ${model} from ${src}" | |
| - name: Pre-flight provider spend gate | |
| shell: bash | |
| env: | |
| OPENROUTER_SPEND_CAP_USD: ${{ vars.OPENROUTER_SPEND_CAP_USD }} | |
| run: | | |
| cap="${OPENROUTER_SPEND_CAP_USD:-5}" | |
| if [ -z "${OPENROUTER_SPEND_CAP_USD}" ]; then | |
| echo "NOTE: no vars.OPENROUTER_SPEND_CAP_USD set — using DEFAULT cap ${cap} USD. Set the variable to make the bound an explicit user decision." | |
| fi | |
| resp="$(curl -sf -H "Authorization: Bearer ${OPENROUTER_API_KEY}" https://openrouter.ai/api/v1/credits)" || { | |
| echo "::error::Cannot read OpenRouter credits endpoint — failing closed before spending." | |
| exit 1 | |
| } | |
| avail="$(printf '%s' "$resp" | python3 -c 'import json,sys;d=json.load(sys.stdin)["data"];print(round(d["total_credits"]-d["total_usage"],2))')" | |
| echo "Pre-flight: available credit ${avail} USD; run cap ${cap} USD; model ${OPENROUTER_MODEL}" | |
| awk -v a="$avail" -v c="$cap" 'BEGIN{exit (a>=c)?0:1}' || { | |
| echo "::error::Available OpenRouter credit (${avail} USD) is below this run's cap (${cap} USD). Top up or raise vars.OPENROUTER_SPEND_CAP_USD consciously." | |
| exit 1 | |
| } | |
| - name: Validate immutable fixtures | |
| run: uv run python tests/search_quality/run_contract.py validate-fixtures | |
| - name: Execute live MCP benchmark | |
| env: | |
| BENCHMARK_ARTIFACT: ${{ runner.temp }}/wet-benchmark-contract.json | |
| run: uv run python tests/search_quality/run_contract.py run --output "${BENCHMARK_ARTIFACT}" | |
| - name: Upload exact-SHA benchmark artifact | |
| # Runs before the gate (and unconditionally) so a failed gate still | |
| # leaves the measured artifact downloadable — diagnosing the next | |
| # failure must not require scraping job logs. | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: wet-benchmark-contract-${{ github.sha }} | |
| path: ${{ runner.temp }}/wet-benchmark-contract.json | |
| # ignore, not error: with `if: always()` this step also runs on a | |
| # pre-flight abort (no model / no credit) where no artifact exists — | |
| # a second failure line there would mask the real error. | |
| if-no-files-found: ignore | |
| retention-days: 90 | |
| - name: Gate measured artifact against exact SHA | |
| env: | |
| BENCHMARK_ARTIFACT: ${{ runner.temp }}/wet-benchmark-contract.json | |
| GIT_SHA: ${{ github.sha }} | |
| run: uv run python tests/search_quality/run_contract.py gate --artifact "${BENCHMARK_ARTIFACT}" --expected-commit "${GIT_SHA}" |