benchmark.yml

name: Benchmark

permissions:
  contents: read

# All benchmark runs share one Moss project and the deterministic
# benchmark-ci-<hash> index, so concurrent jobs could race on first index
# creation and contaminate each other's latency numbers with cross-job
# load. Serialize globally; don't cancel a run that is already measuring.
concurrency:
  group: moss-benchmark
  cancel-in-progress: false

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:
    inputs:
      update_baseline:
        description: 'Update baseline.json with current results'
        type: boolean
        default: false

jobs:
  benchmark:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          # PyPI downloads occasionally break mid-stream on hosted runners
          # (IncompleteRead / ProtocolError), and pip does not resume a
          # partial wheel download — retry the whole install a few times.
          for attempt in 1 2 3; do
            if pip install -r benchmarks/ci/requirements.txt; then
              exit 0
            fi
            echo "pip install failed (attempt ${attempt}/3) — retrying in 15s"
            sleep 15
          done
          echo "pip install failed after 3 attempts"
          exit 1

      - name: Run benchmark suite
        env:
          MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }}
          MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }}
          # Fork PRs cannot read repository secrets, so missing credentials
          # are expected there and the suite may skip. On trusted runs
          # (push to main, same-repo PRs, manual dispatch) missing secrets
          # make the suite FAIL instead of passing as a green no-op.
          ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && '1' || '0' }}
        run: |
          # Baseline-update runs skip the regression comparison: comparing
          # against the baseline being replaced would fail the run (and skip
          # the copy step) exactly when an intentional change moved the numbers.
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && \
             [ "${{ github.event.inputs.update_baseline }}" = "true" ]; then
            pytest benchmarks/ci/ -v \
              --benchmark-output=benchmark_results.json
          else
            pytest benchmarks/ci/ -v \
              --benchmark-output=benchmark_results.json \
              --baseline-file=benchmarks/ci/baseline.json \
              --latency-threshold=0.20 \
              --recall-threshold=0.05
          fi

      - name: Upload results artifact
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results-${{ github.sha }}
          path: benchmark_results.json
          retention-days: 90
        if: always()

      - name: Update baseline (manual trigger only)
        if: >-
          github.event_name == 'workflow_dispatch' &&
          github.event.inputs.update_baseline == 'true'
        run: |
          echo "Copying benchmark_results.json → benchmarks/ci/baseline.json"
          cp benchmark_results.json benchmarks/ci/baseline.json
          echo "New baseline (runner copy only — NOT committed):"
          cat benchmarks/ci/baseline.json
          echo ""
          echo "To persist: download the benchmark-results-${{ github.sha }} artifact,"
          echo "copy it to benchmarks/ci/baseline.json, and commit."