Add CI benchmark suite: latency + recall regression tracking #9
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| permissions: | |
| contents: read | |
| # All benchmark runs share one Moss project and the deterministic | |
| # benchmark-ci-<hash> index, so concurrent jobs could race on first index | |
| # creation and contaminate each other's latency numbers with cross-job | |
| # load. Serialize globally; don't cancel a run that is already measuring. | |
| concurrency: | |
| group: moss-benchmark | |
| cancel-in-progress: false | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| update_baseline: | |
| description: 'Update baseline.json with current results' | |
| type: boolean | |
| default: false | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| # PyPI downloads occasionally break mid-stream on hosted runners | |
| # (IncompleteRead / ProtocolError), and pip does not resume a | |
| # partial wheel download — retry the whole install a few times. | |
| for attempt in 1 2 3; do | |
| if pip install -r benchmarks/ci/requirements.txt; then | |
| exit 0 | |
| fi | |
| echo "pip install failed (attempt ${attempt}/3) — retrying in 15s" | |
| sleep 15 | |
| done | |
| echo "pip install failed after 3 attempts" | |
| exit 1 | |
| - name: Run benchmark suite | |
| env: | |
| MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }} | |
| MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }} | |
| # Fork PRs cannot read repository secrets, so missing credentials | |
| # are expected there and the suite may skip. On trusted runs | |
| # (push to main, same-repo PRs, manual dispatch) missing secrets | |
| # make the suite FAIL instead of passing as a green no-op. | |
| ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && '1' || '0' }} | |
| run: | | |
| # Baseline-update runs skip the regression comparison: comparing | |
| # against the baseline being replaced would fail the run (and skip | |
| # the copy step) exactly when an intentional change moved the numbers. | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ] && \ | |
| [ "${{ github.event.inputs.update_baseline }}" = "true" ]; then | |
| pytest benchmarks/ci/ -v \ | |
| --benchmark-output=benchmark_results.json | |
| else | |
| pytest benchmarks/ci/ -v \ | |
| --benchmark-output=benchmark_results.json \ | |
| --baseline-file=benchmarks/ci/baseline.json \ | |
| --latency-threshold=0.20 \ | |
| --recall-threshold=0.05 | |
| fi | |
| - name: Upload results artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ github.sha }} | |
| path: benchmark_results.json | |
| retention-days: 90 | |
| if: always() | |
| - name: Update baseline (manual trigger only) | |
| if: >- | |
| github.event_name == 'workflow_dispatch' && | |
| github.event.inputs.update_baseline == 'true' | |
| run: | | |
| echo "Copying benchmark_results.json → benchmarks/ci/baseline.json" | |
| cp benchmark_results.json benchmarks/ci/baseline.json | |
| echo "New baseline (runner copy only — NOT committed):" | |
| cat benchmarks/ci/baseline.json | |
| echo "" | |
| echo "To persist: download the benchmark-results-${{ github.sha }} artifact," | |
| echo "copy it to benchmarks/ci/baseline.json, and commit." |