feat: cloud bench harness with Bencher.dev gating #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yamllint disable rule:line-length | |
| name: bench | |
| # yamllint disable rule:truthy | |
| on: | |
| pull_request: | |
| branches: [main, devel] | |
| paths-ignore: | |
| - '*.md' | |
| - '.github/workflows/docs.yml' | |
| push: | |
| branches: [main, devel] | |
| paths-ignore: | |
| - '*.md' | |
| - '.github/workflows/docs.yml' | |
| workflow_dispatch: | |
| # yamllint enable rule:truthy | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| benchmark: | |
| name: Throughput bench (ubuntu-latest) | |
| runs-on: ubuntu-latest | |
| # PRs from forks cannot read BENCHER_API_TOKEN. Skip rather than fail | |
| # noisily; maintainer pushes / merges still record the baseline. | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| permissions: | |
| pull-requests: write | |
| checks: write | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Restore cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${HOME}/.cache | |
| key: cache-bench-${{ runner.os }} | |
| - name: Checkout project | |
| uses: actions/checkout@v4 | |
| - name: Setup Nim | |
| uses: jiro4989/setup-nim-action@v2 | |
| with: | |
| nim-version: 'stable' | |
| - name: Install build deps (Linux) | |
| run: | | |
| sudo apt-get update -q -y | |
| sudo apt-get -qq install -y clang | |
| - name: Clone and install sibling Nim deps (nim-debra, nim-typestates) | |
| # Mirrors build.yml: nim.cfg uses --path:"../nim-debra/src" and | |
| # --path:"../nim-typestates/src", and nimble's resolver runs before | |
| # the compiler so we must install matching branches locally. | |
| run: | | |
| set -e | |
| cd .. | |
| git clone --depth 1 --branch main https://github.com/elijahr/nim-debra.git | |
| git clone --depth 1 --branch main https://github.com/elijahr/nim-typestates.git | |
| (cd nim-typestates && nimble install -y) | |
| (cd nim-debra && nimble install -y) | |
| - name: Vendor unittest2 | |
| run: git clone --depth 1 https://github.com/status-im/nim-unittest2.git deps/unittest2 | |
| - name: Run adapter unit tests | |
| # Cheap pre-flight: fail fast if the parser itself is broken before | |
| # spending minutes compiling and running the bench binary. | |
| run: python3 benchmarks/test_bmf_adapter.py -v | |
| - name: Compile bench_throughput (CI-tuned run shape) | |
| # Cloud runs use a tighter wall-clock budget than the local default | |
| # (1M messages × 33 runs). 100k × 5 keeps the harness honest while | |
| # finishing in single-digit minutes; warmup stays at 2 to absorb | |
| # JIT/cache effects. UnboundedMupsicRuns mirrors DefaultRuns. | |
| run: | | |
| nim c -d:release -d:danger --threads:on \ | |
| -d:MessageCount=100000 \ | |
| -d:DefaultRuns=5 \ | |
| -d:WarmupRuns=2 \ | |
| -d:UnboundedMupsicRuns=5 \ | |
| benchmarks/nim/bench_throughput.nim | |
| - name: Run bench_throughput | |
| run: ./.tmp/bench_throughput | tee bench_output.txt | |
| - name: Convert to BMF JSON | |
| run: python3 benchmarks/bmf_adapter.py bench_output.txt bench_results.json | |
| - name: Show BMF JSON (debug) | |
| run: cat bench_results.json | |
| - name: Install Bencher CLI | |
| uses: bencherdev/bencher@main | |
| # PR runs: compare against base branch and post a comment. | |
| - name: Track PR benchmarks with Bencher | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| bencher run \ | |
| --project lockfreequeues \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch "$GITHUB_HEAD_REF" \ | |
| --start-point "$GITHUB_BASE_REF" \ | |
| --start-point-hash '${{ github.event.pull_request.base.sha }}' \ | |
| --start-point-clone-thresholds \ | |
| --start-point-reset \ | |
| --testbed ubuntu-latest \ | |
| --adapter json \ | |
| --file bench_results.json \ | |
| --github-actions '${{ secrets.GITHUB_TOKEN }}' \ | |
| --err | |
| # Push to main/devel: record the baseline for that branch. | |
| - name: Track base branch benchmarks with Bencher | |
| if: github.event_name == 'push' | |
| run: | | |
| bencher run \ | |
| --project lockfreequeues \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch "${GITHUB_REF##*/}" \ | |
| --testbed ubuntu-latest \ | |
| --threshold-measure throughput \ | |
| --threshold-test t_test \ | |
| --threshold-max-sample-size 64 \ | |
| --threshold-lower-boundary 0.99 \ | |
| --thresholds-reset \ | |
| --adapter json \ | |
| --file bench_results.json \ | |
| --github-actions '${{ secrets.GITHUB_TOKEN }}' \ | |
| --err | |
| # workflow_dispatch: just upload, no comparison. | |
| - name: Track manual benchmarks with Bencher | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| bencher run \ | |
| --project lockfreequeues \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch "${GITHUB_REF##*/}" \ | |
| --testbed ubuntu-latest \ | |
| --adapter json \ | |
| --file bench_results.json |