Benchmarks #510
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Continuous benchmarking with Bencher (https://bencher.dev) | |
| # | |
| # PREREQUISITES: | |
| # 1. Create a project at https://bencher.dev and note its slug. | |
| # 2. Add the following in GitHub → Settings → Secrets and variables → Actions: | |
| # - Secret BENCHER_API_TOKEN: API token from bencher.dev | |
| # - Variable BENCHER_PROJECT: your project slug | |
| # 3. GITHUB_TOKEN is provided automatically — no setup needed. | |
| # | |
| # overhead_bench uses a custom harness (not Criterion). Its --bmf flag | |
| # outputs Bencher Metric Format directly. | |
| name: Benchmarks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| bencher_branch: | |
| description: 'Override Bencher branch name for iai_main (e.g. set to "main" to seed baseline from a feature branch). Leave empty to use the running branch name.' | |
| required: false | |
| default: '' | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| jobs: | |
| # Disabled until a dedicated runner exists. Wall-clock criterion + | |
| # integration benches are too noisy on shared ubuntu-latest. iai jobs | |
| # below cover the deterministic micro tier in the meantime. | |
| benchmark_main: | |
| name: Benchmark — ${{ matrix.bench.name }} (main baseline) | |
| if: ${{ false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} # skip job | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| bench: | |
| - name: writer_encode | |
| adapter: rust_criterion | |
| command: cargo bench --package dial9-tokio-telemetry --bench writer_encode | |
| - name: codec | |
| adapter: rust_criterion | |
| command: cargo bench --package dial9-trace-format --bench codec | |
| - name: overhead_bench | |
| adapter: json | |
| command: cargo bench --bench overhead_bench -- --bmf 10 | |
| - name: overhead_bench_ctimer | |
| adapter: json | |
| command: DIAL9_FORCE_CTIMER=1 cargo bench --bench overhead_bench -- --bmf 10 | |
| - name: e2e_workload | |
| adapter: json | |
| command: cargo bench --bench e2e_workload -- --bmf 10 | |
| env: | |
| RUST_BACKTRACE: 1 | |
| BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Enable perf_event_open and kallsyms | |
| run: | | |
| sudo sysctl kernel.perf_event_paranoid=1 | |
| sudo sysctl kernel.kptr_restrict=0 | |
| - uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2 | |
| - name: Run benchmark | |
| run: | | |
| bencher run \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch '${{ github.ref_name }}' \ | |
| --testbed ubuntu-latest \ | |
| --adapter '${{ matrix.bench.adapter }}' \ | |
| "${{ matrix.bench.command }}" | |
| # iai micro tier on push to main: populates the Bencher baseline used | |
| # by iai_micro PR gate below. Instruction counts are deterministic so | |
| # shared ubuntu-latest is fine. | |
| iai_main: | |
| if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' | |
| name: iai micro benches (main baseline) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| RUST_BACKTRACE: 1 | |
| IAI_CALLGRIND_VERSION: "0.16.1" | |
| BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }} | |
| # Mirror .cargo/config.toml [build] rustflags + add iai_enabled cfg | |
| # so the bench files compile their real iai entry point. Without | |
| # this cfg they compile to a no-op stub (so plain `cargo test | |
| # --all-targets` doesn't try to spawn iai-callgrind-runner). | |
| RUSTFLAGS: "--cfg tokio_unstable -C force-frame-pointers=yes --cfg iai_enabled" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install valgrind | |
| run: sudo apt-get update && sudo apt-get install -y valgrind | |
| - name: Cache iai-callgrind-runner | |
| id: runner_cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cargo/bin/iai-callgrind-runner | |
| key: iai-runner-${{ env.IAI_CALLGRIND_VERSION }} | |
| - name: Install iai-callgrind-runner | |
| if: steps.runner_cache.outputs.cache-hit != 'true' | |
| run: cargo install iai-callgrind-runner --version ${{ env.IAI_CALLGRIND_VERSION }} --locked | |
| - uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2 | |
| - name: Run iai benches → Bencher | |
| env: | |
| # workflow_dispatch can override target branch (e.g. seed Bencher's | |
| # `main` history from a feature branch when bootstrapping). | |
| BENCHER_BRANCH: ${{ github.event.inputs.bencher_branch != '' && github.event.inputs.bencher_branch || github.ref_name }} | |
| run: | | |
| set -euo pipefail | |
| for entry in \ | |
| "dial9-tokio-telemetry:writer_encode_iai" \ | |
| "dial9-tokio-telemetry:writer_write_encoded_iai" \ | |
| "dial9-tokio-telemetry:threadlocal_encode_iai" \ | |
| "dial9-trace-format:codec_iai"; do | |
| pkg="${entry%:*}"; bench="${entry#*:}" | |
| bencher run \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch "$BENCHER_BRANCH" \ | |
| --testbed ubuntu-latest \ | |
| --adapter rust_iai_callgrind \ | |
| "cargo bench -p $pkg --bench $bench" | |
| done | |
| # iai micro tier on PRs: regression gate against main baseline on | |
| # Bencher (>1% instruction-count delta fails + PR comment + dashboard | |
| # alarm). Same-repo PRs only (requires BENCHER_API_TOKEN). | |
| iai_micro: | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | |
| name: iai micro benches (PR gate) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| RUST_BACKTRACE: 1 | |
| IAI_CALLGRIND_VERSION: "0.16.1" | |
| BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }} | |
| RUSTFLAGS: "--cfg tokio_unstable -C force-frame-pointers=yes --cfg iai_enabled" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install valgrind | |
| run: sudo apt-get update && sudo apt-get install -y valgrind | |
| - name: Cache iai-callgrind-runner | |
| id: runner_cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cargo/bin/iai-callgrind-runner | |
| key: iai-runner-${{ env.IAI_CALLGRIND_VERSION }} | |
| - name: Install iai-callgrind-runner | |
| if: steps.runner_cache.outputs.cache-hit != 'true' | |
| run: cargo install iai-callgrind-runner --version ${{ env.IAI_CALLGRIND_VERSION }} --locked | |
| - uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2 | |
| - name: Run iai benches → Bencher (gated vs main) | |
| run: | | |
| set -euo pipefail | |
| for entry in \ | |
| "dial9-tokio-telemetry:writer_encode_iai" \ | |
| "dial9-tokio-telemetry:writer_write_encoded_iai" \ | |
| "dial9-tokio-telemetry:threadlocal_encode_iai" \ | |
| "dial9-trace-format:codec_iai"; do | |
| pkg="${entry%:*}"; bench="${entry#*:}" | |
| bencher run \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch '${{ github.head_ref }}' \ | |
| --start-point main \ | |
| --start-point-reset \ | |
| --testbed ubuntu-latest \ | |
| --adapter rust_iai_callgrind \ | |
| --threshold-measure instructions \ | |
| --threshold-test percentage \ | |
| --threshold-upper-boundary 0.01 \ | |
| --error-on-alert \ | |
| --github-actions '${{ secrets.GITHUB_TOKEN }}' \ | |
| "cargo bench -p $pkg --bench $bench" | |
| done |