Skip to content

Commit 0c3d396

Browse files
yulnryulnr
committed
benches: add iai-callgrind PR gate, retire criterion CI
Replace noisy wall-clock benches on shared ubuntu-latest with a deterministic instruction-count gate. iai-callgrind runs each bench once under callgrind, so same code + same input gives byte-identical counts; a 1% threshold catches real regressions without false fires.
1 parent 00dc9bf commit 0c3d396

8 files changed

Lines changed: 653 additions & 52 deletions

File tree

‎.github/workflows/benchmarks.yml‎

Lines changed: 86 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ permissions:
2424
pull-requests: write
2525

2626
jobs:
27-
# Runs on every push to main to build the statistical baseline.
27+
# Disabled until a dedicated runner exists. Wall-clock criterion +
28+
# integration benches are too noisy on shared ubuntu-latest. iai jobs
29+
# below cover the deterministic micro tier in the meantime.
2830
benchmark_main:
2931
name: Benchmark — ${{ matrix.bench.name }} (main baseline)
3032
if: ${{ false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} # skip job
@@ -56,14 +58,11 @@ jobs:
5658
- uses: actions/checkout@v4
5759
- uses: dtolnay/rust-toolchain@stable
5860
- uses: Swatinem/rust-cache@v2
59-
6061
- name: Enable perf_event_open and kallsyms
6162
run: |
6263
sudo sysctl kernel.perf_event_paranoid=1
6364
sudo sysctl kernel.kptr_restrict=0
64-
6565
- uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2
66-
6766
- name: Run benchmark
6867
run: |
6968
bencher run \
@@ -73,64 +72,99 @@ jobs:
7372
--adapter '${{ matrix.bench.adapter }}' \
7473
"${{ matrix.bench.command }}"
7574
76-
# Runs on same-repo PRs. Fork PRs are skipped — they have no access to
77-
# BENCHER_API_TOKEN, so the job would fail rather than silently skip.
78-
benchmark_pr:
79-
name: Benchmark — ${{ matrix.bench.name }} (PR regression check)
80-
if: ${{ false && (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }} # skip job
75+
# iai micro tier on push to main: populates the Bencher baseline used
76+
# by iai_micro PR gate below. Instruction counts are deterministic so
77+
# shared ubuntu-latest is fine.
78+
iai_main:
79+
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
80+
name: iai micro benches (main baseline)
8181
runs-on: ubuntu-latest
8282
timeout-minutes: 30
83-
strategy:
84-
fail-fast: false
85-
matrix:
86-
bench:
87-
- name: writer_encode
88-
adapter: rust_criterion
89-
command: cargo bench --package dial9-tokio-telemetry --bench writer_encode
90-
- name: codec
91-
adapter: rust_criterion
92-
command: cargo bench --package dial9-trace-format --bench codec
93-
- name: overhead_bench
94-
adapter: json
95-
command: cargo bench --bench overhead_bench -- --bmf 10
96-
- name: overhead_bench_ctimer
97-
adapter: json
98-
command: DIAL9_FORCE_CTIMER=1 cargo bench --bench overhead_bench -- --bmf 10
99-
- name: e2e_workload
100-
adapter: json
101-
command: cargo bench --bench e2e_workload -- --bmf 10
10283
env:
10384
RUST_BACKTRACE: 1
85+
IAI_CALLGRIND_VERSION: "0.16.1"
10486
BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
10587
steps:
10688
- uses: actions/checkout@v4
10789
- uses: dtolnay/rust-toolchain@stable
10890
- uses: Swatinem/rust-cache@v2
109-
110-
- name: Enable perf_event_open and kallsyms
91+
- name: Install valgrind
92+
run: sudo apt-get update && sudo apt-get install -y valgrind
93+
- name: Cache iai-callgrind-runner
94+
id: runner_cache
95+
uses: actions/cache@v4
96+
with:
97+
path: ~/.cargo/bin/iai-callgrind-runner
98+
key: iai-runner-${{ env.IAI_CALLGRIND_VERSION }}
99+
- name: Install iai-callgrind-runner
100+
if: steps.runner_cache.outputs.cache-hit != 'true'
101+
run: cargo install iai-callgrind-runner --version ${{ env.IAI_CALLGRIND_VERSION }} --locked
102+
- uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2
103+
- name: Run iai benches → Bencher
111104
run: |
112-
sudo sysctl kernel.perf_event_paranoid=1
113-
sudo sysctl kernel.kptr_restrict=0
105+
set -euo pipefail
106+
for entry in \
107+
"dial9-tokio-telemetry:writer_encode_iai" \
108+
"dial9-tokio-telemetry:writer_write_encoded_iai" \
109+
"dial9-tokio-telemetry:threadlocal_encode_iai" \
110+
"dial9-trace-format:codec_iai"; do
111+
pkg="${entry%:*}"; bench="${entry#*:}"
112+
bencher run \
113+
--token '${{ secrets.BENCHER_API_TOKEN }}' \
114+
--branch '${{ github.ref_name }}' \
115+
--testbed ubuntu-latest \
116+
--adapter rust_iai_callgrind \
117+
"cargo bench -p $pkg --bench $bench"
118+
done
114119
120+
# iai micro tier on PRs: regression gate against main baseline on
121+
# Bencher (>1% instruction-count delta fails + PR comment + dashboard
122+
# alarm). Same-repo PRs only (requires BENCHER_API_TOKEN).
123+
iai_micro:
124+
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
125+
name: iai micro benches (PR gate)
126+
runs-on: ubuntu-latest
127+
timeout-minutes: 30
128+
env:
129+
RUST_BACKTRACE: 1
130+
IAI_CALLGRIND_VERSION: "0.16.1"
131+
BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
132+
steps:
133+
- uses: actions/checkout@v4
134+
- uses: dtolnay/rust-toolchain@stable
135+
- uses: Swatinem/rust-cache@v2
136+
- name: Install valgrind
137+
run: sudo apt-get update && sudo apt-get install -y valgrind
138+
- name: Cache iai-callgrind-runner
139+
id: runner_cache
140+
uses: actions/cache@v4
141+
with:
142+
path: ~/.cargo/bin/iai-callgrind-runner
143+
key: iai-runner-${{ env.IAI_CALLGRIND_VERSION }}
144+
- name: Install iai-callgrind-runner
145+
if: steps.runner_cache.outputs.cache-hit != 'true'
146+
run: cargo install iai-callgrind-runner --version ${{ env.IAI_CALLGRIND_VERSION }} --locked
115147
- uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2
116-
117-
- name: Run benchmark
148+
- name: Run iai benches → Bencher (gated vs main)
118149
run: |
119-
bencher run \
120-
--token '${{ secrets.BENCHER_API_TOKEN }}' \
121-
--branch '${{ github.head_ref }}' \
122-
--start-point main \
123-
--start-point-reset \
124-
--testbed ubuntu-latest \
125-
--adapter '${{ matrix.bench.adapter }}' \
126-
--threshold-measure latency \
127-
--threshold-test percentage \
128-
--threshold-lower-boundary _ \
129-
--threshold-upper-boundary 0.25 \
130-
--threshold-measure throughput \
131-
--threshold-test percentage \
132-
--threshold-lower-boundary 0.25 \
133-
--threshold-upper-boundary _ \
134-
--error-on-alert \
135-
--github-actions '${{ secrets.GITHUB_TOKEN }}' \
136-
"${{ matrix.bench.command }}"
150+
set -euo pipefail
151+
for entry in \
152+
"dial9-tokio-telemetry:writer_encode_iai" \
153+
"dial9-tokio-telemetry:writer_write_encoded_iai" \
154+
"dial9-tokio-telemetry:threadlocal_encode_iai" \
155+
"dial9-trace-format:codec_iai"; do
156+
pkg="${entry%:*}"; bench="${entry#*:}"
157+
bencher run \
158+
--token '${{ secrets.BENCHER_API_TOKEN }}' \
159+
--branch '${{ github.head_ref }}' \
160+
--start-point main \
161+
--start-point-reset \
162+
--testbed ubuntu-latest \
163+
--adapter rust_iai_callgrind \
164+
--threshold-measure instructions \
165+
--threshold-test percentage \
166+
--threshold-upper-boundary 0.01 \
167+
--error-on-alert \
168+
--github-actions '${{ secrets.GITHUB_TOKEN }}' \
169+
"cargo bench -p $pkg --bench $bench"
170+
done

‎Cargo.lock‎

Lines changed: 90 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dial9-tokio-telemetry/Cargo.toml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ aws-sdk-s3 = { version = "1", default-features = false, features = ["behavior-ve
7272
aws-config = { version = "1", features = ["behavior-version-latest"] }
7373
async-trait = "0.1.89"
7474
uuid = { version = "1", features = ["v4"] }
75+
iai-callgrind = "0.16"
7576

7677
[target.'cfg(target_os = "linux")'.dev-dependencies]
7778
dial9-tokio-telemetry = { path = ".", features = ["cpu-profiling", "worker-s3", "analysis", "tracing-layer"] }
@@ -133,3 +134,15 @@ harness = false
133134
name = "tracing_layer_bench"
134135
harness = false
135136
required-features = ["tracing-layer"]
137+
138+
[[bench]]
139+
name = "writer_encode_iai"
140+
harness = false
141+
142+
[[bench]]
143+
name = "threadlocal_encode_iai"
144+
harness = false
145+
146+
[[bench]]
147+
name = "writer_write_encoded_iai"
148+
harness = false

0 commit comments

Comments
 (0)