Skip to content

Commit b38175c

Browse files
yulnryulnr
committed
benches: add iai-callgrind PR gate, retire criterion CI
Replace noisy wall-clock benches on shared ubuntu-latest with a deterministic instruction-count gate. iai-callgrind runs each bench once under callgrind, so same code + same input gives byte-identical counts; a 1% threshold catches real regressions without false fires.
1 parent 82de0df commit b38175c

8 files changed

Lines changed: 653 additions & 54 deletions

File tree

‎.github/workflows/benchmarks.yml‎

Lines changed: 86 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ permissions:
2424
pull-requests: write
2525

2626
jobs:
27-
# Runs on every push to main to build the statistical baseline.
27+
# Disabled until a dedicated runner exists. Wall-clock criterion +
28+
# integration benches are too noisy on shared ubuntu-latest. iai jobs
29+
# below cover the deterministic micro tier in the meantime.
2830
benchmark_main:
2931
if: ${{ false }} # skip job
3032
name: Benchmark — ${{ matrix.bench.name }} (main baseline)
31-
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
3233
runs-on: ubuntu-latest
3334
timeout-minutes: 30
3435
strategy:
@@ -57,14 +58,11 @@ jobs:
5758
- uses: actions/checkout@v4
5859
- uses: dtolnay/rust-toolchain@stable
5960
- uses: Swatinem/rust-cache@v2
60-
6161
- name: Enable perf_event_open and kallsyms
6262
run: |
6363
sudo sysctl kernel.perf_event_paranoid=1
6464
sudo sysctl kernel.kptr_restrict=0
65-
6665
- uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2
67-
6866
- name: Run benchmark
6967
run: |
7068
bencher run \
@@ -74,65 +72,99 @@ jobs:
7472
--adapter '${{ matrix.bench.adapter }}' \
7573
"${{ matrix.bench.command }}"
7674
77-
# Runs on same-repo PRs. Fork PRs are skipped — they have no access to
78-
# BENCHER_API_TOKEN, so the job would fail rather than silently skip.
79-
benchmark_pr:
80-
if: ${{ false }} # skip job
81-
name: Benchmark — ${{ matrix.bench.name }} (PR regression check)
82-
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
75+
# iai micro tier on push to main: populates the Bencher baseline used
76+
# by iai_micro PR gate below. Instruction counts are deterministic so
77+
# shared ubuntu-latest is fine.
78+
iai_main:
79+
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
80+
name: iai micro benches (main baseline)
8381
runs-on: ubuntu-latest
8482
timeout-minutes: 30
85-
strategy:
86-
fail-fast: false
87-
matrix:
88-
bench:
89-
- name: writer_encode
90-
adapter: rust_criterion
91-
command: cargo bench --package dial9-tokio-telemetry --bench writer_encode
92-
- name: codec
93-
adapter: rust_criterion
94-
command: cargo bench --package dial9-trace-format --bench codec
95-
- name: overhead_bench
96-
adapter: json
97-
command: cargo bench --bench overhead_bench -- --bmf 10
98-
- name: overhead_bench_ctimer
99-
adapter: json
100-
command: DIAL9_FORCE_CTIMER=1 cargo bench --bench overhead_bench -- --bmf 10
101-
- name: e2e_workload
102-
adapter: json
103-
command: cargo bench --bench e2e_workload -- --bmf 10
10483
env:
10584
RUST_BACKTRACE: 1
85+
IAI_CALLGRIND_VERSION: "0.16.1"
10686
BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
10787
steps:
10888
- uses: actions/checkout@v4
10989
- uses: dtolnay/rust-toolchain@stable
11090
- uses: Swatinem/rust-cache@v2
111-
112-
- name: Enable perf_event_open and kallsyms
91+
- name: Install valgrind
92+
run: sudo apt-get update && sudo apt-get install -y valgrind
93+
- name: Cache iai-callgrind-runner
94+
id: runner_cache
95+
uses: actions/cache@v4
96+
with:
97+
path: ~/.cargo/bin/iai-callgrind-runner
98+
key: iai-runner-${{ env.IAI_CALLGRIND_VERSION }}
99+
- name: Install iai-callgrind-runner
100+
if: steps.runner_cache.outputs.cache-hit != 'true'
101+
run: cargo install iai-callgrind-runner --version ${{ env.IAI_CALLGRIND_VERSION }} --locked
102+
- uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2
103+
- name: Run iai benches → Bencher
113104
run: |
114-
sudo sysctl kernel.perf_event_paranoid=1
115-
sudo sysctl kernel.kptr_restrict=0
105+
set -euo pipefail
106+
for entry in \
107+
"dial9-tokio-telemetry:writer_encode_iai" \
108+
"dial9-tokio-telemetry:writer_write_encoded_iai" \
109+
"dial9-tokio-telemetry:threadlocal_encode_iai" \
110+
"dial9-trace-format:codec_iai"; do
111+
pkg="${entry%:*}"; bench="${entry#*:}"
112+
bencher run \
113+
--token '${{ secrets.BENCHER_API_TOKEN }}' \
114+
--branch '${{ github.ref_name }}' \
115+
--testbed ubuntu-latest \
116+
--adapter rust_iai_callgrind \
117+
"cargo bench -p $pkg --bench $bench"
118+
done
116119
120+
# iai micro tier on PRs: regression gate against main baseline on
121+
# Bencher (>1% instruction-count delta fails + PR comment + dashboard
122+
# alarm). Same-repo PRs only (requires BENCHER_API_TOKEN).
123+
iai_micro:
124+
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
125+
name: iai micro benches (PR gate)
126+
runs-on: ubuntu-latest
127+
timeout-minutes: 30
128+
env:
129+
RUST_BACKTRACE: 1
130+
IAI_CALLGRIND_VERSION: "0.16.1"
131+
BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
132+
steps:
133+
- uses: actions/checkout@v4
134+
- uses: dtolnay/rust-toolchain@stable
135+
- uses: Swatinem/rust-cache@v2
136+
- name: Install valgrind
137+
run: sudo apt-get update && sudo apt-get install -y valgrind
138+
- name: Cache iai-callgrind-runner
139+
id: runner_cache
140+
uses: actions/cache@v4
141+
with:
142+
path: ~/.cargo/bin/iai-callgrind-runner
143+
key: iai-runner-${{ env.IAI_CALLGRIND_VERSION }}
144+
- name: Install iai-callgrind-runner
145+
if: steps.runner_cache.outputs.cache-hit != 'true'
146+
run: cargo install iai-callgrind-runner --version ${{ env.IAI_CALLGRIND_VERSION }} --locked
117147
- uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2
118-
119-
- name: Run benchmark
148+
- name: Run iai benches → Bencher (gated vs main)
120149
run: |
121-
bencher run \
122-
--token '${{ secrets.BENCHER_API_TOKEN }}' \
123-
--branch '${{ github.head_ref }}' \
124-
--start-point main \
125-
--start-point-reset \
126-
--testbed ubuntu-latest \
127-
--adapter '${{ matrix.bench.adapter }}' \
128-
--threshold-measure latency \
129-
--threshold-test percentage \
130-
--threshold-lower-boundary _ \
131-
--threshold-upper-boundary 0.25 \
132-
--threshold-measure throughput \
133-
--threshold-test percentage \
134-
--threshold-lower-boundary 0.25 \
135-
--threshold-upper-boundary _ \
136-
--error-on-alert \
137-
--github-actions '${{ secrets.GITHUB_TOKEN }}' \
138-
"${{ matrix.bench.command }}"
150+
set -euo pipefail
151+
for entry in \
152+
"dial9-tokio-telemetry:writer_encode_iai" \
153+
"dial9-tokio-telemetry:writer_write_encoded_iai" \
154+
"dial9-tokio-telemetry:threadlocal_encode_iai" \
155+
"dial9-trace-format:codec_iai"; do
156+
pkg="${entry%:*}"; bench="${entry#*:}"
157+
bencher run \
158+
--token '${{ secrets.BENCHER_API_TOKEN }}' \
159+
--branch '${{ github.head_ref }}' \
160+
--start-point main \
161+
--start-point-reset \
162+
--testbed ubuntu-latest \
163+
--adapter rust_iai_callgrind \
164+
--threshold-measure instructions \
165+
--threshold-test percentage \
166+
--threshold-upper-boundary 0.01 \
167+
--error-on-alert \
168+
--github-actions '${{ secrets.GITHUB_TOKEN }}' \
169+
"cargo bench -p $pkg --bench $bench"
170+
done

‎Cargo.lock‎

Lines changed: 90 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dial9-tokio-telemetry/Cargo.toml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ aws-sdk-s3 = "1"
6767
aws-config = { version = "1", features = ["behavior-version-latest"] }
6868
async-trait = "0.1.89"
6969
uuid = { version = "1", features = ["v4"] }
70+
iai-callgrind = "0.16"
7071

7172
[target.'cfg(target_os = "linux")'.dev-dependencies]
7273
dial9-tokio-telemetry = { path = ".", features = ["cpu-profiling", "worker-s3", "analysis", "tracing-layer"] }
@@ -128,3 +129,15 @@ harness = false
128129
name = "tracing_layer_bench"
129130
harness = false
130131
required-features = ["tracing-layer"]
132+
133+
[[bench]]
134+
name = "writer_encode_iai"
135+
harness = false
136+
137+
[[bench]]
138+
name = "threadlocal_encode_iai"
139+
harness = false
140+
141+
[[bench]]
142+
name = "writer_write_encoded_iai"
143+
harness = false

0 commit comments

Comments
 (0)