dial9/.github/workflows/benchmarks.yml at e172a273af3c60edca699daf7c60753532e241a6 · dial9-rs/dial9 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
# Continuous benchmarking with Bencher (https://bencher.dev)
#
# PREREQUISITES:
#   1. Create a project at https://bencher.dev and note its slug.
#   2. Add the following in GitHub → Settings → Secrets and variables → Actions:
#      - Secret  BENCHER_API_TOKEN: API token from bencher.dev
#      - Variable BENCHER_PROJECT:  your project slug
#   3. GITHUB_TOKEN is provided automatically — no setup needed.
#
# overhead_bench uses a custom harness (not Criterion). Its --bmf flag
# outputs Bencher Metric Format directly.

name: Benchmarks

on:
  push:
    branches:
      - main
  pull_request:
  workflow_dispatch:

permissions:
  checks: write
  pull-requests: write

jobs:
  # Runs on every push to main to build the statistical baseline.
  benchmark_main:
    name: Benchmark — ${{ matrix.bench.name }} (main baseline)
    if: ${{ false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} # skip job
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        bench:
          - name: writer_encode
            adapter: rust_criterion
            command: cargo bench --package dial9-tokio-telemetry --bench writer_encode
          - name: codec
            adapter: rust_criterion
            command: cargo bench --package dial9-trace-format --bench codec
          - name: overhead_bench
            adapter: json
            command: cargo bench --bench overhead_bench -- --bmf 10
          - name: overhead_bench_ctimer
            adapter: json
            command: DIAL9_FORCE_CTIMER=1 cargo bench --bench overhead_bench -- --bmf 10
          - name: e2e_workload
            adapter: json
            command: cargo bench --bench e2e_workload -- --bmf 10
    env:
      RUST_BACKTRACE: 1
      BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - name: Enable perf_event_open and kallsyms
        run: |
          sudo sysctl kernel.perf_event_paranoid=1
          sudo sysctl kernel.kptr_restrict=0

      - uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2

      - name: Run benchmark
        run: |
          bencher run \
            --token '${{ secrets.BENCHER_API_TOKEN }}' \
            --branch '${{ github.ref_name }}' \
            --testbed ubuntu-latest \
            --adapter '${{ matrix.bench.adapter }}' \
            "${{ matrix.bench.command }}"

  # Runs on same-repo PRs. Fork PRs are skipped — they have no access to
  # BENCHER_API_TOKEN, so the job would fail rather than silently skip.
  benchmark_pr:
    name: Benchmark — ${{ matrix.bench.name }} (PR regression check)
    if: ${{ false && (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }} # skip job
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        bench:
          - name: writer_encode
            adapter: rust_criterion
            command: cargo bench --package dial9-tokio-telemetry --bench writer_encode
          - name: codec
            adapter: rust_criterion
            command: cargo bench --package dial9-trace-format --bench codec
          - name: overhead_bench
            adapter: json
            command: cargo bench --bench overhead_bench -- --bmf 10
          - name: overhead_bench_ctimer
            adapter: json
            command: DIAL9_FORCE_CTIMER=1 cargo bench --bench overhead_bench -- --bmf 10
          - name: e2e_workload
            adapter: json
            command: cargo bench --bench e2e_workload -- --bmf 10
    env:
      RUST_BACKTRACE: 1
      BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - name: Enable perf_event_open and kallsyms
        run: |
          sudo sysctl kernel.perf_event_paranoid=1
          sudo sysctl kernel.kptr_restrict=0

      - uses: bencherdev/bencher@0f8f620172ccd6225d40a7590598eb7b41718af8 # v0.6.2

      - name: Run benchmark
        run: |
          bencher run \
            --token '${{ secrets.BENCHER_API_TOKEN }}' \
            --branch '${{ github.head_ref }}' \
            --start-point main \
            --start-point-reset \
            --testbed ubuntu-latest \
            --adapter '${{ matrix.bench.adapter }}' \
            --threshold-measure latency \
            --threshold-test percentage \
            --threshold-lower-boundary _ \
            --threshold-upper-boundary 0.25 \
            --threshold-measure throughput \
            --threshold-test percentage \
            --threshold-lower-boundary 0.25 \
            --threshold-upper-boundary _ \
            --error-on-alert \
            --github-actions '${{ secrets.GITHUB_TOKEN }}' \
            "${{ matrix.bench.command }}"