Skip to content

Commit 0f1380e

Browse files
committed
ci(bench): compare base and PR on one runner
Run both revisions sequentially on the same GitHub-hosted machine so runtime SIMD dispatch and host variability cannot create false regression reports. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01071e57-12fd-4f42-bc67-e2b274d0b007
1 parent e1b2653 commit 0f1380e

1 file changed

Lines changed: 30 additions & 34 deletions

File tree

.github/workflows/benchmark.yml

Lines changed: 30 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,14 @@ permissions:
99
pull-requests: write
1010

1111
jobs:
12-
benchmark-base:
13-
name: Benchmark base branch
12+
benchmark:
13+
name: Benchmark base and PR
1414
runs-on: ubuntu-latest
1515
steps:
16-
- name: Checkout base branch
16+
- name: Checkout repository
1717
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
1818
with:
19-
ref: ${{ github.event.pull_request.base.sha }}
19+
fetch-depth: 0
2020

2121
- name: Set up Python 3.11
2222
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
@@ -27,51 +27,47 @@ jobs:
2727
run: |
2828
python -m pip install --upgrade pip
2929
pip install pytest pytest-benchmark
30-
pip install .
3130
32-
- name: Run benchmarks
31+
- name: Record benchmark host
3332
run: |
34-
pytest test/ -k bench --benchmark-only --benchmark-json=results.json --benchmark-disable-gc
33+
uname -a
34+
lscpu
3535
36-
- name: Upload benchmark results
36+
- name: Benchmark base branch
37+
env:
38+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
39+
run: |
40+
git checkout --detach "$BASE_SHA"
41+
pip install --force-reinstall --no-deps .
42+
mkdir -p base
43+
pytest test/ -k bench --benchmark-only --benchmark-json=base/results.json --benchmark-disable-gc
44+
45+
- name: Benchmark PR branch
46+
env:
47+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
48+
run: |
49+
git checkout --detach "$HEAD_SHA"
50+
pip install --force-reinstall --no-deps .
51+
mkdir -p pr
52+
pytest test/ -k bench --benchmark-only --benchmark-json=pr/results.json --benchmark-disable-gc
53+
54+
- name: Upload base benchmark results
3755
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
3856
with:
3957
name: benchmark-base
40-
path: results.json
58+
path: base/results.json
4159
retention-days: 1
4260

43-
benchmark-pr:
44-
name: Benchmark PR branch
45-
runs-on: ubuntu-latest
46-
steps:
47-
- name: Checkout PR branch
48-
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
49-
50-
- name: Set up Python 3.11
51-
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
52-
with:
53-
python-version: "3.11"
54-
55-
- name: Install dependencies
56-
run: |
57-
python -m pip install --upgrade pip
58-
pip install pytest pytest-benchmark
59-
pip install .
60-
61-
- name: Run benchmarks
62-
run: |
63-
pytest test/ -k bench --benchmark-only --benchmark-json=results.json --benchmark-disable-gc
64-
65-
- name: Upload benchmark results
61+
- name: Upload PR benchmark results
6662
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
6763
with:
6864
name: benchmark-pr
69-
path: results.json
65+
path: pr/results.json
7066
retention-days: 1
7167

7268
compare:
7369
name: Compare benchmarks
74-
needs: [benchmark-base, benchmark-pr]
70+
needs: benchmark
7571
runs-on: ubuntu-latest
7672
steps:
7773
- name: Download base benchmark results

0 commit comments

Comments
 (0)