Skip to content

Commit 039b135

Browse files
thetayloredmanthetayloredman
committed
Merge branch 'copilot/fix-2d1d1bd4-db71-4375-98c6-09aa28e69d34' into 176-perf
2 parents bfe7a7b + 88f400d commit 039b135

7 files changed

Lines changed: 1069 additions & 14 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: "benchmark"
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
7+
permissions:
8+
contents: write
9+
pull-requests: write
10+
11+
jobs:
12+
benchmark:
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- name: Checkout base branch
17+
uses: actions/checkout@v5
18+
with:
19+
ref: ${{ github.base_ref || 'main' }}
20+
21+
- name: Set up Rust
22+
uses: actions-rs/toolchain@v1
23+
with:
24+
toolchain: stable
25+
override: true
26+
27+
- name: Install LLVM dependencies
28+
run: |
29+
sudo apt-get update -qq
30+
sudo apt-get install -y llvm-16 llvm-16-dev libpolly-16-dev
31+
32+
- name: Use dependency cache
33+
uses: Swatinem/rust-cache@v2
34+
35+
- name: Run benchmarks on base branch
36+
run: |
37+
cargo bench --bench compilation -- --save-baseline base
38+
39+
- name: Checkout PR branch
40+
uses: actions/checkout@v5
41+
with:
42+
ref: ${{ github.head_ref }}
43+
repository: ${{github.event.pull_request.head.repo.full_name || github.repository }}
44+
clean: false
45+
46+
- name: Run benchmarks on PR branch
47+
run: |
48+
cargo bench --bench compilation -- --baseline base 2>&1 | tee benchmark_output.txt
49+
50+
- name: Generate comparison report
51+
run: |
52+
echo "# 📊 Benchmark Comparison Report" > benchmark_report.md
53+
echo "" >> benchmark_report.md
54+
echo "Comparing performance of PR against base branch (\`${{ github.base_ref || 'main' }}\`)" >> benchmark_report.md
55+
echo "" >> benchmark_report.md
56+
57+
# Check if we have any benchmark output
58+
if [ -s benchmark_output.txt ]; then
59+
# Extract benchmark results in diff format
60+
echo "## Results" >> benchmark_report.md
61+
echo "" >> benchmark_report.md
62+
echo '```diff' >> benchmark_report.md
63+
64+
# Process each benchmark - extract clean results in diff format
65+
# Only highlight changes >= 5%
66+
awk '
67+
/^[a-z_]+[[:space:]]+time:/ {
68+
bench_name = $1;
69+
# Extract the median time (middle value)
70+
match($0, /\[[0-9.]+[[:space:]][µnm]?s[[:space:]]+([0-9.]+[[:space:]][µnm]?s)[[:space:]]+[0-9.]+[[:space:]][µnm]?s\]/, time_arr);
71+
median_time = time_arr[1];
72+
# Read next line for change
73+
getline;
74+
if ($0 ~ /change:/) {
75+
match($0, /\[[+-][0-9.]+%[[:space:]]+([+-][0-9.]+%)[[:space:]]+[+-][0-9.]+%\]/, change_arr);
76+
median_change = change_arr[1];
77+
78+
# Extract numeric value from change percentage
79+
match(median_change, /[+-]([0-9.]+)%/, num_arr);
80+
change_value = num_arr[1] + 0; # Convert to number
81+
82+
# Only highlight if change >= 5%
83+
prefix = " ";
84+
if (change_value >= 5.0) {
85+
if (median_change ~ /^[+]/) {
86+
prefix = "-"; # Regression (slower) - shows in red
87+
} else if (median_change ~ /^[-]/) {
88+
prefix = "+"; # Improvement (faster) - shows in green
89+
}
90+
}
91+
92+
# Format: diff-style with change percentage
93+
printf("%s %-25s %12s (%s)\n", prefix, bench_name, median_time, median_change);
94+
}
95+
}
96+
' benchmark_output.txt >> benchmark_report.md
97+
98+
echo '```' >> benchmark_report.md
99+
echo "" >> benchmark_report.md
100+
101+
# Calculate summary statistics
102+
improvements=$(grep -c "change:.*\[-[0-9]" benchmark_output.txt 2>/dev/null || echo "0")
103+
regressions=$(grep -c "change:.*\[+[0-9]" benchmark_output.txt 2>/dev/null || echo "0")
104+
total_benches=$(grep -c "^[a-z_][a-z_]*[[:space:]]*time:" benchmark_output.txt 2>/dev/null || echo "0")
105+
106+
echo "### Summary" >> benchmark_report.md
107+
echo "" >> benchmark_report.md
108+
echo "- **Total benchmarks:** $total_benches" >> benchmark_report.md
109+
echo "- **Improvements (faster):** $improvements 🚀" >> benchmark_report.md
110+
echo "- **Regressions (slower):** $regressions 📉" >> benchmark_report.md
111+
echo "" >> benchmark_report.md
112+
echo "> **Note:** Only changes ≥5% are highlighted in the diff. Smaller changes are shown but not color-coded, as they often represent normal variance." >> benchmark_report.md
113+
else
114+
echo "❌ No benchmark output captured. The benchmark run may have failed." >> benchmark_report.md
115+
fi
116+
117+
echo "" >> benchmark_report.md
118+
echo "---" >> benchmark_report.md
119+
echo "" >> benchmark_report.md
120+
echo "📥 **[Download Full Results & HTML Report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts)** - Click to view detailed criterion reports with charts" >> benchmark_report.md
121+
122+
- name: Comment PR with benchmark results
123+
if: github.event_name == 'pull_request'
124+
uses: actions/github-script@v7
125+
with:
126+
script: |
127+
const fs = require('fs');
128+
const report = fs.readFileSync('benchmark_report.md', 'utf8');
129+
130+
// Find existing benchmark comment
131+
const comments = await github.rest.issues.listComments({
132+
owner: context.repo.owner,
133+
repo: context.repo.repo,
134+
issue_number: context.issue.number,
135+
});
136+
137+
const botComment = comments.data.find(comment =>
138+
comment.user.type === 'Bot' &&
139+
comment.body.includes('Benchmark Comparison Report')
140+
);
141+
142+
if (botComment) {
143+
// Update existing comment
144+
await github.rest.issues.updateComment({
145+
owner: context.repo.owner,
146+
repo: context.repo.repo,
147+
comment_id: botComment.id,
148+
body: report
149+
});
150+
} else {
151+
// Create new comment
152+
await github.rest.issues.createComment({
153+
owner: context.repo.owner,
154+
repo: context.repo.repo,
155+
issue_number: context.issue.number,
156+
body: report
157+
});
158+
}
159+
160+
- name: Upload benchmark results
161+
uses: actions/upload-artifact@v4
162+
with:
163+
name: benchmark-results
164+
path: |
165+
benchmark_report.md
166+
benchmark_output.txt
167+
target/criterion/
168+
retention-days: 30

0 commit comments

Comments
 (0)