Skip to content

Commit d27cb42

Browse files
jialinli98jialinli98
authored
chore: add benchmarks (#22)
1 parent 31c1442 commit d27cb42

File tree

4 files changed

+184
-21
lines changed

4 files changed

+184
-21
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 63 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,35 +15,79 @@ jobs:
1515
uses: actions/checkout@v4
1616

1717
- name: Install Nargo
18-
uses: noir-lang/[email protected].3
18+
uses: noir-lang/[email protected].4
1919
with:
20-
toolchain: 1.0.0-beta.3
20+
toolchain: 1.0.0-beta.17
2121

2222
- name: Install bb
2323
run: |
24-
npm install -g bbup
25-
bbup -nv 1.0.0-beta.3
26-
sudo apt install libc++-dev
24+
curl -L https://raw.githubusercontent.com/AztecProtocol/aztec-packages/refs/heads/next/barretenberg/bbup/install | bash
25+
~/.bb/bbup -nv 1.0.0-beta.17
2726
2827
- name: Build Noir benchmark programs
2928
run: nargo export
30-
31-
- name: Generate gates report
29+
30+
- name: Generate gate report
3231
run: ./scripts/build-gates-report.sh
3332
env:
34-
BACKEND: /home/runner/.bb/bb
33+
BACKEND: /home/runner/.bb/bb
34+
35+
36+
- name: Store ACIR opcode benchmark result
37+
uses: benchmark-action/github-action-benchmark@v1
38+
with:
39+
name: "ACIR Opcodes"
40+
tool: "customSmallerIsBetter"
41+
output-file-path: "benchmark-opcodes.json"
42+
gh-pages-branch: "gh-pages"
43+
benchmark-data-dir-path: "dev/bench"
44+
github-token: ${{ secrets.GITHUB_TOKEN }}
45+
auto-push: ${{ github.ref == 'refs/heads/master' }}
46+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
47+
comment-on-alert: true
48+
alert-threshold: "101%"
49+
fail-on-alert: false
50+
max-items-in-chart: 50
3551

36-
- name: Compare gates reports
37-
id: gates_diff
38-
uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1
52+
- name: Store gates benchmark result
53+
uses: benchmark-action/github-action-benchmark@v1
3954
with:
40-
report: gates_report.json
41-
summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%)
55+
name: "Circuit Size"
56+
tool: "customSmallerIsBetter"
57+
output-file-path: "benchmark-circuit.json"
58+
gh-pages-branch: "gh-pages"
59+
benchmark-data-dir-path: "dev/bench"
60+
github-token: ${{ secrets.GITHUB_TOKEN }}
61+
auto-push: ${{ github.ref == 'refs/heads/master' }}
62+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
63+
comment-on-alert: true
64+
alert-threshold: "101%"
65+
fail-on-alert: false
66+
max-items-in-chart: 50
67+
skip-fetch-gh-pages: true
68+
69+
- name: Delete export files
70+
run: rm -rf export
71+
72+
- name: Build Brillig benchmark programs
73+
run: nargo export --force-brillig
74+
75+
- name: Generate brillig report
76+
run: ./scripts/build-brillig-report.sh
4277

43-
- name: Add gates diff to sticky comment
44-
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
45-
uses: marocchino/sticky-pull-request-comment@v2
78+
- name: Store brillig benchmark result
79+
uses: benchmark-action/github-action-benchmark@v1
4680
with:
47-
# delete the comment in case changes no longer impact circuit sizes
48-
delete: ${{ !steps.gates_diff.outputs.markdown }}
49-
message: ${{ steps.gates_diff.outputs.markdown }}
81+
name: "Brillig Bytecode Size"
82+
tool: "customSmallerIsBetter"
83+
output-file-path: "benchmark-brillig.json"
84+
gh-pages-branch: "gh-pages"
85+
benchmark-data-dir-path: "dev/bench"
86+
github-token: ${{ secrets.GITHUB_TOKEN }}
87+
auto-push: ${{ github.ref == 'refs/heads/master' }}
88+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
89+
comment-on-alert: true
90+
alert-threshold: "101%"
91+
fail-on-alert: false
92+
max-items-in-chart: 50
93+
skip-fetch-gh-pages: true

‎scripts/build-brillig-report.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
INSPECTOR=${INSPECTOR:-noir-inspector}
5+
cd $(dirname "$0")/../
6+
7+
artifacts_path="./export"
8+
artifacts=$(ls $artifacts_path)
9+
10+
# Start the JSON array
11+
REPORTS=$(jq --null-input '[]')
12+
13+
for artifact in $artifacts; do
14+
# Get and format the opcode info
15+
OP_CODE_INFO=$($INSPECTOR info --json "$artifacts_path/$artifact")
16+
17+
# Simplified jq expression to output only package_name and opcodes from unconstrained_functions
18+
REPORTS=$(echo $OP_CODE_INFO | jq -c '.programs[0] | del(.functions)' | jq -c --argjson old_reports $REPORTS '$old_reports + [.]')
19+
done
20+
21+
echo $REPORTS | jq '{ programs: . }' > brillig_report.json
22+
23+
# Convert brillig report to benchmark format
24+
output_file_brillig="benchmark-brillig.json"
25+
jq -r '[.programs[] | .unconstrained_functions[] | {
26+
"name": (if (.package_name // "") == "" then .name else "\(.package_name | sub("^null/"; ""))/\(.name)" end),
27+
"unit": "opcodes",
28+
"value": (.opcodes // 0)
29+
}]' brillig_report.json > $output_file_brillig

‎scripts/build-gates-report.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,21 @@ done
3232

3333
echo "]}" >> gates_report.json
3434

35+
# Convert the gates report into separate benchmark files
36+
output_file_opcodes="benchmark-opcodes.json"
37+
output_file_circuit="benchmark-circuit.json"
38+
39+
# Convert gates report - opcodes
40+
jq -r '[.programs[] | {
41+
"name": "\(.package_name)/main",
42+
"unit": "acir_opcodes",
43+
"value": (.functions[0].acir_opcodes // 0)
44+
}]' gates_report.json > $output_file_opcodes
45+
46+
# Convert gates report - circuit size
47+
jq -r '[.programs[] | {
48+
"name": "\(.package_name)/main",
49+
"unit": "circuit_size",
50+
"value": (.functions[0].circuit_size // 0)
51+
}]' gates_report.json > $output_file_circuit
3552

‎src/bench.nr‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,79 @@
1-
use super::poseidon2;
1+
use super::{poseidon, poseidon2};
22

3+
// Poseidon2 benchmarks
34
#[export]
4-
fn poseidon2_length_10(input: [Field; 10]) -> Field {
5+
fn poseidon2_hash_1(input: [Field; 1]) -> Field {
56
poseidon2::Poseidon2::hash(input, input.len())
67
}
8+
9+
#[export]
10+
fn poseidon2_hash_2(input: [Field; 2]) -> Field {
11+
poseidon2::Poseidon2::hash(input, input.len())
12+
}
13+
14+
#[export]
15+
fn poseidon2_hash_4(input: [Field; 4]) -> Field {
16+
poseidon2::Poseidon2::hash(input, input.len())
17+
}
18+
19+
#[export]
20+
fn poseidon2_hash_8(input: [Field; 8]) -> Field {
21+
poseidon2::Poseidon2::hash(input, input.len())
22+
}
23+
24+
#[export]
25+
fn poseidon2_hash_10(input: [Field; 10]) -> Field {
26+
poseidon2::Poseidon2::hash(input, input.len())
27+
}
28+
29+
#[export]
30+
fn poseidon2_hash_20(input: [Field; 20]) -> Field {
31+
poseidon2::Poseidon2::hash(input, input.len())
32+
}
33+
34+
// Poseidon benchmarks
35+
#[export]
36+
fn poseidon_hash_1(input: [Field; 1]) -> Field {
37+
poseidon::bn254::hash_1(input)
38+
}
39+
40+
#[export]
41+
fn poseidon_hash_2(input: [Field; 2]) -> Field {
42+
poseidon::bn254::hash_2(input)
43+
}
44+
45+
#[export]
46+
fn poseidon_hash_4(input: [Field; 4]) -> Field {
47+
poseidon::bn254::hash_4(input)
48+
}
49+
50+
#[export]
51+
fn poseidon_hash_8(input: [Field; 8]) -> Field {
52+
poseidon::bn254::hash_8(input)
53+
}
54+
55+
// Variable-length sponge benchmarks
56+
#[export]
57+
fn poseidon_sponge_1(input: [Field; 1]) -> Field {
58+
poseidon::bn254::sponge(input)
59+
}
60+
61+
#[export]
62+
fn poseidon_sponge_4(input: [Field; 4]) -> Field {
63+
poseidon::bn254::sponge(input)
64+
}
65+
66+
#[export]
67+
fn poseidon_sponge_8(input: [Field; 8]) -> Field {
68+
poseidon::bn254::sponge(input)
69+
}
70+
71+
#[export]
72+
fn poseidon_sponge_16(input: [Field; 16]) -> Field {
73+
poseidon::bn254::sponge(input)
74+
}
75+
76+
#[export]
77+
fn poseidon_sponge_32(input: [Field; 32]) -> Field {
78+
poseidon::bn254::sponge(input)
79+
}

0 commit comments

Comments
 (0)