Skip to content

Commit 3b9e364

Browse files
jialinli98jialinli98
committed
benchmark
1 parent 4cfe4f6 commit 3b9e364

File tree

6 files changed

+247
-1
lines changed

6 files changed

+247
-1
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
name: Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
9+
jobs:
10+
test:
11+
name: Benchmark library
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout sources
15+
uses: actions/checkout@v4
16+
17+
- name: Install Nargo
18+
uses: noir-lang/[email protected]
19+
with:
20+
toolchain: 1.0.0-beta.7
21+
22+
- name: Install bb
23+
run: |
24+
curl -L https://bbup.aztec.network | bash
25+
~/.bb/bbup -nv 1.0.0-beta.7
26+
sudo apt install libc++-dev
27+
28+
- name: Build Noir benchmark programs
29+
run: nargo export
30+
31+
- name: Generate gates report
32+
run: ./scripts/build-gates-report.sh
33+
env:
34+
BACKEND: /home/runner/.bb/bb
35+
36+
- name: Compare gates reports
37+
id: gates_diff
38+
uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1
39+
with:
40+
report: gates_report.json
41+
summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%)
42+
43+
- name: Store ACIR opcode benchmark result
44+
uses: benchmark-action/github-action-benchmark@v1
45+
with:
46+
name: "ACIR Opcodes"
47+
tool: "customSmallerIsBetter"
48+
output-file-path: "benchmark-opcodes.json"
49+
gh-pages-branch: "gh-pages"
50+
benchmark-data-dir-path: "dev/bench"
51+
github-token: ${{ secrets.GITHUB_TOKEN }}
52+
auto-push: ${{ github.ref == 'refs/heads/main' }}
53+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
54+
comment-on-alert: true
55+
alert-threshold: "101%"
56+
fail-on-alert: false
57+
max-items-in-chart: 50
58+
59+
- name: Store gates benchmark result
60+
uses: benchmark-action/github-action-benchmark@v1
61+
with:
62+
name: "Circuit Size"
63+
tool: "customSmallerIsBetter"
64+
output-file-path: "benchmark-circuit.json"
65+
gh-pages-branch: "gh-pages"
66+
benchmark-data-dir-path: "dev/bench"
67+
github-token: ${{ secrets.GITHUB_TOKEN }}
68+
auto-push: ${{ github.ref == 'refs/heads/main' }}
69+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
70+
comment-on-alert: true
71+
alert-threshold: "101%"
72+
fail-on-alert: false
73+
max-items-in-chart: 50
74+
skip-fetch-gh-pages: true
75+
76+
- name: Add gates diff to sticky comment
77+
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
78+
uses: marocchino/sticky-pull-request-comment@v2
79+
with:
80+
# delete the comment in case changes no longer impact circuit sizes
81+
delete: ${{ !steps.gates_diff.outputs.markdown }}
82+
message: ${{ steps.gates_diff.outputs.markdown }}
83+
84+
# Delete the export files
85+
- name: Delete export files
86+
run: rm -rf export
87+
# Run nargo export again with force-brillig flag
88+
- name: Build Brillig benchmark programs
89+
run: nargo export --force-brillig
90+
91+
- name: Generate brillig report
92+
run: ./scripts/build-brillig-report.sh
93+
94+
- name: Compare brillig reports
95+
id: brillig_bytecode_diff
96+
uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1
97+
with:
98+
report: brillig_report.json
99+
header: |
100+
# Changes to Brillig bytecode sizes
101+
brillig_report: true
102+
summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%)
103+
104+
- name: Store brillig benchmark result
105+
uses: benchmark-action/github-action-benchmark@v1
106+
with:
107+
name: "Brillig Bytecode Size"
108+
tool: "customSmallerIsBetter"
109+
output-file-path: "benchmark-brillig.json"
110+
gh-pages-branch: "gh-pages"
111+
benchmark-data-dir-path: "dev/bench"
112+
github-token: ${{ secrets.GITHUB_TOKEN }}
113+
auto-push: ${{ github.ref == 'refs/heads/main' }}
114+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
115+
comment-on-alert: true
116+
alert-threshold: "101%"
117+
fail-on-alert: false
118+
max-items-in-chart: 50
119+
skip-fetch-gh-pages: true
120+
121+
- name: Add brillig diff to sticky comment
122+
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
123+
uses: marocchino/sticky-pull-request-comment@v2
124+
with:
125+
# delete the comment in case changes no longer impact circuit sizes
126+
delete: ${{ !steps.brillig_diff.outputs.markdown }}
127+
message: ${{ steps.brillig_diff.outputs.markdown }}

‎scripts/build-brillig-report.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
INSPECTOR=${INSPECTOR:-noir-inspector}
5+
cd $(dirname "$0")/../
6+
7+
artifacts_path="./export"
8+
artifacts=$(ls $artifacts_path)
9+
10+
# Start the JSON array
11+
REPORTS=$(jq --null-input '[]')
12+
13+
for artifact in $artifacts; do
14+
# Get and format the opcode info
15+
OP_CODE_INFO=$($INSPECTOR info --json "$artifacts_path/$artifact")
16+
17+
# Simplified jq expression to output only package_name and opcodes from unconstrained_functions
18+
REPORTS=$(echo $OP_CODE_INFO | jq -c '.programs[0] | del(.functions)' | jq -c --argjson old_reports $REPORTS '$old_reports + [.]')
19+
done
20+
21+
echo $REPORTS | jq '{ programs: . }' > brillig_report.json
22+
23+
# Convert brillig report to benchmark format
24+
output_file_brillig="benchmark-brillig.json"
25+
jq -r '[.programs[] | .unconstrained_functions[] | {
26+
"name": (if (.package_name // "") == "" then .name else "\(.package_name | sub("^null/"; ""))/\(.name)" end),
27+
"unit": "opcodes",
28+
"value": (.opcodes // 0)
29+
}]' brillig_report.json > $output_file_brillig

‎scripts/build-gates-report.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
BACKEND=${BACKEND:-bb}
5+
6+
cd $(dirname "$0")/../
7+
8+
artifacts_path="./export"
9+
artifacts=$(ls $artifacts_path)
10+
11+
echo "{\"programs\": [" > gates_report.json
12+
13+
# Bound for checking where to place last parentheses
14+
NUM_ARTIFACTS=$(ls -1q "$artifacts_path" | wc -l)
15+
16+
ITER="1"
17+
for artifact in $artifacts; do
18+
ARTIFACT_NAME=$(basename "$artifact")
19+
20+
GATES_INFO=$($BACKEND gates -b "$artifacts_path/$artifact")
21+
MAIN_FUNCTION_INFO=$(echo $GATES_INFO | jq -r '.functions[0] | .name = "main"')
22+
echo "{\"package_name\": \"$ARTIFACT_NAME\", \"functions\": [$MAIN_FUNCTION_INFO]" >> gates_report.json
23+
24+
if (($ITER == $NUM_ARTIFACTS)); then
25+
echo "}" >> gates_report.json
26+
else
27+
echo "}, " >> gates_report.json
28+
fi
29+
30+
ITER=$(( $ITER + 1 ))
31+
done
32+
33+
echo "]}" >> gates_report.json
34+
35+
# Convert the gates report into separate benchmark files
36+
output_file_opcodes="benchmark-opcodes.json"
37+
output_file_circuit="benchmark-circuit.json"
38+
39+
# Convert gates report - opcodes
40+
jq -r '[.programs[] | {
41+
"name": "\(.package_name)/main",
42+
"unit": "acir_opcodes",
43+
"value": (.functions[0].acir_opcodes // 0)
44+
}]' gates_report.json > $output_file_opcodes
45+
46+
# Convert gates report - circuit size
47+
jq -r '[.programs[] | {
48+
"name": "\(.package_name)/main",
49+
"unit": "circuit_size",
50+
"value": (.functions[0].circuit_size // 0)
51+
}]' gates_report.json > $output_file_circuit

‎src/benchmarks/bigcurve_benchmarks.nr‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
use crate::curves::bn254::{BN254};
2+
use crate::curves::secp256k1::{Secp256k1};
3+
comptime fn make_bench(m: Module, params: Quoted) -> Quoted {
4+
let module_name = m.name();
5+
let add_bench_name = f"add_{module_name}".quoted_contents();
6+
let sub_bench_name = f"sub_{module_name}".quoted_contents();
7+
let one_bench_name = f"one_{module_name}".quoted_contents();
8+
let BigCurve = quote { crate::BigCurve };
9+
10+
let from_field_bench_name = f"from_field_{module_name}".quoted_contents();
11+
let typ = params.as_type();
12+
13+
quote {
14+
#[export]
15+
fn $add_bench_name(a: $typ, b: $typ) -> $typ {
16+
a.add(b)
17+
}
18+
19+
#[export]
20+
fn $sub_bench_name(a: $typ, b: $typ) -> $typ {
21+
a.sub(b)
22+
}
23+
24+
#[export]
25+
fn $one_bench_name() -> $typ {
26+
$typ::one()
27+
}
28+
29+
}
30+
}
31+
32+
#[make_bench(quote { BN254 })]
33+
mod BN254_Bench {}
34+
35+
#[make_bench(quote { Secp256k1 })]
36+
mod Secp256k1_Bench {}

‎src/benchmarks/mod.nr‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
mod bigcurve_benchmarks;

‎src/lib.nr‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
pub mod scalar_field;
1+
22
pub(crate) mod curve_jac;
33
mod test_data;
44
mod bigcurve_test;
5+
pub mod scalar_field;
6+
mod benchmarks;
57
pub(crate) mod utils;
68
pub(crate) mod curves;
79

0 commit comments

Comments
 (0)