Skip to content

Commit 8fe9a36

Browse files
jialinli98jialinli98
committed
benchmark
1 parent 340ff96 commit 8fe9a36

File tree

4 files changed

+182
-0
lines changed

4 files changed

+182
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
9+
jobs:
10+
test:
11+
name: Benchmark library
12+
runs-on: ubuntu-latest
13+
permissions:
14+
contents: write
15+
pull-requests: write
16+
steps:
17+
- name: Checkout sources
18+
uses: actions/checkout@v4
19+
20+
- name: Install Nargo
21+
uses: noir-lang/[email protected]
22+
with:
23+
toolchain: 1.0.0-beta.17
24+
25+
- name: Install bb
26+
run: |
27+
curl -L https://raw.githubusercontent.com/AztecProtocol/aztec-packages/refs/heads/next/barretenberg/bbup/install | bash
28+
~/.bb/bbup -nv 1.0.0-beta.12
29+
sudo apt install libc++-dev
30+
31+
- name: Build Noir benchmark programs
32+
run: nargo export
33+
34+
- name: Generate gates report
35+
run: ./scripts/build-gates-report.sh
36+
env:
37+
BACKEND: /home/runner/.bb/bb
38+
39+
- name: Store ACIR opcode benchmark result
40+
uses: benchmark-action/github-action-benchmark@v1
41+
with:
42+
name: "ACIR Opcodes"
43+
tool: "customSmallerIsBetter"
44+
output-file-path: "benchmark-opcodes.json"
45+
gh-pages-branch: "gh-pages"
46+
benchmark-data-dir-path: "dev/bench"
47+
github-token: ${{ secrets.GITHUB_TOKEN }}
48+
auto-push: ${{ github.ref == 'refs/heads/main' }}
49+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
50+
comment-on-alert: true
51+
alert-threshold: "101%"
52+
fail-on-alert: false
53+
max-items-in-chart: 50
54+
55+
- name: Store gates benchmark result
56+
uses: benchmark-action/github-action-benchmark@v1
57+
with:
58+
name: "Circuit Size"
59+
tool: "customSmallerIsBetter"
60+
output-file-path: "benchmark-circuit.json"
61+
gh-pages-branch: "gh-pages"
62+
benchmark-data-dir-path: "dev/bench"
63+
github-token: ${{ secrets.GITHUB_TOKEN }}
64+
auto-push: ${{ github.ref == 'refs/heads/master' }}
65+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
66+
comment-on-alert: true
67+
alert-threshold: "101%"
68+
fail-on-alert: false
69+
max-items-in-chart: 50
70+
skip-fetch-gh-pages: true
71+
72+
- name: Delete export files
73+
run: rm -rf export
74+
75+
- name: Build Brillig benchmark programs
76+
run: nargo export --force-brillig
77+
78+
- name: Generate brillig report
79+
run: ./scripts/build-brillig-report.sh
80+
81+
- name: Store brillig benchmark result
82+
uses: benchmark-action/github-action-benchmark@v1
83+
with:
84+
name: "Brillig Bytecode Size"
85+
tool: "customSmallerIsBetter"
86+
output-file-path: "benchmark-brillig.json"
87+
gh-pages-branch: "gh-pages"
88+
benchmark-data-dir-path: "dev/bench"
89+
github-token: ${{ secrets.GITHUB_TOKEN }}
90+
auto-push: ${{ github.ref == 'refs/heads/master' }}
91+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
92+
comment-on-alert: true
93+
alert-threshold: "101%"
94+
fail-on-alert: false
95+
max-items-in-chart: 50
96+
skip-fetch-gh-pages: true

‎ethereum/circuits/lib/benchmark.nr‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
mod benchmark {
2+
#[export]
3+
pub fn benchmark_get_account(chain_id: u32, block_no: u64, address: Address) {
4+
get_account(chain_id, block_no, address)
5+
}
6+
}

‎ethereum/scripts/build-brillig-report.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
INSPECTOR=${INSPECTOR:-noir-inspector}
5+
cd $(dirname "$0")/../
6+
7+
artifacts_path="./export"
8+
artifacts=$(ls $artifacts_path)
9+
10+
# Start the JSON array
11+
REPORTS=$(jq --null-input '[]')
12+
13+
for artifact in $artifacts; do
14+
# Get and format the opcode info
15+
OP_CODE_INFO=$($INSPECTOR info --json "$artifacts_path/$artifact")
16+
17+
# Simplified jq expression to output only package_name and opcodes from unconstrained_functions
18+
REPORTS=$(echo $OP_CODE_INFO | jq -c '.programs[0] | del(.functions)' | jq -c --argjson old_reports $REPORTS '$old_reports + [.]')
19+
done
20+
21+
echo $REPORTS | jq '{ programs: . }' > brillig_report.json
22+
23+
# Convert brillig report to benchmark format
24+
output_file_brillig="benchmark-brillig.json"
25+
jq -r '[.programs[] | .unconstrained_functions[] | {
26+
"name": (if (.package_name // "") == "" then .name else "\(.package_name | sub("^null/"; ""))/\(.name)" end),
27+
"unit": "opcodes",
28+
"value": (.opcodes // 0)
29+
}]' brillig_report.json > $output_file_brillig

‎ethereum/scripts/build-gates-report.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
BACKEND=${BACKEND:-bb}
5+
6+
cd $(dirname "$0")/../
7+
8+
artifacts_path="./export"
9+
artifacts=$(ls $artifacts_path)
10+
11+
echo "{\"programs\": [" > gates_report.json
12+
13+
# Bound for checking where to place last parentheses
14+
NUM_ARTIFACTS=$(ls -1q "$artifacts_path" | wc -l)
15+
16+
ITER="1"
17+
for artifact in $artifacts; do
18+
ARTIFACT_NAME=$(basename "$artifact")
19+
20+
GATES_INFO=$($BACKEND gates -b "$artifacts_path/$artifact")
21+
MAIN_FUNCTION_INFO=$(echo $GATES_INFO | jq -r '.functions[0] | .name = "main"')
22+
echo "{\"package_name\": \"$ARTIFACT_NAME\", \"functions\": [$MAIN_FUNCTION_INFO]" >> gates_report.json
23+
24+
if (($ITER == $NUM_ARTIFACTS)); then
25+
echo "}" >> gates_report.json
26+
else
27+
echo "}, " >> gates_report.json
28+
fi
29+
30+
ITER=$(( $ITER + 1 ))
31+
done
32+
33+
echo "]}" >> gates_report.json
34+
35+
# Convert the gates report into separate benchmark files
36+
output_file_opcodes="benchmark-opcodes.json"
37+
output_file_circuit="benchmark-circuit.json"
38+
39+
# Convert gates report - opcodes
40+
jq -r '[.programs[] | {
41+
"name": "\(.package_name)/main",
42+
"unit": "acir_opcodes",
43+
"value": (.functions[0].acir_opcodes // 0)
44+
}]' gates_report.json > $output_file_opcodes
45+
46+
# Convert gates report - circuit size
47+
jq -r '[.programs[] | {
48+
"name": "\(.package_name)/main",
49+
"unit": "circuit_size",
50+
"value": (.functions[0].circuit_size // 0)
51+
}]' gates_report.json > $output_file_circuit

0 commit comments

Comments
 (0)