Skip to content

Commit 54de613

Browse files
jialinli98TomAFrench
jialinli98
and
TomAFrench
authored
feat: add benchmarks (#87)
Co-authored-by: Tom French <[email protected]>
1 parent 22e4e0f commit 54de613

File tree

8 files changed

+292
-3
lines changed

8 files changed

+292
-3
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
9+
jobs:
10+
test:
11+
name: Benchmark library
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout sources
15+
uses: actions/checkout@v4
16+
17+
- name: Install Nargo
18+
uses: noir-lang/[email protected]
19+
with:
20+
toolchain: 1.0.0-beta.7
21+
22+
- name: Install bb
23+
run: |
24+
curl -L https://bbup.aztec.network | bash
25+
~/.bb/bbup -nv 1.0.0-beta.7
26+
sudo apt install libc++-dev
27+
28+
- name: Build Noir benchmark programs
29+
run: nargo export
30+
31+
- name: Generate gates report
32+
run: ./scripts/build-gates-report.sh
33+
env:
34+
BACKEND: /home/runner/.bb/bb
35+
36+
- name: Store ACIR opcode benchmark result
37+
uses: benchmark-action/github-action-benchmark@v1
38+
with:
39+
name: "ACIR Opcodes"
40+
tool: "customSmallerIsBetter"
41+
output-file-path: "benchmark-opcodes.json"
42+
gh-pages-branch: "gh-pages"
43+
benchmark-data-dir-path: "dev/bench"
44+
github-token: ${{ secrets.GITHUB_TOKEN }}
45+
auto-push: ${{ github.ref == 'refs/heads/main' }}
46+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
47+
comment-on-alert: true
48+
alert-threshold: "101%"
49+
fail-on-alert: false
50+
max-items-in-chart: 50
51+
52+
- name: Store gates benchmark result
53+
uses: benchmark-action/github-action-benchmark@v1
54+
with:
55+
name: "Circuit Size"
56+
tool: "customSmallerIsBetter"
57+
output-file-path: "benchmark-circuit.json"
58+
gh-pages-branch: "gh-pages"
59+
benchmark-data-dir-path: "dev/bench"
60+
github-token: ${{ secrets.GITHUB_TOKEN }}
61+
auto-push: ${{ github.ref == 'refs/heads/main' }}
62+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
63+
comment-on-alert: true
64+
alert-threshold: "101%"
65+
fail-on-alert: false
66+
max-items-in-chart: 50
67+
skip-fetch-gh-pages: true
68+
69+
- name: Add gates diff to sticky comment
70+
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
71+
uses: marocchino/sticky-pull-request-comment@v2
72+
with:
73+
# delete the comment in case changes no longer impact circuit sizes
74+
delete: ${{ !steps.gates_diff.outputs.markdown }}
75+
message: ${{ steps.gates_diff.outputs.markdown }}
76+
77+
# Delete the export files
78+
- name: Delete export files
79+
run: rm -rf export
80+
# Run nargo export again with force-brillig flag
81+
- name: Build Brillig benchmark programs
82+
run: nargo export --force-brillig
83+
84+
- name: Generate brillig report
85+
run: ./scripts/build-brillig-report.sh
86+
87+
- name: Store brillig benchmark result
88+
uses: benchmark-action/github-action-benchmark@v1
89+
with:
90+
name: "Brillig Bytecode Size"
91+
tool: "customSmallerIsBetter"
92+
output-file-path: "benchmark-brillig.json"
93+
gh-pages-branch: "gh-pages"
94+
benchmark-data-dir-path: "dev/bench"
95+
github-token: ${{ secrets.GITHUB_TOKEN }}
96+
auto-push: ${{ github.ref == 'refs/heads/main' }}
97+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
98+
comment-on-alert: true
99+
alert-threshold: "101%"
100+
fail-on-alert: false
101+
max-items-in-chart: 50
102+
skip-fetch-gh-pages: true
103+
104+
- name: Add brillig diff to sticky comment
105+
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
106+
uses: marocchino/sticky-pull-request-comment@v2
107+
with:
108+
# delete the comment in case changes no longer impact circuit sizes
109+
delete: ${{ !steps.brillig_diff.outputs.markdown }}
110+
message: ${{ steps.brillig_diff.outputs.markdown }}

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ The most efficient method to evaluate curve operations is `BigCurve::evaluate_li
2929
- Add method to check point is in prime-order subgroup for curves with a cofactor
3030
- Parametrise and test with a degree-2 extension field instead of `BigNum`
3131
- [x] Add curve parameters for commonly used curves (BN254, BLS12-381, MNT4, MNT6, Pasta, Vella, Secp256K1, Secp256R1)
32-
- Create benchmarks
3332
- Add support for curve endomorphisms where applicable (if base field and scalar field both contain cube roots of unity, we can reduce the number of point doublings required for an MSM in half)
3433

3534
# FAQ

‎scripts/build-brillig-report.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
INSPECTOR=${INSPECTOR:-noir-inspector}
5+
cd $(dirname "$0")/../
6+
7+
artifacts_path="./export"
8+
artifacts=$(ls $artifacts_path)
9+
10+
# Start the JSON array
11+
REPORTS=$(jq --null-input '[]')
12+
13+
for artifact in $artifacts; do
14+
# Get and format the opcode info
15+
OP_CODE_INFO=$($INSPECTOR info --json "$artifacts_path/$artifact")
16+
17+
# Simplified jq expression to output only package_name and opcodes from unconstrained_functions
18+
REPORTS=$(echo $OP_CODE_INFO | jq -c '.programs[0] | del(.functions)' | jq -c --argjson old_reports $REPORTS '$old_reports + [.]')
19+
done
20+
21+
echo $REPORTS | jq '{ programs: . }' > brillig_report.json
22+
23+
# Convert brillig report to benchmark format
24+
output_file_brillig="benchmark-brillig.json"
25+
jq -r '[.programs[] | .unconstrained_functions[] | {
26+
"name": (if (.package_name // "") == "" then .name else "\(.package_name | sub("^null/"; ""))/\(.name)" end),
27+
"unit": "opcodes",
28+
"value": (.opcodes // 0)
29+
}]' brillig_report.json > $output_file_brillig

‎scripts/build-gates-report.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
BACKEND=${BACKEND:-bb}
5+
6+
cd $(dirname "$0")/../
7+
8+
artifacts_path="./export"
9+
artifacts=$(ls $artifacts_path)
10+
11+
echo "{\"programs\": [" > gates_report.json
12+
13+
# Bound for checking where to place last parentheses
14+
NUM_ARTIFACTS=$(ls -1q "$artifacts_path" | wc -l)
15+
16+
ITER="1"
17+
for artifact in $artifacts; do
18+
ARTIFACT_NAME=$(basename "$artifact")
19+
20+
GATES_INFO=$($BACKEND gates -b "$artifacts_path/$artifact")
21+
MAIN_FUNCTION_INFO=$(echo $GATES_INFO | jq -r '.functions[0] | .name = "main"')
22+
echo "{\"package_name\": \"$ARTIFACT_NAME\", \"functions\": [$MAIN_FUNCTION_INFO]" >> gates_report.json
23+
24+
if (($ITER == $NUM_ARTIFACTS)); then
25+
echo "}" >> gates_report.json
26+
else
27+
echo "}, " >> gates_report.json
28+
fi
29+
30+
ITER=$(( $ITER + 1 ))
31+
done
32+
33+
echo "]}" >> gates_report.json
34+
35+
# Convert the gates report into separate benchmark files
36+
output_file_opcodes="benchmark-opcodes.json"
37+
output_file_circuit="benchmark-circuit.json"
38+
39+
# Convert gates report - opcodes
40+
jq -r '[.programs[] | {
41+
"name": "\(.package_name)/main",
42+
"unit": "acir_opcodes",
43+
"value": (.functions[0].acir_opcodes // 0)
44+
}]' gates_report.json > $output_file_opcodes
45+
46+
# Convert gates report - circuit size
47+
jq -r '[.programs[] | {
48+
"name": "\(.package_name)/main",
49+
"unit": "circuit_size",
50+
"value": (.functions[0].circuit_size // 0)
51+
}]' gates_report.json > $output_file_circuit

‎src/benchmarks/bigcurve_benchmarks.nr‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
use crate::curves::bls12_377::BLS12_377;
2+
use crate::curves::bls12_381::BLS12_381;
3+
use crate::curves::bn254::BN254;
4+
use crate::curves::mnt4_753::MNT4_753;
5+
use crate::curves::mnt6_753::MNT6_753;
6+
use crate::curves::pallas::Pallas;
7+
use crate::curves::secp256k1::Secp256k1;
8+
use crate::curves::secp256r1::Secp256r1;
9+
use crate::curves::secp384r1::Secp384r1;
10+
use crate::curves::vesta::Vesta;
11+
comptime fn make_bench(m: Module, params: Quoted) -> Quoted {
12+
let module_name = m.name();
13+
let add_bench_name = f"add_{module_name}".quoted_contents();
14+
let sub_bench_name = f"sub_{module_name}".quoted_contents();
15+
let neg_bench_name = f"neg_{module_name}".quoted_contents();
16+
let mul_bench_name = f"mul_{module_name}".quoted_contents();
17+
let eq_bench_name = f"eq_{module_name}".quoted_contents();
18+
let validate_on_curve_bench_name = f"validate_on_curve_{module_name}".quoted_contents();
19+
let evaluate_linear_expression_bench_name =
20+
f"evaluate_linear_expression_{module_name}".quoted_contents();
21+
let hash_to_curve_bench_name = f"hash_to_curve_{module_name}".quoted_contents();
22+
let BigCurve = quote { crate::BigCurve };
23+
let ScalarField = quote { crate::ScalarField };
24+
25+
let typ = params.as_type();
26+
27+
quote {
28+
#[export]
29+
fn $add_bench_name(a: $typ, b: $typ) -> $typ {
30+
a + b
31+
}
32+
33+
#[export]
34+
fn $sub_bench_name(a: $typ, b: $typ) -> $typ {
35+
a-b
36+
}
37+
38+
#[export]
39+
fn $neg_bench_name(a: $typ) -> $typ {
40+
-a
41+
}
42+
43+
#[export]
44+
fn $eq_bench_name(a: $typ, b: $typ) -> bool {
45+
a==b
46+
}
47+
#[export]
48+
fn $validate_on_curve_bench_name(a: $typ) {
49+
$BigCurve::validate_on_curve(a);
50+
}
51+
52+
#[export]
53+
fn $mul_bench_name(a: $typ, b: $ScalarField<64>) -> $typ {
54+
$BigCurve::mul(a, b)
55+
}
56+
57+
#[export]
58+
fn $hash_to_curve_bench_name(a: [u8; 10]) -> $typ {
59+
$BigCurve::hash_to_curve(a)
60+
}
61+
62+
#[export]
63+
fn $evaluate_linear_expression_bench_name(a: [$typ; 3], b: [$ScalarField<64>; 3], c: [$typ; 2]) -> $typ {
64+
$BigCurve::evaluate_linear_expression(a, b, c)
65+
}
66+
67+
}
68+
}
69+
70+
#[make_bench(quote { BN254 })]
71+
mod BN254_Bench {}
72+
73+
#[make_bench(quote { BLS12_381 })]
74+
mod BLS12_381_Bench {}
75+
76+
#[make_bench(quote { BLS12_377 })]
77+
mod BLS12_377_Bench {}
78+
79+
#[make_bench(quote { MNT4_753 })]
80+
mod MNT4_753_Bench {}
81+
82+
#[make_bench(quote { MNT6_753 })]
83+
mod MNT6_753_Bench {}
84+
85+
#[make_bench(quote { Pallas })]
86+
mod Pallas_Bench {}
87+
88+
#[make_bench(quote { Secp256k1 })]
89+
mod Secp256k1_Bench {}
90+
91+
#[make_bench(quote { Secp256r1 })]
92+
mod Secp256r1_Bench {}
93+
94+
#[make_bench(quote { Secp384r1 })]
95+
mod Secp384r1_Bench {}
96+
97+
#[make_bench(quote { Vesta })]
98+
mod Vesta_Bench {}

‎src/benchmarks/mod.nr‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
mod bigcurve_benchmarks;

‎src/lib.nr‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
pub mod scalar_field;
21
pub(crate) mod curve_jac;
32
mod test_data;
43
mod bigcurve_test;
4+
pub mod scalar_field;
55
pub(crate) mod utils;
66
pub(crate) mod curves;
77

@@ -11,6 +11,7 @@ use bignum::bignum::evaluate_quadratic_expression;
1111
use crate::curve_jac::AffineTranscript;
1212
use crate::scalar_field::ScalarField;
1313
use std::ops::{Add, Neg, Sub};
14+
mod benchmarks;
1415

1516
/// Implements an elliptic curve over a prime field that is not the circuit's native field.
1617

‎src/utils/derive_offset_generators.nr‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ where
102102
#[test]
103103
fn test_compute_and_print_offset_generators() {
104104
unsafe {
105-
let _ = <BN254 as BigCurve<BN254_Fq>>::one();
105+
let _ = BN254::one();
106106
compute_and_print_offset_generators::<BN254_Fq, _, _, 64, BN254>("BN254_Fq", "BN254", 1);
107107
// compute_and_print_offset_generators::<PallasBN, _, _, 64>("Pallas_Fq", "Pallas", 1);
108108
// compute_and_print_offset_generators::<VestaBN, _, _, 64>("Vesta_Fq", "Vesta", 1);

0 commit comments

Comments
 (0)