github action benchmark

Author: jialinli98

.github/workflows/benchmark.yml

@@ -34,19 +34,63 @@ jobs:
       - name: Generate gates report
         run: ./scripts/build-gates-report.sh
         env:
-          BACKEND: /home/runner/.bb/bb
+            BACKEND: /home/runner/.bb/bb
 
-      - name: Compare gates reports
-        id: gates_diff
-        uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1
+      - name: Store ACIR opcode benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
         with:
-          report: gates_report.json
-          summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%)
+            name: "ACIR Opcodes"
+            tool: "customSmallerIsBetter"
+            output-file-path: "benchmark-opcodes.json"
+            gh-pages-branch: "gh-pages"
+            benchmark-data-dir-path: "dev/bench"
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            auto-push: ${{ github.ref == 'refs/heads/main' }}
+            comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
+            comment-on-alert: true
+            alert-threshold: "101%"
+            fail-on-alert: false
+            max-items-in-chart: 50
 
-      - name: Add gates diff to sticky comment
-        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
-        uses: marocchino/sticky-pull-request-comment@v2
+      - name: Store gates benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
         with:
-          # delete the comment in case changes no longer impact circuit sizes
-          delete: ${{ !steps.gates_diff.outputs.markdown }}
-          message: ${{ steps.gates_diff.outputs.markdown }}
+            name: "Circuit Size"
+            tool: "customSmallerIsBetter"
+            output-file-path: "benchmark-circuit.json"
+            gh-pages-branch: "gh-pages"
+            benchmark-data-dir-path: "dev/bench"
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            auto-push: ${{ github.ref == 'refs/heads/main' }}
+            comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
+            comment-on-alert: true
+            alert-threshold: "101%"
+            fail-on-alert: false
+            max-items-in-chart: 50
+            skip-fetch-gh-pages: true
+
+      - name: Delete export files
+        run: rm -rf export
+
+      - name: Build Brillig benchmark programs
+        run: nargo export --force-brillig
+
+      - name: Generate brillig report
+        run: ./ethereum/scripts/build-brillig-report.sh
+
+      - name: Store brillig benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+            name: "Brillig Bytecode Size"
+            tool: "customSmallerIsBetter"
+            output-file-path: "benchmark-brillig.json"
+            gh-pages-branch: "gh-pages"
+            benchmark-data-dir-path: "dev/bench"
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            auto-push: ${{ github.ref == 'refs/heads/main' }}
+            comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
+            comment-on-alert: true
+            alert-threshold: "101%"
+            fail-on-alert: false
+            max-items-in-chart: 50
+            skip-fetch-gh-pages: true

scripts/build-brillig-report.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -e
+
+INSPECTOR=${INSPECTOR:-noir-inspector}
+cd $(dirname "$0")/../
+
+artifacts_path="./export"
+artifacts=$(ls $artifacts_path)
+
+# Start the JSON array
+REPORTS=$(jq --null-input '[]')
+
+for artifact in $artifacts; do    
+    # Get and format the opcode info
+    OP_CODE_INFO=$($INSPECTOR info --json "$artifacts_path/$artifact")
+
+    # Simplified jq expression to output only package_name and opcodes from unconstrained_functions
+    REPORTS=$(echo $OP_CODE_INFO | jq -c '.programs[0] | del(.functions)' | jq -c --argjson old_reports $REPORTS '$old_reports + [.]')
+done
+
+echo $REPORTS | jq '{ programs: . }' > brillig_report.json
+
+# Convert brillig report to benchmark format
+output_file_brillig="benchmark-brillig.json"
+jq -r '[.programs[] | .unconstrained_functions[] | {
+    "name": (if (.package_name // "") == "" then .name else "\(.package_name | sub("^null/"; ""))/\(.name)" end),
+    "unit": "opcodes",
+    "value": (.opcodes // 0)
+}]' brillig_report.json > $output_file_brillig

scripts/build-gates-report.sh

@@ -32,4 +32,21 @@ done
 
 echo "]}" >> gates_report.json 
 
+# Convert the gates report into separate benchmark files
+output_file_opcodes="benchmark-opcodes.json"
+output_file_circuit="benchmark-circuit.json"
+
+# Convert gates report - opcodes
+jq -r '[.programs[] | {
+    "name": "\(.package_name)/main",
+    "unit": "acir_opcodes",
+    "value": (.functions[0].acir_opcodes // 0)
+}]' gates_report.json > $output_file_opcodes
+
+# Convert gates report - circuit size
+jq -r '[.programs[] | {
+    "name": "\(.package_name)/main",
+    "unit": "circuit_size",
+    "value": (.functions[0].circuit_size // 0)
+}]' gates_report.json > $output_file_circuit