feat(bench): add managed nightly RPC benchmarks

.dockerignore

@@ -11,6 +11,9 @@ coverage/
 **/target/debug
 **/target/release
 
+bench/rpc/corpus/*
+!bench/rpc/corpus/*.json
+
 # GitHub metadata 
 .github/
 
@@ -23,4 +26,4 @@ CODE_OF_CONDUCT.md
 CONTRIBUTING.md
 
 # Scripts
-scripts/
+scripts/

.github/workflows/build-image.yaml

@@ -62,6 +62,6 @@ jobs:
           context: .
           push: true
           platforms: linux/amd64
-          tags: ${{ env.DOCKER_REGISTRY }}/${{ env.REPO_DEV }}/juno:${{ steps.set_tag.outputs.DOCKER_IMAGE_TAG  }}
+          tags: ${{ env.DOCKER_REGISTRY }}/${{ env.REPO_DEV }}/juno:${{ steps.set_tag.outputs.DOCKER_IMAGE_TAG }}${{ github.ref == 'refs/heads/main' && format(',{0}/{1}/juno:main', env.DOCKER_REGISTRY, env.REPO_DEV) || '' }}
           cache-from: type=gha,scope=${{ github.workflow }}
           cache-to: type=gha,mode=max,scope=${{ github.workflow }}

.github/workflows/rpc-benchmark.yaml

@@ -0,0 +1,68 @@
+name: RPC Benchmark Image
+
+on:
+  pull_request:
+    paths:
+      - "bench/rpc/**"
+      - ".github/workflows/rpc-benchmark.yaml"
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+env:
+  DOCKER_REGISTRY: nethermind.jfrog.io
+  BENCHMARK_IMAGE_REPOSITORY: nubia-oci-local-dev/juno-rpc-benchmark
+
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Validate nightly catalog
+        run: bench/rpc/nightly/test/test-nightly.sh
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      - name: Install JFrog CLI
+        if: github.ref == 'refs/heads/main'
+        id: jfrog
+        uses: jfrog/setup-jfrog-cli@deda456d982fc5e9a7a020b63eb0d2968aedd33e # v5.1.0
+        env:
+          JF_URL: https://${{ env.DOCKER_REGISTRY }}
+        with:
+          oidc-provider-name: github-nethermindeth
+
+      - name: Login to registry with OIDC
+        if: github.ref == 'refs/heads/main'
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ${{ env.DOCKER_REGISTRY }}
+          username: ${{ steps.jfrog.outputs.oidc-user }}
+          password: ${{ steps.jfrog.outputs.oidc-token }}
+
+      - name: Build benchmark image
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.0.0
+        with:
+          context: .
+          file: bench/rpc/nightly/Dockerfile
+          tags: |
+            ${{ env.DOCKER_REGISTRY }}/${{ env.BENCHMARK_IMAGE_REPOSITORY }}:latest
+            ${{ env.DOCKER_REGISTRY }}/${{ env.BENCHMARK_IMAGE_REPOSITORY }}:sha-${{ github.sha }}
+          cache-from: type=gha,scope=rpc-benchmark
+          cache-to: type=gha,mode=max,scope=rpc-benchmark
+          push: ${{ github.ref == 'refs/heads/main' }}

bench/rpc/corpus/nightly.json

@@ -0,0 +1,29 @@
+{
+  "blockHashAndNumber": "blockHashAndNumber --count=100000 --seed=1",
+  "blockNumber": "blockNumber --count=100000 --seed=1",
+  "chainId": "chainId --count=100000 --seed=1",
+  "syncing": "syncing --count=100000 --seed=1",
+
+  "getBlockTransactionCount": "getBlockTransactionCount --count=100000 --seed=1",
+  "getBlockWithReceipts": "getBlockWithReceipts --count=100000 --seed=1",
+  "getBlockWithTxHashes": "getBlockWithTxHashes --count=100000 --seed=1",
+  "getBlockWithTxs": "getBlockWithTxs --count=100000 --seed=1",
+  "getStateUpdate": "getStateUpdate --count=100000 --seed=1",
+  "traceBlockTransactions": "traceBlockTransactions --count=10000 --seed=1",
+
+  "getTransactionByBlockIdAndIndex": "getTransactionByBlockIdAndIndex --count=100000 --seed=1",
+  "getTransactionByHash": "getTransactionByHash --count=100000 --seed=1",
+  "getTransactionReceipt": "getTransactionReceipt --count=100000 --seed=1",
+  "getTransactionStatus": "getTransactionStatus --count=100000 --seed=1",
+  "traceTransaction": "traceTransaction --count=10000 --seed=1",
+
+  "getClass": "getClass --count=100000 --seed=1",
+  "getClassAt": "getClassAt --count=100000 --seed=1",
+  "getClassHashAt": "getClassHashAt --count=100000 --seed=1",
+  "getCompiledCasm": "getCompiledCasm --count=100000 --seed=1",
+  "getNonce": "getNonce --count=100000 --seed=1",
+  "getStorageAt": "getStorageAt --count=100000 --seed=1",
+
+  "getEvents": "getEvents --count=100000 --seed=1",
+  "getStorageProof": "getStorageProof --count=100000 --seed=1"
+}

bench/rpc/nightly/Dockerfile

@@ -0,0 +1,29 @@
+# syntax=docker/dockerfile:1.7.0
+
+FROM golang:1.26-bookworm AS builder
+
+WORKDIR /src
+COPY go.mod go.sum ./
+COPY starknet-p2p-specs/go.mod starknet-p2p-specs/go.mod
+RUN go mod download
+
+COPY Makefile ./
+COPY bench/rpc/cmd/corpus-gen/ bench/rpc/cmd/corpus-gen/
+RUN CGO_ENABLED=0 make corpus-gen
+
+FROM docker.io/grafana/k6:1.7.1@sha256:4fd3a694926b064d3491d9b02b01cde886583c4931f1223816e3d9a7bdfa7e0f
+
+USER root
+RUN apk add --no-cache bash curl jq && \
+    mkdir -p /corpus /results && \
+    chown k6:k6 /corpus /results
+
+WORKDIR /app
+COPY --from=builder /src/build/corpus-gen build/corpus-gen
+COPY bench/rpc/nightly/nightly.sh bench/rpc/nightly/ci.json bench/rpc/nightly/
+COPY bench/rpc/nightly/internal/ bench/rpc/nightly/internal/
+COPY bench/rpc/run.js bench/rpc/run.js
+COPY bench/rpc/corpus/nightly.json bench/rpc/corpus/nightly.json
+
+USER k6
+ENTRYPOINT ["/app/bench/rpc/nightly/nightly.sh"]

bench/rpc/nightly/ci.json

@@ -0,0 +1,9 @@
+{
+  "vus": 50,
+  "duration": "30s",
+  "summaryTrendStats": ["avg", "min", "med", "p(90)", "p(95)", "p(99)", "max"],
+  "thresholds": {
+    "checks": ["rate==1"],
+    "http_req_failed": ["rate==0"]
+  }
+}

bench/rpc/nightly/internal/corpus.sh

@@ -0,0 +1,73 @@
+generate_case() (
+  set -euo pipefail
+  local name=$1 command=$2 work case_dir
+  local -a sub
+
+  case_dir="$CORPUS_ROOT/$name"
+  work=$(mktemp -d "$CORPUS_ROOT/.staging/$name.XXXXXX")
+  trap 'rm -rf "$work"' EXIT INT TERM
+
+  read -ra sub <<<"$command"
+  "$CORPUS_GEN" --source-url "$NODE_URL" \
+    --concurrency="${CONCURRENCY:-32}" \
+    "${sub[@]}" >"$work/corpus.json" || return 1
+  printf '%s\n' "$command" >"$work/command"
+  [[ ! -e $case_dir ]] || {
+    echo "error: case appeared during generation: $name" >&2
+    return 1
+  }
+  mv -T "$work" "$case_dir"
+  work=
+  trap - EXIT INT TERM
+)
+
+corpus_is_current() {
+  local case_dir=$1 command=$2
+  [[ -f $case_dir/command ]] &&
+    [[ $(<"$case_dir/command") == "$command" ]]
+}
+
+prune_removed_cases() {
+  local case_dir name
+  for case_dir in "$CORPUS_ROOT"/*; do
+    [[ -d $case_dir ]] || continue
+    name=${case_dir##*/}
+    if ! jq -e --arg name "$name" 'has($name)' "$CATALOG" >/dev/null; then
+      echo "removing $name"
+      rm -rf -- "$case_dir"
+    fi
+  done
+}
+
+generate() {
+  [[ -x $CORPUS_GEN ]] || {
+    echo "error: $CORPUS_GEN not found; run 'make corpus-gen' first" >&2
+    return 1
+  }
+  mkdir -p "$CORPUS_ROOT/.staging"
+
+  local name command case_dir
+  local -a failed=()
+  while IFS=$'\t' read -r name command; do
+    echo "==> $name"
+    case_dir="$CORPUS_ROOT/$name"
+    if [[ -d $case_dir ]]; then
+      if corpus_is_current "$case_dir" "$command"; then
+        echo "reusing $name"
+        continue
+      fi
+      echo "removing outdated $name"
+      rm -rf -- "$case_dir"
+    fi
+    if ! generate_case "$name" "$command"; then
+      failed+=("$name")
+    fi
+  done < <(jq -r 'to_entries[] | [.key, .value] | @tsv' "$CATALOG")
+
+  if ((${#failed[@]} > 0)); then
+    echo "failed: ${failed[*]}" >&2
+    return 1
+  fi
+  prune_removed_cases
+  echo "corpora written to $CORPUS_ROOT"
+}

bench/rpc/nightly/internal/preflight.sh

@@ -0,0 +1,14 @@
+rpc_result() {
+  local method=$1
+  curl --fail --silent --show-error \
+    --connect-timeout 5 --max-time "${RPC_TIMEOUT:-30}" \
+    --header 'Content-Type: application/json' \
+    --data "$(jq -cn --arg method "$method" \
+      '{jsonrpc:"2.0",id:1,method:$method,params:[]}')" \
+    "$NODE_URL" |
+    jq -er 'if .error then error(.error.message // "JSON-RPC error") else .result end'
+}
+
+preflight() {
+  rpc_result starknet_chainId >/dev/null
+}

bench/rpc/nightly/nightly.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Stable entrypoint for the managed nightly RPC benchmark.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RPC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+CATALOG="$RPC_DIR/corpus/nightly.json"
+CI_CONFIG="$SCRIPT_DIR/ci.json"
+CORPUS_GEN=${CORPUS_GEN:-"$RPC_DIR/../../build/corpus-gen"}
+RUN_SCRIPT="$RPC_DIR/run.js"
+
+: "${NODE_URL:?NODE_URL is required}"
+: "${SNAPSHOT_SHA256:?SNAPSHOT_SHA256 is required}"
+[[ $SNAPSHOT_SHA256 =~ ^[0-9a-f]{64}$ ]] || {
+  echo "error: SNAPSHOT_SHA256 must be a lowercase SHA-256 digest" >&2
+  exit 1
+}
+CORPUS_ROOT="${CORPUS_ROOT:-/corpus/cases}/$SNAPSHOT_SHA256"
+
+source "$SCRIPT_DIR/internal/preflight.sh"
+source "$SCRIPT_DIR/internal/corpus.sh"
+
+case_path() {
+  printf '%s/%s/corpus.json\n' "$CORPUS_ROOT" "$1"
+}
+
+benchmark() {
+  [[ -f $CI_CONFIG ]] || {
+    echo "error: k6 config not found: $CI_CONFIG" >&2
+    return 1
+  }
+  local out_dir=${RESULTS_DIR:-/results}
+  local warmup_iterations=${WARMUP_ITERATIONS:-200}
+  mkdir -p "$out_dir"
+
+  local name corpus summary case_failed
+  local -a failed=()
+  while IFS= read -r name; do
+    summary="$out_dir/$name.json"
+
+    echo
+    echo "==> $name"
+    corpus=$(case_path "$name")
+    if [[ ! -f $corpus ]]; then
+      echo "error: missing corpus for $name: $corpus" >&2
+      failed+=("$name")
+      continue
+    fi
+    corpus="$(cd "$(dirname "$corpus")" && pwd)/$(basename "$corpus")"
+
+    case_failed=0
+    if [[ $warmup_iterations != 0 ]]; then
+      echo "warming up"
+      (
+        K6_WEB_DASHBOARD=false k6 run \
+          --vus 1 \
+          --iterations "$warmup_iterations" \
+          --tag "case_id=$name" \
+          --tag phase=warmup \
+          -e NODE_URL="$NODE_URL" \
+          "$RUN_SCRIPT" <"$corpus" >/dev/null
+      ) || {
+        echo "error: warmup did not complete for $name" >&2
+        case_failed=1
+      }
+    fi
+
+    if ((case_failed == 0)); then
+      k6 run \
+        --config "$CI_CONFIG" \
+        --tag "case_id=$name" \
+        --tag phase=measure \
+        --summary-export "$summary" \
+        -e NODE_URL="$NODE_URL" \
+        "$RUN_SCRIPT" <"$corpus" || case_failed=1
+    fi
+    ((case_failed == 0)) || failed+=("$name")
+  done < <(jq -r 'keys_unsorted[]' "$CATALOG")
+
+  if ((${#failed[@]} > 0)); then
+    echo "failed: ${failed[*]}" >&2
+    return 1
+  fi
+}
+
+nightly_run() {
+  preflight
+
+  local failed=0
+  generate || failed=1
+  benchmark || failed=1
+  return "$failed"
+}
+
+nightly_run

bench/rpc/nightly/test/test-nightly.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -eu
+
+repo_root=$(CDPATH= cd -- "$(dirname "$0")/../../../.." && pwd)
+catalog=${repo_root}/bench/rpc/corpus/nightly.json
+
+# Validate filesystem-safe case definitions.
+jq -e '
+  type == "object" and length > 0 and
+  all(to_entries[];
+    (.key | test("^[A-Za-z0-9][A-Za-z0-9._-]*$")) and
+    (.value | type == "string" and test("[^[:space:]]")))
+' "${catalog}" >/dev/null
+
+# Reject duplicate case IDs.
+test -z "$(jq --stream -r '
+  select(length == 2 and (.[0] | length) == 1) | .[0][0]
+' "${catalog}" | sort | uniq -d)"
+
+echo "nightly catalog validation passed"