Skip to content

Commit 2ea3475

Browse files
danielntmddanielntmd
committed
feat(bench): add managed nightly RPC benchmarks
Add snapshot-scoped, per-case corpus reconciliation and isolated warmup and measured k6 runs. Publish a dedicated benchmark image and validate its catalog while preserving the existing local tools.
1 parent d5d68c0 commit 2ea3475

9 files changed

Lines changed: 344 additions & 1 deletion

File tree

‎.dockerignore‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ coverage/
1111
**/target/debug
1212
**/target/release
1313

14+
# Generated RPC benchmark corpora (keep the catalogs)
15+
bench/rpc/corpus/*
16+
!bench/rpc/corpus/*.json
17+
1418
# GitHub metadata
1519
.github/
1620

@@ -23,4 +27,4 @@ CODE_OF_CONDUCT.md
2327
CONTRIBUTING.md
2428

2529
# Scripts
26-
scripts/
30+
scripts/

‎.github/workflows/rpc-benchmark.yaml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: RPC Benchmark Image
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "bench/rpc/**"
7+
- ".github/workflows/rpc-benchmark.yaml"
8+
push:
9+
branches:
10+
- main
11+
workflow_dispatch:
12+
13+
permissions:
14+
contents: read
15+
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ github.ref }}
18+
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
19+
20+
env:
21+
DOCKER_REGISTRY: nethermind.jfrog.io
22+
BENCHMARK_IMAGE_REPOSITORY: nubia-oci-local-dev/juno-rpc-benchmark
23+
24+
jobs:
25+
build-image:
26+
runs-on: ubuntu-latest
27+
timeout-minutes: 60
28+
permissions:
29+
contents: read
30+
id-token: write
31+
steps:
32+
- name: Checkout
33+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
34+
35+
- name: Validate nightly catalog
36+
run: bench/rpc/nightly/test/test-nightly.sh
37+
38+
- name: Setup Docker Buildx
39+
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
40+
41+
- name: Install JFrog CLI
42+
if: github.ref == 'refs/heads/main'
43+
id: jfrog
44+
uses: jfrog/setup-jfrog-cli@deda456d982fc5e9a7a020b63eb0d2968aedd33e # v5.1.0
45+
env:
46+
JF_URL: https://${{ env.DOCKER_REGISTRY }}
47+
with:
48+
oidc-provider-name: github-nethermindeth
49+
50+
- name: Login to registry with OIDC
51+
if: github.ref == 'refs/heads/main'
52+
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
53+
with:
54+
registry: ${{ env.DOCKER_REGISTRY }}
55+
username: ${{ steps.jfrog.outputs.oidc-user }}
56+
password: ${{ steps.jfrog.outputs.oidc-token }}
57+
58+
- name: Build benchmark image
59+
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.0.0
60+
with:
61+
context: .
62+
file: bench/rpc/nightly/Dockerfile
63+
tags: |
64+
${{ env.DOCKER_REGISTRY }}/${{ env.BENCHMARK_IMAGE_REPOSITORY }}:latest
65+
${{ env.DOCKER_REGISTRY }}/${{ env.BENCHMARK_IMAGE_REPOSITORY }}:sha-${{ github.sha }}
66+
cache-from: type=gha,scope=rpc-benchmark
67+
cache-to: type=gha,mode=max,scope=rpc-benchmark
68+
push: ${{ github.ref == 'refs/heads/main' }}

‎bench/rpc/corpus/nightly.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"blockHashAndNumber": "blockHashAndNumber --count=100000 --seed=1",
3+
"blockNumber": "blockNumber --count=100000 --seed=1",
4+
"chainId": "chainId --count=100000 --seed=1",
5+
"syncing": "syncing --count=100000 --seed=1",
6+
7+
"getBlockTransactionCount": "getBlockTransactionCount --count=100000 --seed=1",
8+
"getBlockWithReceipts": "getBlockWithReceipts --count=100000 --seed=1",
9+
"getBlockWithTxHashes": "getBlockWithTxHashes --count=100000 --seed=1",
10+
"getBlockWithTxs": "getBlockWithTxs --count=100000 --seed=1",
11+
"getStateUpdate": "getStateUpdate --count=100000 --seed=1",
12+
"traceBlockTransactions": "traceBlockTransactions --count=10000 --seed=1",
13+
14+
"getTransactionByBlockIdAndIndex": "getTransactionByBlockIdAndIndex --count=100000 --seed=1",
15+
"getTransactionByHash": "getTransactionByHash --count=100000 --seed=1",
16+
"getTransactionReceipt": "getTransactionReceipt --count=100000 --seed=1",
17+
"getTransactionStatus": "getTransactionStatus --count=100000 --seed=1",
18+
"traceTransaction": "traceTransaction --count=10000 --seed=1",
19+
20+
"getClass": "getClass --count=100000 --seed=1",
21+
"getClassAt": "getClassAt --count=100000 --seed=1",
22+
"getClassHashAt": "getClassHashAt --count=100000 --seed=1",
23+
"getCompiledCasm": "getCompiledCasm --count=100000 --seed=1",
24+
"getNonce": "getNonce --count=100000 --seed=1",
25+
"getStorageAt": "getStorageAt --count=100000 --seed=1",
26+
27+
"getEvents": "getEvents --count=100000 --seed=1",
28+
"getStorageProof": "getStorageProof --count=100000 --seed=1"
29+
}

‎bench/rpc/nightly/Dockerfile‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# syntax=docker/dockerfile:1.7.0
2+
3+
FROM golang:1.26-bookworm AS builder
4+
5+
WORKDIR /src
6+
COPY go.mod go.sum ./
7+
COPY Makefile ./
8+
COPY starknet-p2p-specs/go.mod starknet-p2p-specs/go.mod
9+
RUN go mod download
10+
11+
COPY bench/rpc/cmd/corpus-gen/ bench/rpc/cmd/corpus-gen/
12+
RUN CGO_ENABLED=0 make corpus-gen
13+
14+
FROM docker.io/grafana/k6:1.7.1@sha256:4fd3a694926b064d3491d9b02b01cde886583c4931f1223816e3d9a7bdfa7e0f
15+
16+
USER root
17+
RUN apk add --no-cache bash curl jq && \
18+
mkdir -p /corpus /results && \
19+
chown k6:k6 /corpus /results
20+
21+
WORKDIR /app
22+
COPY --from=builder /src/build/corpus-gen build/corpus-gen
23+
COPY bench/rpc/nightly/nightly.sh bench/rpc/nightly/ci.json bench/rpc/nightly/
24+
COPY bench/rpc/nightly/internal/ bench/rpc/nightly/internal/
25+
COPY bench/rpc/run.js bench/rpc/run.js
26+
COPY bench/rpc/corpus/nightly.json bench/rpc/corpus/nightly.json
27+
28+
USER k6
29+
ENTRYPOINT ["/app/bench/rpc/nightly/nightly.sh"]

‎bench/rpc/nightly/ci.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"vus": 50,
3+
"duration": "30s",
4+
"summaryTrendStats": ["avg", "min", "med", "p(90)", "p(95)", "p(99)", "max"],
5+
"thresholds": {
6+
"checks": ["rate==1"],
7+
"http_req_failed": ["rate==0"]
8+
}
9+
}

‎bench/rpc/nightly/internal/corpus.sh‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
generate_case() (
2+
set -euo pipefail
3+
local name=$1 command=$2 work case_dir
4+
local -a sub
5+
6+
case_dir="$CORPUS_ROOT/$name"
7+
work=$(mktemp -d "$CORPUS_ROOT/.staging/$name.XXXXXX")
8+
trap 'rm -rf "$work"' EXIT INT TERM
9+
10+
read -ra sub <<<"$command"
11+
"$CORPUS_GEN" --source-url "$NODE_URL" \
12+
--concurrency="${CONCURRENCY:-32}" \
13+
"${sub[@]}" >"$work/corpus.json" || return 1
14+
printf '%s\n' "$command" >"$work/command"
15+
[[ ! -e $case_dir ]] || {
16+
echo "error: case appeared during generation: $name" >&2
17+
return 1
18+
}
19+
mv -T "$work" "$case_dir"
20+
work=
21+
trap - EXIT INT TERM
22+
)
23+
24+
corpus_is_current() {
25+
local case_dir=$1 command=$2
26+
[[ -f $case_dir/command ]] &&
27+
[[ $(<"$case_dir/command") == "$command" ]]
28+
}
29+
30+
prune_removed_cases() {
31+
local case_dir name
32+
for case_dir in "$CORPUS_ROOT"/*; do
33+
[[ -d $case_dir ]] || continue
34+
name=${case_dir##*/}
35+
if ! jq -e --arg name "$name" 'has($name)' "$CATALOG" >/dev/null; then
36+
echo "removing $name"
37+
rm -rf -- "$case_dir"
38+
fi
39+
done
40+
}
41+
42+
generate() {
43+
[[ -x $CORPUS_GEN ]] || {
44+
echo "error: $CORPUS_GEN not found; run 'make corpus-gen' first" >&2
45+
return 1
46+
}
47+
mkdir -p "$CORPUS_ROOT/.staging"
48+
49+
local name command case_dir
50+
local -a failed=()
51+
while IFS=$'\t' read -r name command; do
52+
echo "==> $name"
53+
case_dir="$CORPUS_ROOT/$name"
54+
if [[ -d $case_dir ]]; then
55+
if corpus_is_current "$case_dir" "$command"; then
56+
echo "reusing $name"
57+
continue
58+
fi
59+
echo "removing outdated $name"
60+
rm -rf -- "$case_dir"
61+
fi
62+
if ! generate_case "$name" "$command"; then
63+
failed+=("$name")
64+
fi
65+
done < <(jq -r 'to_entries[] | [.key, .value] | @tsv' "$CATALOG")
66+
67+
if ((${#failed[@]} > 0)); then
68+
echo "failed: ${failed[*]}" >&2
69+
return 1
70+
fi
71+
prune_removed_cases
72+
echo "corpora written to $CORPUS_ROOT"
73+
}

‎bench/rpc/nightly/internal/preflight.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
rpc_result() {
2+
local method=$1
3+
curl --fail --silent --show-error \
4+
--connect-timeout 5 --max-time "${RPC_TIMEOUT:-30}" \
5+
--header 'Content-Type: application/json' \
6+
--data "$(jq -cn --arg method "$method" \
7+
'{jsonrpc:"2.0",id:1,method:$method,params:[]}')" \
8+
"$NODE_URL" |
9+
jq -er 'if .error then error(.error.message // "JSON-RPC error") else .result end'
10+
}
11+
12+
preflight() {
13+
rpc_result starknet_chainId >/dev/null
14+
}

‎bench/rpc/nightly/nightly.sh‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/usr/bin/env bash
2+
# Stable entrypoint for the managed nightly RPC benchmark.
3+
set -euo pipefail
4+
5+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
6+
RPC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
7+
CATALOG="$RPC_DIR/corpus/nightly.json"
8+
CI_CONFIG="$SCRIPT_DIR/ci.json"
9+
CORPUS_GEN=${CORPUS_GEN:-"$RPC_DIR/../../build/corpus-gen"}
10+
RUN_SCRIPT="$RPC_DIR/run.js"
11+
12+
: "${NODE_URL:?NODE_URL is required}"
13+
: "${SNAPSHOT_SHA256:?SNAPSHOT_SHA256 is required}"
14+
[[ $SNAPSHOT_SHA256 =~ ^[0-9a-f]{64}$ ]] || {
15+
echo "error: SNAPSHOT_SHA256 must be a lowercase SHA-256 digest" >&2
16+
exit 1
17+
}
18+
CORPUS_ROOT="${CORPUS_ROOT:-/corpus/cases}/$SNAPSHOT_SHA256"
19+
20+
source "$SCRIPT_DIR/internal/preflight.sh"
21+
source "$SCRIPT_DIR/internal/corpus.sh"
22+
23+
case_path() {
24+
printf '%s/%s/corpus.json\n' "$CORPUS_ROOT" "$1"
25+
}
26+
27+
benchmark() {
28+
[[ -f $CI_CONFIG ]] || {
29+
echo "error: k6 config not found: $CI_CONFIG" >&2
30+
return 1
31+
}
32+
local out_dir=${RESULTS_DIR:-/results}
33+
local warmup_iterations=${WARMUP_ITERATIONS:-200}
34+
mkdir -p "$out_dir"
35+
36+
local name corpus summary case_failed
37+
local -a failed=()
38+
while IFS= read -r name; do
39+
summary="$out_dir/$name.json"
40+
41+
echo
42+
echo "==> $name"
43+
corpus=$(case_path "$name")
44+
if [[ ! -f $corpus ]]; then
45+
echo "error: missing corpus for $name: $corpus" >&2
46+
failed+=("$name")
47+
continue
48+
fi
49+
corpus="$(cd "$(dirname "$corpus")" && pwd)/$(basename "$corpus")"
50+
51+
case_failed=0
52+
if [[ $warmup_iterations != 0 ]]; then
53+
echo "warming up"
54+
(
55+
K6_WEB_DASHBOARD=false k6 run \
56+
--vus 1 \
57+
--iterations "$warmup_iterations" \
58+
--tag "case_id=$name" \
59+
--tag phase=warmup \
60+
--summary-mode=disabled \
61+
-e NODE_URL="$NODE_URL" \
62+
"$RUN_SCRIPT" <"$corpus" >/dev/null
63+
) || {
64+
echo "error: warmup did not complete for $name" >&2
65+
case_failed=1
66+
}
67+
fi
68+
69+
if ((case_failed == 0)); then
70+
k6 run \
71+
--config "$CI_CONFIG" \
72+
--tag "case_id=$name" \
73+
--tag phase=measure \
74+
--summary-export "$summary" \
75+
-e NODE_URL="$NODE_URL" \
76+
"$RUN_SCRIPT" <"$corpus" || case_failed=1
77+
fi
78+
((case_failed == 0)) || failed+=("$name")
79+
done < <(jq -r 'keys_unsorted[]' "$CATALOG")
80+
81+
if ((${#failed[@]} > 0)); then
82+
echo "failed: ${failed[*]}" >&2
83+
return 1
84+
fi
85+
}
86+
87+
nightly_run() {
88+
preflight
89+
90+
local failed=0
91+
generate || failed=1
92+
benchmark || failed=1
93+
return "$failed"
94+
}
95+
96+
nightly_run

‎bench/rpc/nightly/test/test-nightly.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/sh
2+
3+
set -eu
4+
5+
repo_root=$(CDPATH= cd -- "$(dirname "$0")/../../../.." && pwd)
6+
catalog=${repo_root}/bench/rpc/corpus/nightly.json
7+
8+
# Validate filesystem-safe case definitions.
9+
jq -e '
10+
type == "object" and length > 0 and
11+
all(to_entries[];
12+
(.key | test("^[A-Za-z0-9][A-Za-z0-9._-]*$")) and
13+
(.value | type == "string" and test("[^[:space:]]")))
14+
' "${catalog}" >/dev/null
15+
16+
# Reject duplicate case IDs.
17+
test -z "$(jq --stream -r '
18+
select(length == 2 and (.[0] | length) == 1) | .[0][0]
19+
' "${catalog}" | sort | uniq -d)"
20+
21+
echo "nightly catalog validation passed"

0 commit comments

Comments
 (0)