QA - RPC Performance Tests #142
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QA - RPC Performance Tests | |
| on: | |
| push: | |
| branches: | |
| - 'release/3.*' | |
| workflow_dispatch: | |
| inputs: | |
| run_geth: | |
| description: 'Run the Geth benchmark besides the Erigon one' | |
| type: boolean | |
| required: false | |
| default: false | |
| schedule: | |
| - cron: '0 3 * * *' # Run daily at 03:00 UTC | |
| concurrency: | |
| group: rpc-performance-historic-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| CHAIN: mainnet | |
| ERIGON_QA_PATH: /home/qarunner/erigon-qa | |
| RPC_PAST_TEST_DIR: /opt/rpc-past-tests | |
| jobs: | |
| setup: | |
| name: Setup benchmark run | |
| runs-on: [self-hosted, qa] | |
| permissions: | |
| contents: read | |
| outputs: | |
| run_id: ${{ steps.vars.outputs.run_id }} | |
| network: ${{ steps.vars.outputs.network }} | |
| run_geth: ${{ steps.vars.outputs.run_geth }} | |
| steps: | |
| - name: Compute shared variables | |
| id: vars | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # Unique namespace for this workflow execution | |
| RUN_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| # Decide whether to run geth: | |
| # - schedule: only on Sunday (UTC) | |
| # - workflow_dispatch: controlled by input | |
| RUN_GETH="false" | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| DOW="$(date -u +%w)" | |
| if [ "$DOW" = "0" ]; then # Sunday, (0..6 = Sun..Sat) | |
| RUN_GETH="true" | |
| fi | |
| elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| if [ "${{ github.event.inputs.run_geth }}" = "true" ]; then | |
| RUN_GETH="true" | |
| fi | |
| fi | |
| # Variables | |
| echo "run_id=${RUN_ID}" >> "$GITHUB_OUTPUT" | |
| echo "network=${{ env.CHAIN }}" >> "$GITHUB_OUTPUT" | |
| echo "run_geth=${RUN_GETH}" >> "$GITHUB_OUTPUT" | |
| - name: Print configuration (for logs) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| echo "Run ID: ${{ steps.vars.outputs.run_id }}" | |
| echo "Network: ${{ steps.vars.outputs.network }}" | |
| echo "Run Erigon: true" | |
| echo "Run Geth: ${{ steps.vars.outputs.run_geth }}" | |
| test: | |
| name: Benchmark Historic ${{ matrix.client }} | |
| needs: [setup] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - client: erigon | |
| runs-on: [self-hosted, qa, Ethereum, rpc-performance] | |
| - client: geth | |
| runs-on: [self-hosted, qa, Ethereum, rpc-historical] | |
| runs-on: ${{ matrix.runs-on }} | |
| env: | |
| RUN_ID: ${{ needs.setup.outputs.run_id }} | |
| NETWORK: ${{ needs.setup.outputs.network }} | |
| ERIGON_REFERENCE_DIR: /opt/erigon-versions/reference-version | |
| ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir | |
| ERIGON_TESTBED_AREA: /opt/erigon-testbed | |
| ERIGON_QA_PATH: /home/qarunner/erigon-qa | |
| GETH_INSTALL_DIR: /opt/go-ethereum | |
| CLIENT: ${{ matrix.client }} | |
| steps: | |
| - name: Checkout Erigon repository | |
| if: matrix.client == 'erigon' | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: recursive | |
| fetch-depth: "0" | |
| - name: Checkout RPC Tests Repository | |
| if: matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true' | |
| run: | | |
| rm -rf ${{runner.workspace}}/rpc-tests | |
| git -c advice.detachedHead=false clone --depth 1 --branch v1.115.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests | |
| cd ${{runner.workspace}}/rpc-tests | |
| - name: Clean Erigon Build Directory | |
| if: matrix.client == 'erigon' | |
| run: | | |
| make clean | |
| - name: Build Erigon RPCDaemon | |
| if: matrix.client == 'erigon' | |
| run: | | |
| make rpcdaemon | |
| working-directory: ${{ github.workspace }} | |
| - name: Pause the Erigon instance dedicated to db maintenance | |
| if: matrix.client == 'erigon' | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true | |
| - name: Save Erigon Chaindata Directory | |
| if: matrix.client == 'erigon' | |
| id: save_chaindata_step | |
| run: | | |
| rm -rf $ERIGON_TESTBED_AREA/chaindata-prev || true | |
| echo "Backup chaindata" | |
| cp -r $ERIGON_REFERENCE_DATA_DIR/chaindata $ERIGON_TESTBED_AREA/chaindata-prev | |
| rm -f $ERIGON_REFERENCE_DATA_DIR/logs/rpcdaemon.log || true | |
| echo "datadir_saved=true" >> $GITHUB_OUTPUT | |
| - name: Run RpcDaemon | |
| id: rpcdaemon_running_step | |
| if: matrix.client == 'erigon' | |
| working-directory: ${{ github.workspace }}/build/bin | |
| run: | | |
| echo "Starting RpcDaemon..." | |
| ./rpcdaemon --datadir $ERIGON_REFERENCE_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws > erigon.log 2>&1 & | |
| RPC_DAEMON_PID=$! | |
| RPC_DAEMON_EXIT_STATUS=$? | |
| echo "RPC_DAEMON_PID=$RPC_DAEMON_PID" >> $GITHUB_ENV | |
| echo "rpc_daemon_started=true" >> $GITHUB_OUTPUT | |
| sleep 5 | |
| tail erigon.log | |
| if [ $RPC_DAEMON_EXIT_STATUS -ne 0 ]; then | |
| echo "RpcDaemon failed to start" | |
| echo "::error::Error detected during tests: RpcDaemon failed to start" | |
| exit 1 | |
| fi | |
| echo "RpcDaemon started" | |
| - name: Wait for port 8545 to be opened | |
| if: matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true' | |
| run: | | |
| # Geth is expected to be pre-running for the rpc-latest-geth runner | |
| # Erigon was started above | |
| for i in {1..30}; do | |
| if nc -z localhost 8545; then | |
| echo "Port 8545 is open" | |
| break | |
| fi | |
| echo "Waiting for port 8545 to open..." | |
| sleep 10 | |
| done | |
| if ! nc -z localhost 8545; then | |
| echo "Port 8545 did not open in time" | |
| echo "::error::Error detected during tests: Port 8545 did not open in time" | |
| exit 1 | |
| fi | |
| - name: Run RPC Performance Tests | |
| id: test_step | |
| if: matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true' | |
| run: | | |
| set +e # Disable exit on error | |
| failed_test=0 | |
| if [ "${{ matrix.client }}" == "erigon" ]; then | |
| commit=$(git -C ${{runner.workspace}}/erigon rev-parse --short HEAD) | |
| else | |
| commit=$(git -C $GETH_INSTALL_DIR rev-parse --short HEAD) | |
| fi | |
| # Prepare historical test results directory | |
| # a) Save text results to a directory with timestamp and commit hash | |
| past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_$(date +%Y%m%d_%H%M%S)_comp_perf_$commit | |
| echo "past_test_dir=$past_test_dir" >> $GITHUB_ENV | |
| mkdir -p $past_test_dir | |
| # b) Save binary results to a fixed directory | |
| bin_past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_bin | |
| rm -rf $bin_past_test_dir # we want only the latest binary files | |
| mkdir -p $bin_past_test_dir | |
| run_perf () { | |
| workspace=$1 | |
| network=$2 | |
| method=$3 | |
| pattern=$4 | |
| sequence=$5 | |
| client=$6 | |
| result_dir=$workspace/rpc-tests/perf/reports/$network | |
| result_file=$client-$method-result.json | |
| # clean temporary area | |
| cd $workspace/rpc-tests/perf | |
| rm -rf ./reports/$network | |
| python3 ./run_perf_tests.py --blockchain "$network" \ | |
| --test-type "$method" \ | |
| --pattern-file pattern/"$network"/"$pattern".tar \ | |
| --test-sequence "$sequence" \ | |
| --repetitions 5 \ | |
| --erigon-dir "/" \ | |
| --silk-dir "/" \ | |
| --test-mode 2 \ | |
| --test-report \ | |
| --json-report $result_dir/$result_file \ | |
| --testing-daemon $client | |
| # Capture test runner script exit status | |
| perf_exit_status=$? | |
| # Detect the pre-built db version | |
| if [ "$client" == "erigon" ]; then | |
| db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DIR/production.ini production erigon_repo_commit) | |
| else | |
| db_version=$commit | |
| fi | |
| # Check test runner script exit status | |
| if [ $perf_exit_status -eq 0 ]; then | |
| outcome=success | |
| # Save all vegeta binary reports | |
| echo "Save current vegeta binary files" | |
| if [ -d "$workspace/rpc-tests/perf/reports/bin" ]; then | |
| cp -r "$workspace/rpc-tests/perf/reports/bin" "$bin_past_test_dir" | |
| else | |
| echo "::warning::vegeta binary reports directory '$workspace/rpc-tests/perf/reports/bin' not found; skipping copy." | |
| fi | |
| echo "Execute Latency Percentile HDR Analysis" | |
| cd $result_dir | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/rpc-tests/perf_hdr_analysis.py \ | |
| --test_name $client-$method \ | |
| --input_file ./$result_file \ | |
| --output_file ./$client-$method-latency_hdr_analysis.pdf | |
| perf_hdr_status=$? | |
| if [ $perf_hdr_status -ne 0 ]; then | |
| echo "::warning::perf_hdr_analysis.py failed with exit code $perf_hdr_status" | |
| fi | |
| else | |
| failed_test=1 | |
| outcome=failure | |
| fi | |
| # Save results on DB | |
| echo "Save test result on DB" | |
| if [ "$client" == "erigon" ]; then | |
| branch_name=${{ github.ref_name }} | |
| commit_hash=$(git -C $workspace/erigon rev-parse HEAD) | |
| else | |
| branch_name="release" | |
| commit_hash=$commit | |
| fi | |
| echo branch_name=$branch_name | |
| echo commit_hash=$commit_hash | |
| echo method=$method | |
| echo db_version=$db_version | |
| echo outcome=$outcome | |
| echo result_file=$result_file | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \ | |
| --repo $client \ | |
| --branch $branch_name \ | |
| --commit $commit_hash \ | |
| --test_name rpc-performance-test-$client-$method \ | |
| --chain $CHAIN \ | |
| --runner ${{ runner.name }} \ | |
| --db_version $db_version \ | |
| --outcome $outcome \ | |
| --result_file $result_dir/$result_file | |
| if [ $? -ne 0 ]; then | |
| failed_test=1 | |
| echo "Failure saving test results on DB" | |
| fi | |
| # Save test results to a directory with timestamp and commit hash | |
| if [ -d "$result_dir" ]; then | |
| cp -r "$result_dir" "$past_test_dir" | |
| else | |
| echo "::warning::Result directory '$result_dir' does not exist; skipping archival to $past_test_dir" | |
| fi | |
| } | |
| # Launch the RPC performance test runner | |
| run_perf ${{runner.workspace}} $CHAIN eth_call stress_test_eth_call_001_14M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN eth_getLogs stress_test_eth_getLogs_15M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN eth_getBalance stress_test_eth_getBalance_15M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN eth_getBlockByHash stress_test_eth_getBlockByHash_14M 1:1,100:30,1000:20,5000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN eth_getBlockByNumber stress_test_eth_getBlockByNumber_13M 1:1,100:30,1000:20,5000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN eth_getTransactionByHash stress_test_eth_getTransactionByHash_13M 1:1,100:30,1000:20,10000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN eth_getTransactionReceipt stress_test_eth_getTransactionReceipt_14M 1:1,100:30,1000:20,5000:20,10000:20,20000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN eth_createAccessList stress_test_eth_createAccessList_16M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN debug_traceTransaction stress_test_debug_trace_transaction_21M 1:1,100:30,1000:20,5000:20 $CLIENT | |
| run_perf ${{runner.workspace}} $CHAIN debug_storageRangeAt stress_test_debug_storageRangeAt_20M 1:1,100:30,1000:20,5000:20 $CLIENT | |
| # Save the subsection reached status | |
| echo "test_executed=true" >> $GITHUB_OUTPUT | |
| if [ $failed_test -eq 0 ]; then | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| echo "Tests completed successfully" | |
| else | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| echo "Error detected during tests" | |
| echo "::error::Error detected during tests" | |
| exit 1 | |
| fi | |
| - name: Stop Erigon RpcDaemon | |
| if: always() && matrix.client == 'erigon' | |
| working-directory: ${{ github.workspace }}/build/bin | |
| run: | | |
| # Clean up rpcdaemon process if it's still running | |
| if [ -n "$RPC_DAEMON_PID" ] && kill -0 $RPC_DAEMON_PID 2> /dev/null; then | |
| echo "RpcDaemon stopping..." | |
| kill $RPC_DAEMON_PID | |
| echo "RpcDaemon stopped" | |
| else | |
| echo "RpcDaemon has already terminated" | |
| fi | |
| - name: Upload RpcDaemon logs | |
| if: matrix.client == 'erigon' && steps.test_step.outputs.test_executed == 'true' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: rpcdaemon-logs | |
| path: $ERIGON_REFERENCE_DATA_DIR/logs/rpcdaemon.log | |
| - name: Restore Erigon Chaindata Directory | |
| if: always() && matrix.client == 'erigon' && steps.save_chaindata_step.outputs.datadir_saved == 'true' | |
| run: | | |
| if [ -d "$ERIGON_TESTBED_AREA/chaindata-prev" ] && [ "${{ steps.save_chaindata_step.outcome }}" == "success" ]; then | |
| rm -rf $ERIGON_REFERENCE_DATA_DIR/chaindata | |
| echo "Restore chaindata" | |
| mv $ERIGON_TESTBED_AREA/chaindata-prev $ERIGON_REFERENCE_DATA_DIR/chaindata | |
| fi | |
| - name: Resume the Erigon instance dedicated to db maintenance | |
| if: always() && matrix.client == 'erigon' | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true | |
| - name: Run change point analysis | |
| if: matrix.client == 'erigon' && steps.test_step.outputs.TEST_RESULT == 'success' | |
| working-directory: ${{runner.workspace}}/rpc-tests/perf/reports/mainnet | |
| run: | | |
| set +e # Disable exit on error | |
| open_change_points=0 | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/change-points/change_point_analysis.py --repo erigon | |
| open_change_points=$? | |
| cp change_point_analysis.pdf $past_test_dir | |
| if [ $open_change_points -ne 0 ]; then | |
| echo "Change point analysis found points that need to be investigated" | |
| echo "::warning title=Change Point Analysis::Found change points that need to be investigated" | |
| #echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" -- enable in the future | |
| fi | |
| - name: Upload test results | |
| if: (matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true') && steps.test_step.outputs.test_executed == 'true' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-results-${{ env.CHAIN }}-${{ matrix.client }} | |
| path: ${{ env.past_test_dir }} | |
| - name: Action to check failure condition | |
| if: failure() | |
| run: | | |
| if [ "${{ steps.test_step.outputs.test_executed }}" != "true" ]; then | |
| echo "::error::Test not executed, workflow failed for infrastructure reasons" | |
| fi | |
| exit 1 | |
| - name: Action for Success | |
| if: steps.test_step.outputs.TEST_RESULT == 'success' | |
| run: echo "::notice::Tests completed successfully" |