qa-rpc-performance-tests.yml

name: QA - RPC Performance Tests

on:
  push:
    branches:
      - 'release/3.*'
  workflow_dispatch:
    inputs:
      run_geth:
        description: 'Run the Geth benchmark besides the Erigon one'
        type: boolean
        required: false
        default: false
  schedule:
    - cron: '0 3 * * *'  # Run daily at 03:00 UTC

concurrency:
  group: rpc-performance-historic-${{ github.ref }}
  cancel-in-progress: false

env:
  CHAIN: mainnet
  ERIGON_QA_PATH: /home/qarunner/erigon-qa
  RPC_PAST_TEST_DIR: /opt/rpc-past-tests

jobs:
  setup:
    name: Setup benchmark run
    runs-on: [self-hosted, qa]
    permissions:
      contents: read
    outputs:
      run_id: ${{ steps.vars.outputs.run_id }}
      network: ${{ steps.vars.outputs.network }}
      run_geth: ${{ steps.vars.outputs.run_geth }}
    steps:
      - name: Compute shared variables
        id: vars
        shell: bash
        run: |
          set -euo pipefail

          # Unique namespace for this workflow execution
          RUN_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"

          # Decide whether to run geth:
          # - schedule: only on Sunday (UTC)
          # - workflow_dispatch: controlled by input
          RUN_GETH="false"
          if [ "${{ github.event_name }}" = "schedule" ]; then
            DOW="$(date -u +%w)"
            if [ "$DOW" = "0" ]; then  # Sunday, (0..6 = Sun..Sat)
              RUN_GETH="true"
            fi
          elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            if [ "${{ github.event.inputs.run_geth }}" = "true" ]; then
              RUN_GETH="true"
            fi
          fi
          
          # Variables
          echo "run_id=${RUN_ID}" >> "$GITHUB_OUTPUT"
          echo "network=${{ env.CHAIN }}" >> "$GITHUB_OUTPUT"
          echo "run_geth=${RUN_GETH}" >> "$GITHUB_OUTPUT"

      - name: Print configuration (for logs)
        shell: bash
        run: |
          set -euo pipefail
          echo "Run ID:             ${{ steps.vars.outputs.run_id }}"
          echo "Network:            ${{ steps.vars.outputs.network }}"
          echo "Run Erigon:         true"
          echo "Run Geth:           ${{ steps.vars.outputs.run_geth }}"

  test:
    name: Benchmark Historic ${{ matrix.client }}
    needs: [setup]
    strategy:
      fail-fast: false
      matrix:
        include:
          - client: erigon
            runs-on: [self-hosted, qa, Ethereum, rpc-performance]
          - client: geth
            runs-on: [self-hosted, qa, Ethereum, rpc-historical]
    runs-on: ${{ matrix.runs-on }}
    env:
      RUN_ID: ${{ needs.setup.outputs.run_id }}
      NETWORK: ${{ needs.setup.outputs.network }}
      ERIGON_REFERENCE_DIR: /opt/erigon-versions/reference-version
      ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
      ERIGON_TESTBED_AREA: /opt/erigon-testbed
      ERIGON_QA_PATH: /home/qarunner/erigon-qa
      GETH_INSTALL_DIR: /opt/go-ethereum
      CLIENT: ${{ matrix.client }}

    steps:
      - name: Checkout Erigon repository
        if: matrix.client == 'erigon'
        uses: actions/checkout@v6
        with:
          submodules: recursive
          fetch-depth: "0"

      - name: Checkout RPC Tests Repository
        if: matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true'
        run: |
          rm -rf ${{runner.workspace}}/rpc-tests
          git -c advice.detachedHead=false clone --depth 1 --branch v1.115.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
          cd ${{runner.workspace}}/rpc-tests

      - name: Clean Erigon Build Directory
        if: matrix.client == 'erigon'
        run: |
          make clean

      - name: Build Erigon RPCDaemon
        if: matrix.client == 'erigon'
        run: |
          make rpcdaemon
        working-directory: ${{ github.workspace }}

      - name: Pause the Erigon instance dedicated to db maintenance
        if: matrix.client == 'erigon'
        run: |
          python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true

      - name: Save Erigon Chaindata Directory
        if: matrix.client == 'erigon'
        id: save_chaindata_step
        run: |
          rm -rf $ERIGON_TESTBED_AREA/chaindata-prev || true
          echo "Backup chaindata"
          cp -r $ERIGON_REFERENCE_DATA_DIR/chaindata $ERIGON_TESTBED_AREA/chaindata-prev
          rm -f $ERIGON_REFERENCE_DATA_DIR/logs/rpcdaemon.log || true
          echo "datadir_saved=true" >> $GITHUB_OUTPUT

      - name: Run RpcDaemon
        id: rpcdaemon_running_step
        if: matrix.client == 'erigon'
        working-directory: ${{ github.workspace }}/build/bin
        run: |
          echo "Starting RpcDaemon..."
          
          ./rpcdaemon --datadir $ERIGON_REFERENCE_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws > erigon.log 2>&1 &

          RPC_DAEMON_PID=$!          
          RPC_DAEMON_EXIT_STATUS=$?
          echo "RPC_DAEMON_PID=$RPC_DAEMON_PID" >> $GITHUB_ENV
          echo "rpc_daemon_started=true" >> $GITHUB_OUTPUT
          
          sleep 5
          tail erigon.log
          
          if [ $RPC_DAEMON_EXIT_STATUS -ne 0 ]; then            
            echo "RpcDaemon failed to start"
            echo "::error::Error detected during tests: RpcDaemon failed to start"
            exit 1
          fi
          echo "RpcDaemon started"

      - name: Wait for port 8545 to be opened
        if: matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true'
        run: |
          # Geth is expected to be pre-running for the rpc-latest-geth runner
          # Erigon was started above
          for i in {1..30}; do
            if nc -z localhost 8545; then
              echo "Port 8545 is open"
              break
            fi
            echo "Waiting for port 8545 to open..."
            sleep 10
          done
          if ! nc -z localhost 8545; then
            echo "Port 8545 did not open in time"
            echo "::error::Error detected during tests: Port 8545 did not open in time"
            exit 1
          fi

      - name: Run RPC Performance Tests
        id: test_step
        if: matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true'
        run: |
          set +e # Disable exit on error
          
          failed_test=0
          
          if [ "${{ matrix.client }}" == "erigon" ]; then
            commit=$(git -C ${{runner.workspace}}/erigon rev-parse --short HEAD)
          else
            commit=$(git -C $GETH_INSTALL_DIR rev-parse --short HEAD)
          fi
          
          # Prepare historical test results directory
          # a) Save text results to a directory with timestamp and commit hash
          past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_$(date +%Y%m%d_%H%M%S)_comp_perf_$commit
          echo "past_test_dir=$past_test_dir" >> $GITHUB_ENV          
          mkdir -p $past_test_dir
          # b) Save binary results to a fixed directory
          bin_past_test_dir=$RPC_PAST_TEST_DIR/${CHAIN}_bin
          rm -rf $bin_past_test_dir  # we want only the latest binary files
          mkdir -p $bin_past_test_dir
          
          run_perf () {
            workspace=$1
            network=$2
            method=$3
            pattern=$4
            sequence=$5
            client=$6
            
            result_dir=$workspace/rpc-tests/perf/reports/$network
            result_file=$client-$method-result.json            
                    
            # clean temporary area
            cd $workspace/rpc-tests/perf
            rm -rf ./reports/$network
          
            python3 ./run_perf_tests.py --blockchain "$network" \
                                      --test-type "$method" \
                                      --pattern-file pattern/"$network"/"$pattern".tar \
                                      --test-sequence "$sequence" \
                                      --repetitions 5 \
                                      --erigon-dir "/" \
                                      --silk-dir "/" \
                                      --test-mode 2 \
                                      --test-report \
                                      --json-report $result_dir/$result_file \
                                      --testing-daemon $client
          
            # Capture test runner script exit status
            perf_exit_status=$?
                    
            # Detect the pre-built db version
            if [ "$client" == "erigon" ]; then
              db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DIR/production.ini production erigon_repo_commit)
            else
              db_version=$commit
            fi
          
            # Check test runner script exit status
            if [ $perf_exit_status -eq 0 ]; then
              outcome=success
          
              # Save all vegeta binary reports
              echo "Save current vegeta binary files"
              if [ -d "$workspace/rpc-tests/perf/reports/bin" ]; then
                cp -r "$workspace/rpc-tests/perf/reports/bin" "$bin_past_test_dir"
              else
                echo "::warning::vegeta binary reports directory '$workspace/rpc-tests/perf/reports/bin' not found; skipping copy."
              fi
                    
              echo "Execute Latency Percentile HDR Analysis"
              cd $result_dir
              python3 $ERIGON_QA_PATH/test_system/qa-tests/rpc-tests/perf_hdr_analysis.py \
                --test_name $client-$method \
                --input_file ./$result_file \
                --output_file ./$client-$method-latency_hdr_analysis.pdf
              perf_hdr_status=$?
              if [ $perf_hdr_status -ne 0 ]; then
                echo "::warning::perf_hdr_analysis.py failed with exit code $perf_hdr_status"
              fi
            else
              failed_test=1
              outcome=failure
            fi
          
            # Save results on DB
            echo "Save test result on DB"

            if [ "$client" == "erigon" ]; then
              branch_name=${{ github.ref_name }}
              commit_hash=$(git -C $workspace/erigon rev-parse HEAD)
            else
              branch_name="release"
              commit_hash=$commit
            fi
            
            echo branch_name=$branch_name
            echo commit_hash=$commit_hash
            echo method=$method
            echo db_version=$db_version
            echo outcome=$outcome
            echo result_file=$result_file
          
            python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
              --repo $client \
              --branch $branch_name \
              --commit $commit_hash \
              --test_name rpc-performance-test-$client-$method \
              --chain $CHAIN \
              --runner ${{ runner.name }} \
              --db_version $db_version \
              --outcome $outcome \
              --result_file $result_dir/$result_file
        
            if [ $? -ne 0 ]; then
              failed_test=1
              echo "Failure saving test results on DB"
            fi
          
            # Save test results to a directory with timestamp and commit hash
            if [ -d "$result_dir" ]; then
              cp -r "$result_dir" "$past_test_dir"
            else
              echo "::warning::Result directory '$result_dir' does not exist; skipping archival to $past_test_dir"
            fi
          }

          # Launch the RPC performance test runner
          run_perf ${{runner.workspace}} $CHAIN eth_call stress_test_eth_call_001_14M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT
          run_perf ${{runner.workspace}} $CHAIN eth_getLogs stress_test_eth_getLogs_15M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT
          run_perf ${{runner.workspace}} $CHAIN eth_getBalance stress_test_eth_getBalance_15M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT
          run_perf ${{runner.workspace}} $CHAIN eth_getBlockByHash stress_test_eth_getBlockByHash_14M 1:1,100:30,1000:20,5000:20 $CLIENT
          run_perf ${{runner.workspace}} $CHAIN eth_getBlockByNumber stress_test_eth_getBlockByNumber_13M 1:1,100:30,1000:20,5000:20 $CLIENT
          run_perf ${{runner.workspace}} $CHAIN eth_getTransactionByHash stress_test_eth_getTransactionByHash_13M 1:1,100:30,1000:20,10000:20 $CLIENT
          run_perf ${{runner.workspace}} $CHAIN eth_getTransactionReceipt stress_test_eth_getTransactionReceipt_14M 1:1,100:30,1000:20,5000:20,10000:20,20000:20 $CLIENT
          run_perf ${{runner.workspace}} $CHAIN eth_createAccessList stress_test_eth_createAccessList_16M 1:1,100:30,1000:20,10000:20,20000:20 $CLIENT          
          run_perf ${{runner.workspace}} $CHAIN debug_traceTransaction stress_test_debug_trace_transaction_21M 1:1,100:30,1000:20,5000:20 $CLIENT          
          run_perf ${{runner.workspace}} $CHAIN debug_storageRangeAt stress_test_debug_storageRangeAt_20M 1:1,100:30,1000:20,5000:20 $CLIENT          
          
          # Save the subsection reached status
          
          echo "test_executed=true" >> $GITHUB_OUTPUT
          
          if [ $failed_test -eq 0 ]; then
            echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
            echo "Tests completed successfully"
          else
            echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
            echo "Error detected during tests"
            echo "::error::Error detected during tests"
            exit 1
          fi

      - name: Stop Erigon RpcDaemon
        if: always() && matrix.client == 'erigon'
        working-directory: ${{ github.workspace }}/build/bin
        run: |
          # Clean up rpcdaemon process if it's still running
          if [ -n "$RPC_DAEMON_PID" ] && kill -0 $RPC_DAEMON_PID 2> /dev/null; then
            echo "RpcDaemon stopping..."
            kill $RPC_DAEMON_PID
            echo "RpcDaemon stopped"
          else
            echo "RpcDaemon has already terminated"
          fi

      - name: Upload RpcDaemon logs
        if: matrix.client == 'erigon' && steps.test_step.outputs.test_executed == 'true'
        uses: actions/upload-artifact@v6
        with:
          name: rpcdaemon-logs
          path: $ERIGON_REFERENCE_DATA_DIR/logs/rpcdaemon.log

      - name: Restore Erigon Chaindata Directory
        if: always() && matrix.client == 'erigon' && steps.save_chaindata_step.outputs.datadir_saved == 'true'
        run: |
          if [ -d "$ERIGON_TESTBED_AREA/chaindata-prev" ] && [ "${{ steps.save_chaindata_step.outcome }}" == "success" ]; then
          rm -rf $ERIGON_REFERENCE_DATA_DIR/chaindata
          echo "Restore chaindata"
          mv $ERIGON_TESTBED_AREA/chaindata-prev $ERIGON_REFERENCE_DATA_DIR/chaindata
          fi

      - name: Resume the Erigon instance dedicated to db maintenance
        if: always() && matrix.client == 'erigon'
        run: |
          python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true

      - name: Run change point analysis
        if: matrix.client == 'erigon' && steps.test_step.outputs.TEST_RESULT == 'success'
        working-directory: ${{runner.workspace}}/rpc-tests/perf/reports/mainnet
        run: |
          set +e # Disable exit on error
          open_change_points=0
          python3 $ERIGON_QA_PATH/test_system/qa-tests/change-points/change_point_analysis.py --repo erigon
          open_change_points=$?
          cp change_point_analysis.pdf $past_test_dir
          if [ $open_change_points -ne 0 ]; then
            echo "Change point analysis found points that need to be investigated"
            echo "::warning title=Change Point Analysis::Found change points that need to be investigated"
            #echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"  -- enable in the future
          fi

      - name: Upload test results
        if: (matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true') && steps.test_step.outputs.test_executed == 'true'
        uses: actions/upload-artifact@v6
        with:
          name: test-results-${{ env.CHAIN }}-${{ matrix.client }}
          path: ${{ env.past_test_dir }}

      - name: Action to check failure condition
        if: failure()
        run: |
          if [ "${{ steps.test_step.outputs.test_executed }}" != "true" ]; then
            echo "::error::Test not executed, workflow failed for infrastructure reasons"
          fi
          exit 1

      - name: Action for Success
        if: steps.test_step.outputs.TEST_RESULT == 'success'
        run: echo "::notice::Tests completed successfully"