qa-rpc-integration-tests-latest.yml

name: QA - RPC Integration Tests Latest

on:
  workflow_call:
  workflow_dispatch:     # Run manually
    inputs:
      force_dump_response:
        description: 'Force dump results even if they match (by default only if actual != expected)'
        type: boolean
        required: false
        default: false
  schedule:
    - cron: '0 0 * * *'  # Run nightly at 00:00 AM UTC
  push:
    branches:
      - 'release/3.*'
  # Auto-fire on PRs that touch this workflow's own YAML so changes to the
  # exec-mode matrix can be validated in-PR without a manual dispatch.
  # Regular PRs that don't touch this file are unaffected.
  pull_request:
    paths:
      - '.github/workflows/qa-rpc-integration-tests-latest.yml'

jobs:
  mainnet-rpc-integ-tests-latest:
    name: mainnet-rpc-integ-tests-latest (${{ matrix.exec_mode }})
    concurrency:
      group: >-
        ${{
          (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')) &&
          format('{0}-{1}', github.workflow, github.run_id) ||
          format('{0}-{1}', github.workflow, github.ref)
        }}
      cancel-in-progress: true
    runs-on: [ self-hosted, qa, Ethereum, rpc-latest-erigon ]
    strategy:
      fail-fast: false
      # Self-hosted pool typically has a single rpc-latest-erigon runner; the
      # workflow also touches shared state on the runner box (testbed datadir,
      # reference datadir). max-parallel=1 forces the two matrix entries to
      # run sequentially on the same runner — both still execute automatically,
      # just one after the other (~2× wall-clock).
      max-parallel: 1
      matrix:
        exec_mode:
          - serial
          - parallel
    env:
      ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
      ERIGON_QA_PATH: /home/qarunner/erigon-qa
      TOTAL_TIME_SECONDS: 1800 # 30 minutes
      REFERENCE_SYSTEM_HOST_ADDRESS: 157.180.55.78:8545
      ERIGON_ASSERT: true
      RPC_PAST_TEST_DIR: /opt/rpc-past-tests
      CHAIN: mainnet
      # Toggle dbg.Exec3Parallel from CI without code changes. envLookup
      # in common/dbg/dbg_env.go auto-prepends ERIGON_, so this maps to
      # the EXEC3_PARALLEL flag declared in common/dbg/experiments.go.
      ERIGON_EXEC3_PARALLEL: ${{ matrix.exec_mode == 'parallel' && 'true' || 'false' }}

    steps:
      - name: Check out repository
        uses: actions/checkout@v6

      - name: Read RPC version
        run: |
          source .github/workflows/scripts/rpc_version.env
          echo "RPC_VERSION=$RPC_VERSION" >> $GITHUB_ENV

      - name: Set reference data dir based on branch
        run: |
          # For pull_request events base_ref is the target branch name; for push/dispatch parse from ref.
          BRANCH="${{ github.base_ref }}"
          if [ -z "$BRANCH" ]; then
            BRANCH="${{ github.ref }}"
            BRANCH="${BRANCH#refs/heads/}"
          fi
          if [[ "$BRANCH" == release/* ]]; then
            VERSION="${BRANCH#release/}"
            echo "ERIGON_REFERENCE_DATA_DIR=/opt/erigon-versions/reference-version-${VERSION}/datadir" >> $GITHUB_ENV
          else
            echo "ERIGON_REFERENCE_DATA_DIR=/opt/erigon-versions/reference-version/datadir" >> $GITHUB_ENV
          fi

      - name: Clean Erigon Build Directory
        run: |
          make clean

      - name: Build Erigon and integration
        run: |
          make erigon integration
        working-directory: ${{ github.workspace }}

      - name: Wait for current Erigon instance to reach the tip
        id: preparing_step
        run: |
          ERIGON_PID=$(pgrep -x erigon | head -1)
          if [ -z "$ERIGON_PID" ]; then
            echo "Could not find running Erigon process dedicated to db maintenance"
          else
            echo "Found Erigon process dedicated to db maintenance, PID: $ERIGON_PID"
            python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-sync/run_and_chase_tip.py \
              --total-time=$TOTAL_TIME_SECONDS \
              --stop-erigon=False \
              --erigon-pid=$ERIGON_PID
          fi

      - name: Upload preparing-step logs on failure
        if: failure() && steps.preparing_step.outcome == 'failure'
        uses: actions/upload-artifact@v7
        with:
          name: preparing-step-logs-${{ matrix.exec_mode }}
          path: ${{ env.ERIGON_REFERENCE_DATA_DIR }}/logs/

      - name: Pause the Erigon instance dedicated to db maintenance
        run: |
          python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true

      - name: Save Erigon datadir Directory
        id: save_datadir_step
        run: |
          rm -rf $ERIGON_TESTBED_DATA_DIR || true

          echo "Mirror datadir"
          if ! "$GITHUB_WORKSPACE/cmd/scripts/mirror-datadir.sh" "$ERIGON_REFERENCE_DATA_DIR" "$ERIGON_TESTBED_DATA_DIR" > /dev/null 2>&1; then
            echo "::error::Failed to mirror datadir from $ERIGON_REFERENCE_DATA_DIR to $ERIGON_TESTBED_DATA_DIR"
            echo "Failed to mirror datadir from $ERIGON_REFERENCE_DATA_DIR to $ERIGON_TESTBED_DATA_DIR"
            exit 1
          fi
          echo "datadir_saved=true" >> $GITHUB_OUTPUT

      - name: Run Migrations
        working-directory: ${{ github.workspace }}/build/bin
        run: |
          echo "Running migrations on datadir..."
          ./integration run_migrations --datadir $ERIGON_TESTBED_DATA_DIR --chain $CHAIN

      - name: Run Erigon and wait for sync
        id: pre_test_step
        run: |
          set +e # Disable exit on error
          
          # Launch the testbed Erigon instance & test its ability to sync 
          python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-sync/run_and_chase_tip.py \
            --build-dir=${{ github.workspace }}/build/bin \
            --data-dir=$ERIGON_TESTBED_DATA_DIR \
            --total-time=$TOTAL_TIME_SECONDS \
            --chain=$CHAIN \
            --node-type=minimal_node \
            --stop-erigon=False
          
          # Capture monitoring script exit status
          test_exit_status=$?
          
          # Check test runner script exit status
          if [ $test_exit_status -eq 0 ]; then
             echo "Tip-chasing completed successfully"
             echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
          else
             echo "Tip-chasing encountered an error test aborted"
             echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
             exit 1
          fi

      - name: Upload pre-test logs on failure
        if: failure() && steps.pre_test_step.outcome == 'failure'
        uses: actions/upload-artifact@v7
        with:
          name: pre-test-logs-${{ matrix.exec_mode }}
          path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/

      - name: Run RPC Integration Tests
        id: test_step
        run: |
          commit=$(git -C ${{runner.workspace}}/erigon rev-parse --short HEAD)
          TEST_RESULT_DIR="$RPC_PAST_TEST_DIR/mainnet_$(date +%Y%m%d_%H%M%S)_integration_${commit}_http"
          echo "TEST_RESULT_DIR=$TEST_RESULT_DIR" >> $GITHUB_ENV

          DUMP_RESPONSE=""
          if [[ "${{ inputs.force_dump_response }}" == "true" ]]; then
            DUMP_RESPONSE="always-dump-response"
            echo "Force response dump is enabled"
          fi

          chmod +x ${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests_ethereum_latest.sh

          set +e # Disable exit on error for test run
          ${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests_ethereum_latest.sh ${{ runner.workspace }} $TEST_RESULT_DIR $REFERENCE_SYSTEM_HOST_ADDRESS $DUMP_RESPONSE
          test_exit_status=$? # Capture test runner script exit status
          set -e # Re-enable exit on error after test run

          echo "test_executed=true" >> $GITHUB_OUTPUT

          if [ $test_exit_status -eq 0 ]; then
            echo "RPC tests completed successfully"
            echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
          else
            echo "::error::Error detected during RPC tests: some tests failed, check the logs or the artifacts for more details"
            echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
            exit 1
          fi

      - name: Generate Summary
        if: always() && steps.test_step.outputs.test_executed == 'true'
        run: |
          SUMMARY_FILE="${{ env.TEST_RESULT_DIR }}/summary.md"
          LOG_FILE="${{ env.TEST_RESULT_DIR }}/output.log"

          cat << 'EOF' > $SUMMARY_FILE
          # ${{ github.workflow }} Report

          ## Test Configuration
          - **Chain:** ${{ env.CHAIN }}
          - **Result:** ${{ steps.test_step.outputs.TEST_RESULT }}

          ## Test Output
          ```
          EOF

          cat $LOG_FILE >> $SUMMARY_FILE

          echo '```' >> $SUMMARY_FILE

          cat $SUMMARY_FILE
          cat $SUMMARY_FILE >> $GITHUB_STEP_SUMMARY

      - name: Stop Erigon
        if: always()
        run: |
          # the erigon pid is stored in erigon.pid file
          ERIGON_PID=$(cat ./erigon.pid)
          # Clean up rpcdaemon process if it's still running
          if [ -n "$ERIGON_PID" ] && kill -0 $ERIGON_PID 2> /dev/null; then
            echo "Erigon stopping..."
            kill $ERIGON_PID
            sleep 5
            echo "Erigon stopped"
          else
            echo "Erigon has already terminated"
          fi

      - name: Upload test results
        if: always() && steps.test_step.outputs.test_executed == 'true'
        uses: actions/upload-artifact@v7
        with:
          name: test-results-${{ matrix.exec_mode }}
          path: |
            ${{ env.TEST_RESULT_DIR }}
            ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/

      - name: Save test results
        if: always() && steps.test_step.outputs.test_executed == 'true'
        working-directory: ${{ github.workspace }}
        env:
          TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }}
        run: |
          db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit)
          if [ -z "$db_version" ]; then
            db_version="no-version"
          fi
          
          python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
            --repo erigon \
            --commit $(git rev-parse HEAD) \
            --branch ${{ github.ref_name }} \
            --test_name rpc-integration-tests-latest${{ matrix.exec_mode == 'parallel' && '-parallel' || '' }} \
            --chain $CHAIN \
            --runner ${{ runner.name }} \
            --db_version $db_version \
            --outcome $TEST_RESULT \
            --result_file $TEST_RESULT_DIR/results/test_report.json

      - name: Delete Erigon Testbed Data Directory
        if: always() && steps.save_datadir_step.outputs.datadir_saved == 'true'
        run: |
          if [ -d "$ERIGON_TESTBED_DATA_DIR" ]; then
            rm -rf $ERIGON_TESTBED_DATA_DIR
          fi

      - name: Resume the Erigon instance dedicated to db maintenance
        if: always()
        run: |
          python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true

      - name: Action to check failure condition
        if: failure()
        run: |
          if [ "${{ steps.test_step.outputs.test_executed }}" != "true" ]; then
            echo "::error::Test not executed, workflow failed for infrastructure reasons"
          fi
          exit 1

      - name: Action for Success
        if: steps.test_step.outputs.TEST_RESULT == 'success'
        run: echo "::notice::Tests completed successfully"