qa-tip-tracking-gnosis.yml

name: QA - Tip tracking & migration (Gnosis)

on:
  schedule:
    - cron: '0 1 * * 0'  # Run on Sunday at 01:00 AM UTC
  push:
    branches:
      - 'release/3.*'
  workflow_dispatch: # Run manually

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:
  gnosis-tip-tracking-test:
    runs-on: [self-hosted, qa, Gnosis, tip-tracking]
    timeout-minutes: 1200
    env:
      ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
      ERIGON_QA_PATH: /home/qarunner/erigon-qa
      TRACKING_TIME_SECONDS: 7200 # 2 hours
      TOTAL_TIME_SECONDS: 28800 # 8 hours
      CHAIN: gnosis

    steps:
    - name: Check out repository
      uses: actions/checkout@v7

    - name: Set reference data dir based on branch
      run: |
        # For pull_request events base_ref is the target branch name; for push/dispatch parse from ref.
        BRANCH="${{ github.base_ref }}"
        if [ -z "$BRANCH" ]; then
          BRANCH="${{ github.ref }}"
          BRANCH="${BRANCH#refs/heads/}"
        fi
        if [[ "$BRANCH" == release/* ]]; then
          VERSION="${BRANCH#release/}"
          MAJOR="${VERSION%.*}"
          MINOR="${VERSION##*.}"
          PREV_VERSION="${MAJOR}.$((MINOR - 1))"
          echo "ERIGON_REFERENCE_DATA_DIR=/opt/erigon-versions/gnosis-reference-version-${PREV_VERSION}/datadir" >> $GITHUB_ENV
        else
          echo "ERIGON_REFERENCE_DATA_DIR=/opt/erigon-versions/gnosis-reference-version/datadir" >> $GITHUB_ENV
        fi

    - name: Clean Erigon Build Directory
      run: |
        make clean

    - name: Build Erigon
      run: |
        make erigon
      working-directory: ${{ github.workspace }}

    - name: Pause the Erigon instance dedicated to db maintenance
      run: |
        python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true

    # Preparation steps

    - name: Run previous Erigon version and wait for sync (stabilization step)
      id: pre_test_step
      run: |
        set +e # Disable exit on error
        
        # Launch the testbed Erigon instance & test its ability to maintain sync for 2 minutes
        python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
          ${{ env.ERIGON_REFERENCE_DATA_DIR }}/../ $ERIGON_REFERENCE_DATA_DIR 120 $TOTAL_TIME_SECONDS Erigon3 $CHAIN
        
        # Capture monitoring script exit status
        test_exit_status=$?
        
        # Save the subsection reached status
        echo "test_executed=true" >> $GITHUB_OUTPUT
        
        # Clean up Erigon process if it's still running
        if kill -0 $ERIGON_PID 2> /dev/null; then
          echo "Terminating Erigon"
          kill $ERIGON_PID
          wait $ERIGON_PID
        fi

        # Check test runner script exit status
        if [ $test_exit_status -eq 0 ]; then
          echo "Pre-sync step completed successfully"
          echo "::notice::Pre-sync step completed successfully"
          echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
        else
          echo "Pre-sync step encountered an error"
          echo "::error::Pre-sync step encountered an error"
          echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
          exit 1
        fi

    - name: Restore Erigon Testbed Data Directory & Erigon binary
      id: save_chaindata_step
      run: |
        rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/
        cp $ERIGON_REFERENCE_DATA_DIR/../erigon $ERIGON_TESTBED_DATA_DIR/../
        echo "chaindata_saved=true" >> $GITHUB_OUTPUT

    # Upgrade & track the chain tip

    - name: Run Erigon, wait sync and check ability to maintain sync
      id: test_step
      run: |
        set +e # Disable exit on error
        
        # Launch the testbed Erigon instance & test its ability to maintain sync
        python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
          ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 $CHAIN
  
        # Capture monitoring script exit status
        test_exit_status=$?
        
        # Save the subsection reached status
        echo "test_executed=true" >> $GITHUB_OUTPUT

        # Check test runner script exit status
        if [ $test_exit_status -eq 0 ]; then
          echo "Upgrade & tip-tracking test completed successfully"
          echo "::notice::Upgrade & tip-tracking test completed successfully"
          echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
        else
          echo "Upgrade & tip-tracking test encountered an error"
          echo "::error::Upgrade & tip-tracking test encountered an error"
          echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
          exit 1
        fi

    - name: Upload Downloader Torrent Client Status
      if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }}
      uses: actions/upload-artifact@v7
      with:
        name: torrent-client-status-${{ env.CHAIN }}
        path: torrent-client-status.txt

    - name: Save test results
      if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }}
      env:
        TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }}
      run: |
        db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit)
        if [ -z "$db_version" ]; then
          db_version="no-version"
        fi
        
        python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
          --repo erigon \
          --commit $(git rev-parse HEAD) \
          --branch ${{ github.ref_name }} \
          --test_name tip-tracking \
          --chain $CHAIN \
          --runner ${{ runner.name }} \
          --db_version $db_version \
          --outcome $TEST_RESULT \
          --result_file ${{ github.workspace }}/result-$CHAIN.json

    - name: Upload test results
      if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }}
      uses: actions/upload-artifact@v7
      with:
        name: test-results
        path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json

    - name: Upload erigon logs
      if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }}
      uses: actions/upload-artifact@v7
      with:
        name: erigon-logs
        path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/

    - name: Upload metric plots
      if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }}
      uses: actions/upload-artifact@v7
      with:
        name: metric-plots
        path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots*

    - name: Upload FD Leak Analysis
      if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }}
      uses: actions/upload-artifact@v7
      with:
        name: fd-leak-analysis
        path: ${{ github.workspace }}/fd-leak-analysis-${{ env.CHAIN }}.md

    - name: Delete Erigon Testbed Data Directory
      if: ${{ always() && steps.save_chaindata_step.outputs.chaindata_saved == 'true' }}
      run: |
        if [ -d "$ERIGON_TESTBED_DATA_DIR" ]; then
          rm -rf $ERIGON_TESTBED_DATA_DIR
        fi

    - name: Resume the Erigon instance dedicated to db maintenance
      if: ${{ always() }}
      run: |
        python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true