QA - Tip tracking & migration #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QA - Tip tracking & migration | |
| on: | |
| schedule: | |
| - cron: '0 1 * * 0' # Run on Sunday at 01:00 AM UTC | |
| push: | |
| branches: | |
| - 'release/3.*' | |
| workflow_dispatch: # Run manually | |
| inputs: | |
| explicit_upgrade: | |
| description: 'If true, perform explicit upgrade steps (by default they happen automatically)' | |
| type: boolean | |
| required: false | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| mainnet-tip-tracking-test: | |
| runs-on: [self-hosted, qa, Ethereum, tip-tracking] | |
| timeout-minutes: 1300 # 21.66667 hours | |
| env: | |
| ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version-3.2/datadir | |
| ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir | |
| ERIGON_QA_PATH: /home/qarunner/erigon-qa | |
| TRACKING_TIME_SECONDS: 7200 # 2 hours | |
| TOTAL_TIME_SECONDS: 36000 # 10 hours | |
| CHAIN: mainnet | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v5 | |
| - name: Clean Erigon Build Directory | |
| run: | | |
| make clean | |
| - name: Build Erigon | |
| run: | | |
| make erigon | |
| working-directory: ${{ github.workspace }} | |
| - name: Pause the Erigon instance dedicated to db maintenance | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true | |
| - name: Run previous Erigon version and wait for sync (stabilization step) | |
| id: pre_test_step | |
| run: | | |
| set +e # Disable exit on error | |
| # Launch the testbed Erigon instance & test its ability to maintain sync for 2 minutes | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \ | |
| ${{ env.ERIGON_REFERENCE_DATA_DIR }}/../ $ERIGON_REFERENCE_DATA_DIR 120 $TOTAL_TIME_SECONDS Erigon3 $CHAIN | |
| # Capture monitoring script exit status | |
| test_exit_status=$? | |
| # Save the subsection reached status | |
| echo "test_executed=true" >> $GITHUB_OUTPUT | |
| # Clean up Erigon process if it's still running | |
| if kill -0 $ERIGON_PID 2> /dev/null; then | |
| echo "Terminating Erigon" | |
| kill $ERIGON_PID | |
| wait $ERIGON_PID | |
| fi | |
| # Check test runner script exit status | |
| if [ $test_exit_status -eq 0 ]; then | |
| echo "Pre-sync step completed successfully" | |
| echo "::notice::Pre-sync step completed successfully" | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Pre-sync step encountered an error" | |
| echo "::error::Pre-sync step encountered an error" | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| fi | |
| - name: Restore Erigon Testbed Data Directory & Erigon binary | |
| id: save_chaindata_step | |
| run: | | |
| rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/ | |
| cp $ERIGON_REFERENCE_DATA_DIR/../erigon $ERIGON_TESTBED_DATA_DIR/../ | |
| # Upgrade + tip-tracking test | |
| - name: Print datadir contents before upgrade (for debugging) | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| find $ERIGON_TESTBED_DATA_DIR | |
| # The following task runs the datadir upgrade procedure explicitly. | |
| # It is unnecessary and is executed automatically by erigon when it detects the old format. | |
| - name: Run the datadir upgrade procedure | |
| if: ${{ inputs.explicit_upgrade }} | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| ./build/bin/erigon snapshots update-to-new-ver-format --datadir $ERIGON_TESTBED_DATA_DIR | |
| - name: Print datadir contents after upgrade (for debugging) | |
| if: ${{ inputs.explicit_upgrade }} | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| find $ERIGON_TESTBED_DATA_DIR | |
| - name: Run Erigon, wait sync and check ability to maintain sync | |
| id: test_step | |
| run: | | |
| set +e # Disable exit on error | |
| # Launch the testbed Erigon instance & test its ability to maintain sync | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \ | |
| ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 $CHAIN | |
| # Capture monitoring script exit status | |
| test_exit_status=$? | |
| # Save the subsection reached status | |
| echo "test_executed=true" >> $GITHUB_OUTPUT | |
| # Clean up Erigon process if it's still running | |
| if kill -0 $ERIGON_PID 2> /dev/null; then | |
| echo "Terminating Erigon" | |
| kill $ERIGON_PID | |
| wait $ERIGON_PID | |
| fi | |
| # Check test runner script exit status | |
| if [ $test_exit_status -eq 0 ]; then | |
| echo "Upgrade & tip-tracking test completed successfully" | |
| echo "::notice::Upgrade & tip-tracking test completed successfully" | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Upgrade & tip-tracking test encountered an error" | |
| echo "::error::Upgrade & tip-tracking test encountered an error" | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Upload Downloader Torrent Client Status | |
| if: always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: torrent-client-status | |
| path: torrent-client-status.txt | |
| - name: Save test results | |
| if: steps.test_step.outputs.test_executed == 'true' | |
| env: | |
| TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} | |
| run: | | |
| db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit) | |
| if [ -z "$db_version" ]; then | |
| db_version="no-version" | |
| fi | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \ | |
| --repo erigon \ | |
| --commit $(git rev-parse HEAD) \ | |
| --branch ${{ github.ref_name }} \ | |
| --test_name tip-tracking \ | |
| --chain $CHAIN \ | |
| --runner ${{ runner.name }} \ | |
| --db_version $db_version \ | |
| --outcome $TEST_RESULT \ | |
| --result_file ${{ github.workspace }}/result-$CHAIN.json | |
| - name: Upload test results | |
| if: steps.test_step.outputs.test_executed == 'true' | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: test-results | |
| path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json | |
| - name: Upload erigon logs | |
| if: steps.test_step.outputs.test_executed == 'true' | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: erigon-logs | |
| path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/ | |
| - name: Upload metric plots | |
| if: steps.test_step.outputs.test_executed == 'true' | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: metric-plots | |
| path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots* | |
| # Downgrade + tip-tracking test | |
| - name: Print datadir contents before downgrade (for debugging) | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| find $ERIGON_TESTBED_DATA_DIR | |
| - name: Run the datadir downgrade procedure | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| ./build/bin/erigon --datadir $ERIGON_TESTBED_DATA_DIR snapshots reset-to-old-ver-format | |
| - name: Print datadir contents after downgrade (for debugging) | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| find $ERIGON_TESTBED_DATA_DIR | |
| - name: Run previous Erigon version, wait sync and check ability to maintain sync | |
| id: downgrade_test_step | |
| run: | | |
| set +e # Disable exit on error | |
| # Launch the testbed Erigon instance & test its ability to maintain sync | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \ | |
| ${{ env.ERIGON_TESTBED_DATA_DIR }}/../ $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 $CHAIN | |
| # Capture monitoring script exit status | |
| test_exit_status=$? | |
| # Save the subsection reached status | |
| echo "test_executed=true" >> $GITHUB_OUTPUT | |
| # Clean up Erigon process if it's still running | |
| if kill -0 $ERIGON_PID 2> /dev/null; then | |
| echo "Terminating Erigon" | |
| kill $ERIGON_PID | |
| wait $ERIGON_PID | |
| fi | |
| # Check test runner script exit status | |
| if [ $test_exit_status -eq 0 ]; then | |
| echo "Downgrade & tip-tracking test completed successfully" | |
| echo "::notice::Downgrade & tip-tracking test completed successfully" | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Downgrade tests encountered an error" | |
| echo "::error::Downgrade & tip-tracking test encountered an error" | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Upload Downloader Torrent Client Status of the downgrade test | |
| if: always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: torrent-client-status-downgrade-test | |
| path: torrent-client-status.txt | |
| - name: Upload erigon logs of the downgrade test | |
| if: steps.downgrade_test_step.outputs.test_executed == 'true' | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: erigon-logs-downgrade-test | |
| path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/ | |
| - name: Delete Erigon Testbed Data Directory | |
| if: ${{ always() }} | |
| run: | | |
| if [ -d "$ERIGON_TESTBED_DATA_DIR" ]; then | |
| rm -rf $ERIGON_TESTBED_DATA_DIR | |
| fi | |
| - name: Resume the Erigon instance dedicated to db maintenance | |
| if: ${{ always() }} | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true | |
| - name: Action for Not Success | |
| if: ${{ always() }} | |
| run: | | |
| if [[ "${{ steps.test_step.outputs.TEST_RESULT }}" != "success" ]]; then | |
| echo "::error::Error detected during upgrade & tip-tracking test" | |
| fi | |
| if [[ "${{ steps.downgrade_test_step.outputs.TEST_RESULT }}" != "success" ]]; then | |
| echo "::error::Error detected during downgrade & tip-tracking test" | |
| fi | |
| if [[ "${{ steps.test_step.outputs.TEST_RESULT }}" != "success" ]] || \ | |
| [[ "${{ steps.downgrade_test_step.outputs.TEST_RESULT }}" != "success" ]]; then | |
| exit 1 | |
| fi | |