QA - Tip tracking & migration #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QA - Tip tracking & migration | |
| on: | |
| schedule: | |
| - cron: '0 1 * * 0' # Run on Sunday at 01:00 AM UTC | |
| push: | |
| branches: | |
| - 'release/3.*' | |
| workflow_dispatch: # Run manually | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| mainnet-tip-tracking-test: | |
| runs-on: [self-hosted, qa, Ethereum, tip-tracking] | |
| timeout-minutes: 1300 # 21.66667 hours | |
| env: | |
| ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version-3.3/datadir | |
| ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir | |
| ERIGON_QA_PATH: /home/qarunner/erigon-qa | |
| TRACKING_TIME_SECONDS: 7200 # 2 hours | |
| TOTAL_TIME_SECONDS: 36000 # 10 hours | |
| CHAIN: mainnet | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| - name: Clean Erigon Build Directory | |
| run: | | |
| make clean | |
| - name: Build Erigon | |
| run: | | |
| make erigon | |
| working-directory: ${{ github.workspace }} | |
| - name: Pause the Erigon instance dedicated to db maintenance | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true | |
| # Preparation steps | |
| - name: Run previous Erigon version and wait for sync (stabilization step) | |
| id: pre_test_step | |
| run: | | |
| set +e # Disable exit on error | |
| # Launch the testbed Erigon instance & test its ability to maintain sync for 2 minutes | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \ | |
| ${{ env.ERIGON_REFERENCE_DATA_DIR }}/../ $ERIGON_REFERENCE_DATA_DIR 120 $TOTAL_TIME_SECONDS Erigon3 $CHAIN | |
| # Capture monitoring script exit status | |
| test_exit_status=$? | |
| # Save the subsection reached status | |
| echo "test_executed=true" >> $GITHUB_OUTPUT | |
| # Clean up Erigon process if it's still running | |
| if kill -0 $ERIGON_PID 2> /dev/null; then | |
| echo "Terminating Erigon" | |
| kill $ERIGON_PID | |
| wait $ERIGON_PID | |
| fi | |
| # Check test runner script exit status | |
| if [ $test_exit_status -eq 0 ]; then | |
| echo "Pre-sync step completed successfully" | |
| echo "::notice::Pre-sync step completed successfully" | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Pre-sync step encountered an error" | |
| echo "::error::Pre-sync step encountered an error" | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| fi | |
| - name: Restore Erigon Testbed Data Directory & Erigon binary | |
| id: save_chaindata_step | |
| run: | | |
| rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/ | |
| cp $ERIGON_REFERENCE_DATA_DIR/../erigon $ERIGON_TESTBED_DATA_DIR/../ | |
| echo "chaindata_saved=true" >> $GITHUB_OUTPUT | |
| # Upgrade & track the chain tip | |
| - name: Run Erigon, wait sync and check ability to maintain sync | |
| id: test_step | |
| run: | | |
| set +e # Disable exit on error | |
| # Launch the testbed Erigon instance & test its ability to maintain sync | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \ | |
| ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 $CHAIN | |
| # Capture monitoring script exit status | |
| test_exit_status=$? | |
| # Save the subsection reached status | |
| echo "test_executed=true" >> $GITHUB_OUTPUT | |
| # Check test runner script exit status | |
| if [ $test_exit_status -eq 0 ]; then | |
| echo "Upgrade & tip-tracking test completed successfully" | |
| echo "::notice::Upgrade & tip-tracking test completed successfully" | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Upgrade & tip-tracking test encountered an error" | |
| echo "::error::Upgrade & tip-tracking test encountered an error" | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| fi | |
| - name: Upload Downloader Torrent Client Status | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: torrent-client-status-${{ env.CHAIN }} | |
| path: torrent-client-status.txt | |
| - name: Save test results | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| env: | |
| TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} | |
| run: | | |
| db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit) | |
| if [ -z "$db_version" ]; then | |
| db_version="no-version" | |
| fi | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \ | |
| --repo erigon \ | |
| --commit $(git rev-parse HEAD) \ | |
| --branch ${{ github.ref_name }} \ | |
| --test_name tip-tracking \ | |
| --chain $CHAIN \ | |
| --runner ${{ runner.name }} \ | |
| --db_version $db_version \ | |
| --outcome $TEST_RESULT \ | |
| --result_file ${{ github.workspace }}/result-$CHAIN.json | |
| - name: Upload test results | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-results | |
| path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json | |
| - name: Upload erigon logs | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: erigon-logs | |
| path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/ | |
| - name: Upload metric plots | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: metric-plots | |
| path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots* | |
| - name: Upload FD Leak Analysis | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: fd-leak-analysis | |
| path: ${{ github.workspace }}/fd-leak-analysis-${{ env.CHAIN }}.md | |
| - name: Delete Erigon Testbed Data Directory | |
| if: ${{ always() && steps.save_chaindata_step.outputs.chaindata_saved == 'true' }} | |
| run: | | |
| if [ -d "$ERIGON_TESTBED_DATA_DIR" ]; then | |
| rm -rf $ERIGON_TESTBED_DATA_DIR | |
| fi | |
| - name: Resume the Erigon instance dedicated to db maintenance | |
| if: ${{ always() }} | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true | |