QA - Sync with external CL #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QA - Sync with external CL | |
| on: | |
| push: | |
| branches: | |
| - 'release/3.*' | |
| schedule: | |
| - cron: "0 8 * * 0" # Run on Sunday at 08:00 AM UTC | |
| workflow_dispatch: # Run manually | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| sync-with-externalcl: | |
| runs-on: [self-hosted, qa, linux, X64, long-running] | |
| timeout-minutes: 1440 # 24 hours | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| client: [lighthouse, prysm] | |
| chain: [mainnet, gnosis] | |
| exclude: | |
| - client: prysm | |
| chain: gnosis | |
| env: | |
| ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data | |
| CL_DATA_DIR: ${{ github.workspace }}/consensus | |
| ERIGON_QA_PATH: /home/qarunner/erigon-qa | |
| TRACKING_TIME_SECONDS: 3600 # 1 hour | |
| TOTAL_TIME_SECONDS: 28800 # 8 hours | |
| ERIGON_ASSERT: true | |
| LIGHTHOUSE_VERSION: v8.1.0 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| - name: Clean Erigon Build & Data Directories | |
| run: | | |
| make clean | |
| rm -rf $ERIGON_DATA_DIR | |
| - name: Install ${{ matrix.client }} and generate JWT secret | |
| env: | |
| LIGHTHOUSE_FILE: lighthouse-${{ env.LIGHTHOUSE_VERSION }}-x86_64-unknown-linux-gnu.tar.gz | |
| run: | | |
| mkdir -p $CL_DATA_DIR | |
| if [ "${{ matrix.client }}" == "lighthouse" ]; then | |
| echo "Downloading Lighthouse: version $LIGHTHOUSE_VERSION - file $LIGHTHOUSE_FILE" | |
| curl -LO https://github.com/sigp/lighthouse/releases/download/$LIGHTHOUSE_VERSION/$LIGHTHOUSE_FILE | |
| tar -xvf $LIGHTHOUSE_FILE -C $CL_DATA_DIR | |
| rm $LIGHTHOUSE_FILE | |
| elif [ "${{ matrix.client }}" == "prysm" ]; then | |
| echo "Downloading Prysm: automatically using the latest version via the prysm.sh script" | |
| curl -L https://raw.githubusercontent.com/prysmaticlabs/prysm/master/prysm.sh -o $CL_DATA_DIR/prysm.sh | |
| chmod +x $CL_DATA_DIR/prysm.sh | |
| fi | |
| echo "Generating JWT secret for communication between Erigon and ${{ matrix.client }}" | |
| openssl rand -hex 32 > $CL_DATA_DIR/jwt.hex | |
| - name: Build Erigon | |
| run: | | |
| make erigon | |
| working-directory: ${{ github.workspace }} | |
| - name: Pause the Erigon instance dedicated to db maintenance | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true | |
| - name: Run Erigon and monitor chain sync | |
| id: test_step | |
| run: | | |
| set +e # Disable exit on error | |
| # Run Erigon, wait sync and check ability to maintain sync | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \ | |
| ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 ${{ matrix.chain }} minimal_node no_statistics ${{ matrix.client }} $CL_DATA_DIR | |
| # Capture monitoring script exit status | |
| test_exit_status=$? | |
| # Save the subsection reached status | |
| echo "test_executed=true" >> $GITHUB_OUTPUT | |
| # Check test runner script exit status | |
| if [ $test_exit_status -eq 0 ]; then | |
| echo "Tests completed successfully" | |
| echo "::notice::Tests completed successfully" | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Error detected during tests" | |
| echo "::error::Error detected during tests" | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| fi | |
| - name: Save test results | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| env: | |
| TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \ | |
| --repo erigon \ | |
| --commit $(git rev-parse HEAD) \ | |
| --branch ${{ github.ref_name }} \ | |
| --test_name sync-from-scratch-${{ matrix.client }}-minimal-node \ | |
| --chain ${{ matrix.chain }} \ | |
| --runner ${{ runner.name }} \ | |
| --outcome $TEST_RESULT \ | |
| --result_file ${{ github.workspace }}/result-${{ matrix.chain }}.json | |
| - name: Upload test results (Lighthouse) | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' && matrix.client == 'lighthouse' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-results-${{ matrix.client }}-${{ matrix.chain }} | |
| path: | | |
| ${{ github.workspace }}/result-${{ matrix.chain }}.json | |
| ${{ github.workspace }}/erigon_data/logs/ | |
| ${{ github.workspace }}/consensus/data/beacon/logs/beacon.log | |
| - name: Upload test results (Prysm) | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' && matrix.client == 'prysm' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-results-${{ matrix.client }}-${{ matrix.chain }} | |
| path: | | |
| ${{ github.workspace }}/result-${{ matrix.chain }}.json | |
| ${{ github.workspace }}/erigon_data/logs/ | |
| ${{ github.workspace }}/consensus/data/beacon.log | |
| - name: Upload metric plots | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: metric-plots--${{ matrix.client }}-${{ matrix.chain }} | |
| path: ${{ github.workspace }}/metrics-${{ matrix.chain }}-plots* | |
| - name: Upload FD Leak Analysis | |
| if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: fd-leak-analysis-${{ env.CHAIN }} | |
| path: ${{ github.workspace }}/fd-leak-analysis-${{ env.CHAIN }}.md | |
| - name: Clean up Erigon data directory | |
| if: always() | |
| run: | | |
| rm -rf $ERIGON_DATA_DIR | |
| - name: Cleanup consensus runner directory | |
| if: always() | |
| run: | | |
| rm -rf $CL_DATA_DIR | |
| - name: Resume the Erigon instance dedicated to db maintenance | |
| if: always() | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true | |
| - name: Display Error Reason | |
| if: always() | |
| run: | | |
| if [ -f "${{ github.workspace }}/result-${{ matrix.chain }}.json" ]; then | |
| OUTCOME=$(jq -r '.outcome' "${{ github.workspace }}/result-${{ matrix.chain }}.json" 2>/dev/null || echo "UNKNOWN") | |
| if [ "$OUTCOME" != "SUCCESS" ]; then | |
| REASON=$(jq -r '.reason // "No reason provided"' "${{ github.workspace }}/result-${{ matrix.chain }}.json") | |
| echo "::error::Test failed with reason: $REASON" | |
| fi | |
| fi |