[branch-out] Quarantine Flaky Tests - 2025-10-18 #8312
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CRE Workflow DON Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| chainlink_image_tag: | |
| required: true | |
| type: string | |
| description: "The tag of the Chainlink image to use for the tests." | |
| chainlink_version: | |
| required: true | |
| type: string | |
| description: "The version of Chainlink repository to use for the tests." | |
| default: "develop" | |
| workflow_call: | |
| inputs: | |
| chainlink_image_tag: | |
| required: true | |
| type: string | |
| description: "The tag of the Chainlink image to use for the tests." | |
| chainlink_version: | |
| required: true | |
| type: string | |
| description: "The version of Chainlink repository to use for the tests." | |
| default: "develop" | |
| pull_request: | |
| types: [labeled, synchronize] | |
| jobs: | |
| workflow-don-benchmark: | |
| runs-on: ubuntu-latest | |
| environment: "integration" | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event_name == 'workflow_call' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-cre-workflow-don-benchmark')) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.sha || inputs.chainlink_version }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: core/scripts/go.mod | |
| # We need to login to ECR to allow the test to pull the Job Distributor and Chainlink images | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 | |
| with: | |
| aws-region: ${{ secrets.QA_AWS_REGION }} | |
| role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| role-duration-seconds: 1800 | |
| mask-aws-account-id: true | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 | |
| with: | |
| registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.AWS_ACCOUNT_ID_PROD) }} | |
| env: | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| - name: Set date variable | |
| id: set-date | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: echo "date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT | |
| - name: Build Image | |
| if: github.event_name == 'pull_request' | |
| id: build-chainlink-image | |
| uses: smartcontractkit/.github/actions/ctf-build-image@ctf-build-image/v1 | |
| with: | |
| image-tag: ${{ format('PR-{0}-{1}',github.event.pull_request.number, github.sha ) }} | |
| dockerfile: core/chainlink.Dockerfile | |
| docker-registry-url: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com | |
| docker-repository-name: 'chainlink' | |
| docker-additional-build-args: | | |
| CL_INSTALL_PRIVATE_PLUGINS=true | |
| CL_INSTALL_TESTING_PLUGINS=true | |
| aws-account-number: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| aws-region: ${{ secrets.QA_AWS_REGION }} | |
| aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| gati-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }} | |
| gati-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }} | |
| - name: Set up cAdvisor | |
| shell: bash | |
| run: | | |
| sudo docker run \ | |
| --volume=/:/rootfs:ro \ | |
| --volume=/var/run:/var/run:ro \ | |
| --volume=/sys:/sys:ro \ | |
| --volume=/var/lib/docker/:/var/lib/docker:ro \ | |
| --volume=/dev/disk/:/dev/disk:ro \ | |
| --network host \ | |
| --detach=true \ | |
| --name=cadvisor \ | |
| --privileged \ | |
| --device=/dev/kmsg \ | |
| gcr.io/cadvisor/cadvisor:latest | |
| - name: Create Prometheus config | |
| shell: bash | |
| run: | | |
| cat > prometheus.yml<<EOF | |
| global: | |
| scrape_interval: 5s | |
| scrape_configs: | |
| - job_name: 'cadvisor' | |
| static_configs: | |
| - targets: ['localhost:8080'] | |
| EOF | |
| - name: Start Prometheus | |
| shell: bash | |
| run: | | |
| docker run -d \ | |
| --name prometheus \ | |
| --network host \ | |
| -p 9090:9090 \ | |
| -v ${{ github.workspace }}/prometheus.yml:/etc/prometheus/prometheus.yml:ro \ | |
| prom/prometheus:latest \ | |
| --config.file=/etc/prometheus/prometheus.yml | |
| - name: Start the test | |
| shell: bash | |
| env: | |
| E2E_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor" | |
| E2E_JD_VERSION: "0.9.0" | |
| E2E_TEST_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink" | |
| E2E_TEST_CHAINLINK_VERSION: ${{ github.event_name == 'pull_request' && format('PR-{0}-{1}',github.event.pull_request.number, github.sha ) || inputs.chainlink_image_tag }} | |
| # Anvil developer key, not a secret | |
| PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" | |
| CTF_CONFIGS: "workflow-don-benchmark-ci.toml" | |
| LOAD_TEST: "true" | |
| CI: "true" | |
| GH_SHA: ${{ github.sha }} | |
| # Use local prometheus | |
| PROMETHEUS_URL: "http://localhost:9090" | |
| run: | | |
| cd system-tests/tests/load/cre | |
| go test -run TestLoad_Workflow_Streams_MockCapabilities -timeout 20m 2>&1 | tee test_output.log | |
| - name: Parse logs for performance regressions | |
| if: always() | |
| id: parse-regressions | |
| run: | | |
| cd system-tests/tests/load/cre | |
| # Count the number of regressions - handle exit code properly | |
| if grep -c "PERFORMANCE REGRESSION:" test_output.log > /tmp/count.txt; then | |
| regression_count=$(cat /tmp/count.txt) | |
| else | |
| regression_count=0 | |
| fi | |
| echo "regression_count=${regression_count}" >> $GITHUB_OUTPUT | |
| if [ $regression_count -gt 0 ]; then | |
| echo "::warning::$regression_count performance regressions detected" | |
| echo "See the regression_reports artifact for details" | |
| # Extract detailed PERFORMANCE REGRESSION lines | |
| grep -o "PERFORMANCE REGRESSION.*)" test_output.log | awk '{printf "%s\ ", $0}' > logs/regression_details.txt | |
| else | |
| echo "No performance regressions detected" | |
| mkdir -p logs | |
| touch logs/regression_details.txt | |
| fi | |
| - name: Upload node logs | |
| uses: actions/upload-artifact@v4 | |
| timeout-minutes: 2 | |
| continue-on-error: true | |
| with: | |
| name: node logs | |
| path: | | |
| system-tests/tests/load/cre/logs | |
| retention-days: 5 | |
| - name: Upload performance reports | |
| uses: actions/upload-artifact@v4 | |
| timeout-minutes: 2 | |
| continue-on-error: true | |
| with: | |
| name: performance reports | |
| path: | | |
| system-tests/tests/load/cre/performance_reports | |
| retention-days: 30 | |
| - name: Create regression slack payload | |
| if: success() && steps.parse-regressions.outputs.regression_count > 0 | |
| id: create-regression-payload | |
| run: | | |
| cd system-tests/tests/load/cre | |
| # Read regression details | |
| REGRESSION_DETAILS=$(cat logs/regression_details.txt) | |
| # Create JSON payload with properly escaped details | |
| cat > $GITHUB_WORKSPACE/slack_payload.json << EOF | |
| { | |
| "channel": "C094W1V9A5N", | |
| "text": "Performance Regressions Detected", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*:rotating_light: ${{steps.parse-regressions.outputs.regression_count}} Performance Regressions Detected :rotating_light:*" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Performance regression for commit <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> on run ID <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*Regression Details:*\\n $REGRESSION_DETAILS" | |
| } | |
| } | |
| ] | |
| } | |
| EOF | |
| - name: Send regression slack notification | |
| if: success() && steps.parse-regressions.outputs.regression_count > 0 | |
| uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 | |
| with: | |
| errors: "true" | |
| method: chat.postMessage | |
| token: ${{ secrets.QA_SLACK_API_KEY }} | |
| payload-file-path: ${{ github.workspace }}/slack_payload.json | |
| - name: Send failure slack notification | |
| if: failure() | |
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | |
| with: | |
| errors: "true" | |
| method: chat.postMessage | |
| token: ${{ secrets.QA_SLACK_API_KEY }} | |
| payload: | | |
| { | |
| "channel": "C094W1V9A5N", | |
| "text": "Test failure", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*:rotating_light: Test failure :rotating_light:*" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Test failure for commit <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> on run ID <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>. ${{ github.event_name != 'pull_request' && 'This commit was already merged to develop.' || 'This is from a pull request.' }}" | |
| } | |
| } | |
| ] | |
| } | |
| - name: Teardown | |
| if: always() | |
| run: | |