Skip to content

Commit 521e772

Browse files
kalverrakalverra
authored
Fix Flakeguard race attribution (#17119)
* Fix flakeguard race attribution * Update flakeguard version
1 parent 2c75d21 commit 521e772

File tree

3 files changed

+51
-48
lines changed

3 files changed

+51
-48
lines changed

‎.github/workflows/flakeguard.yml‎

Lines changed: 49 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ jobs:
136136
- name: Install flakeguard
137137
if: ${{ inputs.runAllTests == false }}
138138
shell: bash
139-
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984 # [email protected]
139+
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # [email protected]
140140

141141
- name: Find new or updated test packages
142142
if: ${{ inputs.runAllTests == false && env.RUN_CUSTOM_TEST_PACKAGES == '' }}
@@ -331,7 +331,7 @@ jobs:
331331

332332
- name: Install flakeguard
333333
shell: bash
334-
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984 # [email protected]
334+
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # [email protected]
335335

336336
- name: Run tests with flakeguard
337337
shell: bash
@@ -443,7 +443,7 @@ jobs:
443443

444444
- name: Install flakeguard
445445
shell: bash
446-
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984 # [email protected]
446+
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723 # [email protected]
447447

448448
- name: Aggregate Flakeguard Results
449449
id: results
@@ -487,7 +487,9 @@ jobs:
487487
488488
# Read the summary from the generated report
489489
summary=$(jq -c '.summary_data' ./flakeguard-report/all-test-report.json)
490+
unable_to_attribute_test=$(jq 'any(.results.[]; .test_name | startswith("UnableToAttribute"))')
490491
echo "summary=$summary" >> $GITHUB_OUTPUT
492+
echo "unable_to_attribute_test=$unable_to_attribute_test" >> $GITHUB_OUTPUT
491493
492494
- name: Upload All Test Report as Artifact
493495
if: ${{ (success() || failure()) && fromJSON(steps.results.outputs.summary).total_runs > 0 }}
@@ -685,45 +687,46 @@ jobs:
685687
]
686688
}
687689
688-
- name: Send general Slack message
689-
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
690-
if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && fromJSON(steps.results.outputs.summary).flaky_tests == 0 && fromJSON(steps.results.outputs.summary).total_tests > 0 }}
691-
id: slack
692-
env:
693-
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
694-
with:
695-
channel-id: ${{ inputs.slackNotificationAfterTestsChannelId }}
696-
payload: |
697-
{
698-
"attachments": [
699-
{
700-
"color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '2E7D32' }}",
701-
"blocks": [
702-
{
703-
"type": "section",
704-
"text": {
705-
"type": "mrkdwn",
706-
"text": "Flaky Test Detector for `${{ steps.set_project_path_pretty.outputs.path }}` project - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Was cancelled :warning:' || 'Passed :white_check_mark:' }}"
707-
}
708-
},
709-
{
710-
"type": "section",
711-
"text": {
712-
"type": "mrkdwn",
713-
"text": "${{ inputs.runAllTests == true && format('Ran all tests for `{0}` branch.', env.GIT_HEAD_REF) || format('Ran changed tests between `{0}` and `{1}` (`{2}`).', env.GIT_BASE_REF, needs.get-tests.outputs.git_head_short_sha, env.GIT_HEAD_REF) }}"
714-
}
715-
},
716-
{
717-
"type": "section",
718-
"text": {
719-
"type": "mrkdwn",
720-
"text": "${{ inputs.runAllTests == true && format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details>', github.server_url, github.repository, github.run_id) || format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details> | <{3}/compare/{4}...{5}#files_bucket|Compare Changes>{6}', github.server_url, github.repository, github.run_id, inputs.repoUrl, inputs.baseRef, needs.get-tests.outputs.git_head_sha, github.event_name == 'pull_request' && format(' | <{0}|View PR>', github.event.pull_request.html_url) || '') }}"
721-
}
722-
}
723-
]
724-
}
725-
]
726-
}
690+
# This has mostly been noise. We can re-enable it if we find it necessary
691+
# - name: Send general Slack message
692+
# uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
693+
# if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && fromJSON(steps.results.outputs.summary).flaky_tests == 0 && fromJSON(steps.results.outputs.summary).total_tests > 0 }}
694+
# id: slack
695+
# env:
696+
# SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
697+
# with:
698+
# channel-id: ${{ inputs.slackNotificationAfterTestsChannelId }}
699+
# payload: |
700+
# {
701+
# "attachments": [
702+
# {
703+
# "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || contains(join(needs.*.result, ','), 'cancelled') && '#FFA000' || '2E7D32' }}",
704+
# "blocks": [
705+
# {
706+
# "type": "section",
707+
# "text": {
708+
# "type": "mrkdwn",
709+
# "text": "Flaky Test Detector for `${{ steps.set_project_path_pretty.outputs.path }}` project - ${{ contains(join(needs.*.result, ','), 'failure') && 'Failed :x:' || contains(join(needs.*.result, ','), 'cancelled') && 'Was cancelled :warning:' || 'Passed :white_check_mark:' }}"
710+
# }
711+
# },
712+
# {
713+
# "type": "section",
714+
# "text": {
715+
# "type": "mrkdwn",
716+
# "text": "${{ inputs.runAllTests == true && format('Ran all tests for `{0}` branch.', env.GIT_HEAD_REF) || format('Ran changed tests between `{0}` and `{1}` (`{2}`).', env.GIT_BASE_REF, needs.get-tests.outputs.git_head_short_sha, env.GIT_HEAD_REF) }}"
717+
# }
718+
# },
719+
# {
720+
# "type": "section",
721+
# "text": {
722+
# "type": "mrkdwn",
723+
# "text": "${{ inputs.runAllTests == true && format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details>', github.server_url, github.repository, github.run_id) || format('<{0}/{1}/actions/runs/{2}|View Flaky Detector Details> | <{3}/compare/{4}...{5}#files_bucket|Compare Changes>{6}', github.server_url, github.repository, github.run_id, inputs.repoUrl, inputs.baseRef, needs.get-tests.outputs.git_head_sha, github.event_name == 'pull_request' && format(' | <{0}|View PR>', github.event.pull_request.html_url) || '') }}"
724+
# }
725+
# }
726+
# ]
727+
# }
728+
# ]
729+
# }
727730

728731
- name: Send Slack message for Flakeguard Errors
729732
if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && (steps.check-errors.conclusion == 'failure' || steps.results.conclusion == 'failure' || steps.generate-report.conclusion == 'failure') }}
@@ -757,8 +760,8 @@ jobs:
757760
]
758761
}
759762
760-
- name: Send Slack message for Found Panics
761-
if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && fromJSON(steps.results.outputs.summary).panicked_tests > 0 }}
763+
- name: Send Slack message for UnableToAttribute tests
764+
if: ${{ (success() || failure()) && inputs.slackNotificationAfterTestsChannelId != '' && steps.results.outputs.summary == 'true' }}
762765
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
763766
env:
764767
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -774,14 +777,14 @@ jobs:
774777
"type": "header",
775778
"text": {
776779
"type": "plain_text",
777-
"text": "Flakeguard Found a Panic :warning:"
780+
"text": "Flakeguard Unable to Attribute a Test Due to Panic or Race :warning:"
778781
}
779782
},
780783
{
781784
"type": "section",
782785
"text": {
783786
"type": "mrkdwn",
784-
"text": "<@U01Q4N37KFG> ${{ format('<{0}/{1}/actions/runs/{2}|See details>', github.server_url, github.repository, github.run_id) }} and add to panic detection scenarios."
787+
"text": "<@U01Q4N37KFG> <@U04DYU1KLGJ> ${{ format('<{0}/{1}/actions/runs/{2}|See details>', github.server_url, github.repository, github.run_id) }} and add to panic detection scenarios."
785788
}
786789
}
787790
]

‎tools/bin/go_core_ccip_deployment_tests‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ EXTRA_FLAGS=""
88

99
if [[ -n "$USE_FLAKEGUARD" ]]; then
1010
# Install flakeguard
11-
go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@ec5b10aff48c9f4b787d225be30ddf1c0af32645
11+
go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723
1212
# Install gotestsum to parse JSON test outputs from flakeguard to console outputs
1313
go install gotest.tools/gotestsum@latest
1414

‎tools/bin/go_core_tests‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ EXTRA_FLAGS="-timeout 20m"
99

1010
if [[ -n "$USE_FLAKEGUARD" ]]; then
1111
# Install flakeguard
12-
go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@b08cab0f49aa13d5adb849554b56086f4dba8984
12+
go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@7f8fa09251be87f51357ffc517fd93cd408f5723
1313
# Install gotestsum to parse JSON test outputs from flakeguard to console outputs
1414
go install gotest.tools/gotestsum@latest
1515
# Make sure bins are in PATH

0 commit comments

Comments
 (0)