Report intermittent tests #1842
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Report intermittent tests | |
| on: | |
| schedule: | |
| - cron: "42 04 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| backend-build: | |
| name: Build backend 🏗️ | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| - name: Build backend | |
| run: ./gradlew classes jar bootJar buildTestEmails | |
| - name: Upload backend build result | |
| uses: ./.github/actions/upload-backend-build | |
| backend-test: | |
| name: BT 🔎 | |
| needs: [backend-build] | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| command: | |
| [ | |
| "server-app:runContextRecreatingTests", | |
| "server-app:runStandardTests", | |
| "server-app:runWebsocketTests", | |
| "server-app:runWithoutEeTests", | |
| "ee-test:test", | |
| "data:test", | |
| "security:test", | |
| "ktlint:test", | |
| ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| node: "false" | |
| - name: Download backend build result | |
| uses: ./.github/actions/download-backend-build | |
| - name: Remove ee | |
| if: matrix.command == 'server-app:runWithoutEeTests' | |
| run: | | |
| rm -rf ./ee | |
| - name: Run backend tests | |
| timeout-minutes: 30 | |
| run: ./gradlew ${{ matrix.command }} | |
| env: | |
| SKIP_SERVER_BUILD: true | |
| SKIP_EMAIL_BUILD: true | |
| DETECT_FLAKY_TESTS: true | |
| - name: Get report name | |
| id: reportName | |
| if: always() | |
| run: | | |
| reportName=${{ matrix.command }} | |
| echo "reportName=${reportName//\:/_}" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: backend_test_reports_${{ steps.reportName.outputs.reportName }} | |
| path: | | |
| ./**/build/reports/**/* | |
| - name: Test Report | |
| uses: dorny/test-reporter@v1 | |
| if: always() | |
| with: | |
| name: Test report for ${{ matrix.command }} | |
| path: "**/build/test-results/**/TEST-*.xml" | |
| fail-on-error: false | |
| reporter: java-junit | |
| - name: Collect flaky and failed tests | |
| if: always() | |
| continue-on-error: true | |
| run: python3 scripts/collect-flaky-tests.py "${{ steps.reportName.outputs.reportName }}" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: flaky_tests_${{ steps.reportName.outputs.reportName }} | |
| path: | | |
| flaky-tests-${{ steps.reportName.outputs.reportName }}.txt | |
| failed-tests-${{ steps.reportName.outputs.reportName }}.txt | |
| if-no-files-found: ignore | |
| frontend-build: | |
| name: Build frontend 🏗️ | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| - name: Build webapp | |
| run: ./gradlew buildWebapp | |
| env: | |
| TOLGEE_API_KEY: ${{secrets.TOLGEE_API_KEY}} | |
| TOLGEE_API_URL: ${{secrets.TOLGEE_API_URL}} | |
| - name: Upload built webapp | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: webapp | |
| path: webapp/dist | |
| e2e-install-deps: | |
| name: Install E2E dependencies ⬇️ | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| java: "false" | |
| - name: Install dependencies | |
| run: npm ci | |
| working-directory: ./e2e | |
| - name: Tar node modules | |
| run: | | |
| tar --zstd -cf ~/node_modules.tar.zst ./e2e/node_modules | |
| - name: Tar Cypress cache | |
| run: | | |
| tar --zstd -cf ~/cypress_cache.tar.zst ./Cypress | |
| working-directory: /home/runner/.cache/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e_deps | |
| path: | | |
| ~/node_modules.tar.zst | |
| ~/cypress_cache.tar.zst | |
| e2e: | |
| needs: [frontend-build, backend-build, e2e-install-deps] | |
| runs-on: ubuntu-24.04 | |
| name: E2E testing 🔎 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| total_jobs: [15] | |
| job_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| - name: Download backend build result | |
| uses: ./.github/actions/download-backend-build | |
| - name: Download frontend build result | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: webapp | |
| path: ./webapp/dist | |
| - name: Download dependencies | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e_deps | |
| path: ~/ | |
| - name: Untar node modules | |
| run: | | |
| tar --zstd -xf ~/node_modules.tar.zst ./e2e/node_modules | |
| - name: Create cache directory | |
| run: | | |
| mkdir -p ~/.cache | |
| - name: Untar Cypress cache | |
| run: | | |
| tar --zstd -xf ~/cypress_cache.tar.zst | |
| working-directory: /home/runner/.cache/ | |
| - name: Run e2e test | |
| timeout-minutes: 30 | |
| run: ./gradlew runE2e -x bootJar -x buildWebapp | |
| env: | |
| TOLGEE_API_KEY: ${{secrets.TOLGEE_API_KEY}} | |
| TOLGEE_API_URL: ${{secrets.TOLGEE_API_URL}} | |
| SKIP_WEBAPP_BUILD: true | |
| SKIP_SERVER_BUILD: true | |
| SKIP_EMAIL_BUILD: true | |
| SKIP_INSTALL_E2E_DEPS: true | |
| E2E_TOTAL_JOBS: ${{matrix.total_jobs}} | |
| E2E_JOB_INDEX: ${{matrix.job_index}} | |
| DETECT_FLAKY_TESTS: true | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: flaky_tests_e2e_${{ matrix.job_index }} | |
| path: | | |
| ./e2e/flaky-tests-e2e-${{ matrix.job_index }}.txt | |
| ./e2e/failed-tests-e2e-${{ matrix.job_index }}.txt | |
| if-no-files-found: ignore | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: e2e_${{ matrix.job_index }} | |
| path: | | |
| ./e2e/cypress/videos/**/* | |
| ./e2e/cypress/screenshots/**/* | |
| ./e2e/server | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: e2e_server_log_${{ matrix.job_index }} | |
| path: | | |
| ./e2e/server.log | |
| cleanup: | |
| if: always() | |
| needs: [e2e, backend-test] | |
| name: Delete artifacts 🧹 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: | | |
| e2e_deps | |
| webapp | |
| backend | |
| failOnError: false | |
| notify: | |
| name: Report flaky tests 🔔 | |
| needs: [backend-test, e2e] | |
| if: always() | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: flaky_tests_* | |
| merge-multiple: true | |
| path: ./flaky | |
| continue-on-error: true | |
| # Flaky tests (failed then passed on retry) and hard failures (failed | |
| # every retry) are collected into separate artifacts. Only flaky tests | |
| # are reported to the board; hard failures are surfaced in Slack only. | |
| - name: Summarize results | |
| id: summary | |
| run: | | |
| flaky="" | |
| failed="" | |
| if compgen -G './flaky/flaky-tests-*.txt' > /dev/null; then | |
| flaky=$(cat ./flaky/flaky-tests-*.txt | sed '/^[[:space:]]*$/d' | sort -u) | |
| fi | |
| if compgen -G './flaky/failed-tests-*.txt' > /dev/null; then | |
| failed=$(cat ./flaky/failed-tests-*.txt | sed '/^[[:space:]]*$/d' | sort -u) | |
| fi | |
| flaky_count=0 | |
| failed_count=0 | |
| if [ -n "$flaky" ]; then | |
| flaky_count=$(printf '%s\n' "$flaky" | wc -l | tr -d ' ') | |
| fi | |
| if [ -n "$failed" ]; then | |
| failed_count=$(printf '%s\n' "$failed" | wc -l | tr -d ' ') | |
| fi | |
| echo "Found $flaky_count flaky, $failed_count consistently failing tests" | |
| { | |
| echo "flaky_count=$flaky_count" | |
| echo "failed_count=$failed_count" | |
| echo 'flaky_list<<EOF' | |
| [ -n "$flaky" ] && printf '%s\n' "$flaky" | head -30 | sed 's/.*/• `&`/' | |
| echo 'EOF' | |
| echo 'failed_list<<EOF' | |
| [ -n "$failed" ] && printf '%s\n' "$failed" | head -30 | sed 's/.*/• `&`/' | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| # Always run so the sweep pass cleans up stale Suspected items even on | |
| # nights with no flaky detections. | |
| - name: Report flaky tests to project board | |
| id: project | |
| if: always() | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.PROJECTS_TOKEN }} | |
| PROJECT_OWNER: tolgee | |
| PROJECT_NUMBER: "4" | |
| TEAM_ORG: tolgee | |
| TEAM_SLUG: coders | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: python3 scripts/report-flaky-tests.py ./flaky . | |
| # Newly confirmed flakes and hard failures are the actionable signals; | |
| # raw per-run flaky counts (unpromoted Suspected items) don't trigger | |
| # Slack on their own. | |
| - name: Send Slack notification | |
| if: steps.project.outputs.newly_confirmed_count != '0' || steps.summary.outputs.failed_count != '0' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_FLAKY_TESTS_WEBHOOK_URL }} | |
| NEWLY_CONFIRMED_TESTS: ${{ steps.project.outputs.newly_confirmed_list }} | |
| NEWLY_CONFIRMED_COUNT: ${{ steps.project.outputs.newly_confirmed_count }} | |
| FAILED_TESTS: ${{ steps.summary.outputs.failed_list }} | |
| FAILED_COUNT: ${{ steps.summary.outputs.failed_count }} | |
| TOTAL_CONFIRMED: ${{ steps.project.outputs.total_confirmed_count }} | |
| TOTAL_SUSPECTED: ${{ steps.project.outputs.total_suspected_count }} | |
| DELETED_SUSPECT: ${{ steps.project.outputs.deleted_suspect_count }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| PROJECT_URL: ${{ steps.project.outputs.project_url }} | |
| run: | | |
| { | |
| if [ "${NEWLY_CONFIRMED_COUNT:-0}" != "0" ]; then | |
| echo "🟡 ${NEWLY_CONFIRMED_COUNT} newly confirmed flaky test(s) in *${{ github.repository }}*" | |
| echo "" | |
| echo "$NEWLY_CONFIRMED_TESTS" | |
| fi | |
| if [ "${FAILED_COUNT:-0}" != "0" ]; then | |
| if [ "${NEWLY_CONFIRMED_COUNT:-0}" != "0" ]; then echo ""; fi | |
| echo "🔴 ${FAILED_COUNT} test(s) failing on every retry — broken, not flaky" | |
| echo "" | |
| echo "$FAILED_TESTS" | |
| fi | |
| echo "" | |
| echo "<${RUN_URL}|View run>" | |
| if [ -n "$PROJECT_URL" ]; then | |
| board="<${PROJECT_URL}|Flaky tests board> — ${TOTAL_CONFIRMED:-0} confirmed, ${TOTAL_SUSPECTED:-0} suspected" | |
| if [ "${DELETED_SUSPECT:-0}" != "0" ]; then | |
| board="$board (${DELETED_SUSPECT} stale suspected cleaned up)" | |
| fi | |
| echo "$board" | |
| fi | |
| } > /tmp/slack-message.txt | |
| curl --fail --show-error --silent \ | |
| --connect-timeout 5 --max-time 10 \ | |
| --retry 3 --retry-delay 2 --retry-connrefused \ | |
| -X POST -H 'Content-Type: application/json' \ | |
| -d "$(jq -n --rawfile text /tmp/slack-message.txt '{text: $text}')" \ | |
| "$SLACK_WEBHOOK_URL" |