Update Muted tests #90
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Muted tests | |
| on: | |
| schedule: | |
| - cron: "0 * * * *" # Every hour at :00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| branches: | |
| description: 'Optional: comma-separated branches instead of stable_tests_branches.json. Empty = use stable_tests_branches.json from WORKFLOW_CHECKOUT_REF.' | |
| required: false | |
| default: '' | |
| build_types: | |
| description: 'Comma-separated presets or "all" (default). Cannot request presets outside .github/config/mute_config.json (default + per-branch overrides).' | |
| required: false | |
| default: all | |
| type: string | |
| env: | |
| # workflow_dispatch: branch selected in UI. schedule: repository default branch. | |
| WORKFLOW_CHECKOUT_REF: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || github.event.repository.default_branch }} | |
| BRANCHES_CONFIG_PATH: .github/config/stable_tests_branches.json | |
| MUTE_BUILD_TYPES_CONFIG_PATH: .github/config/mute_config.json | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_BRANCH_PREFIX: update-muted-ya | |
| REVIEWERS: '["ci"]' | |
| LABELS: "mute-unmute,not-for-changelog,ok-to-test,automerge" | |
| jobs: | |
| setup: | |
| runs-on: [ self-hosted, auto-provisioned, build-preset-analytic-node] | |
| outputs: | |
| matrix_include: ${{ steps.set-mute-matrix.outputs.matrix_include }} | |
| steps: | |
| - name: Checkout branch list and mute files | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ env.WORKFLOW_CHECKOUT_REF }} | |
| path: config-repo | |
| sparse-checkout: | | |
| .github/config/ | |
| .github/scripts/tests/mute/mute_helper.py | |
| .github/scripts/tests/mute/mute_utils.py | |
| sparse-checkout-cone-mode: false | |
| - id: set-mute-matrix | |
| env: | |
| INPUT_BRANCHES: ${{ github.event.inputs.branches || '' }} | |
| INPUT_BUILD_TYPES: ${{ github.event.inputs.build_types || '' }} | |
| run: | | |
| set -euo pipefail | |
| python3 config-repo/.github/scripts/tests/mute/mute_helper.py matrix \ | |
| --branches-file "config-repo/${{ env.BRANCHES_CONFIG_PATH }}" \ | |
| --build-types-config "config-repo/${{ env.MUTE_BUILD_TYPES_CONFIG_PATH }}" | |
| collect-testowners-and-sync: | |
| runs-on: [ self-hosted, auto-provisioned, build-preset-analytic-node] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ env.WORKFLOW_CHECKOUT_REF }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install ydb[yc] codeowners requests | |
| - name: Setup ydb access | |
| uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials | |
| with: | |
| ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} | |
| ydb_qa_config: ${{ vars.YDB_QA_CONFIG }} | |
| - name: Collect testowners | |
| run: python3 .github/scripts/analytics/upload_testowners.py | |
| # ``mute/manual_unmute.py sync`` selects CLOSED+COMPLETED candidates from YDB ``issues``; | |
| # refresh that table here so fast-unmute does not wait for collect_analytics_fast (~30 min). | |
| - name: Export GitHub issues to YDB | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.YDBOT_TOKEN }} | |
| run: python3 .github/scripts/analytics/export_issues_to_ydb.py | |
| - name: Sync manual fast-unmute state | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.YDBOT_TOKEN }} | |
| run: python3 .github/scripts/tests/mute/manual_unmute.py sync | |
| update-muted-tests: | |
| needs: [setup, collect-testowners-and-sync] | |
| runs-on: [ self-hosted, auto-provisioned, build-preset-analytic-node] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.setup.outputs.matrix_include) }} | |
| steps: | |
| - name: Checkout workflow branch | |
| uses: actions/checkout@v5 | |
| with: | |
| token: ${{ secrets.YDBOT_TOKEN }} | |
| ref: ${{ env.WORKFLOW_CHECKOUT_REF }} | |
| fetch-depth: 0 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install ydb[yc] PyGithub pandas numpy requests | |
| - name: Setup ydb access | |
| uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials | |
| with: | |
| ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} | |
| ydb_qa_config: ${{ vars.YDB_QA_CONFIG }} | |
| - name: Prepare base mute file | |
| run: | | |
| set -euo pipefail | |
| echo "BASE_BRANCH=${{ matrix.BASE_BRANCH }}" >> "$GITHUB_ENV" | |
| echo "BUILD_TYPE=${{ matrix.BUILD_TYPE }}" >> "$GITHUB_ENV" | |
| echo "PR_BRANCH=${{ env.PR_BRANCH_PREFIX }}_${{ matrix.BASE_BRANCH }}_${{ matrix.BUILD_TYPE }}" >> "$GITHUB_ENV" | |
| git fetch origin "${{ matrix.BASE_BRANCH }}:refs/remotes/origin/${{ matrix.BASE_BRANCH }}" | |
| git show "origin/${{ matrix.BASE_BRANCH }}:${{ matrix.MUTED_YA_RELATIVE }}" > base_muted_ya.txt | |
| - name: Collect test history data | |
| run: python3 .github/scripts/analytics/flaky_tests_history.py --branch=${{ env.BASE_BRANCH }} --build_type=${{ env.BUILD_TYPE }} | |
| - name: Update muted tests in DB | |
| run: python3 .github/scripts/tests/mute/get_muted_tests.py upload_muted_tests --branch=${{ env.BASE_BRANCH }} --build_type=${{ env.BUILD_TYPE }} --muted_ya_file=base_muted_ya.txt | |
| - name: Update test monitor | |
| run: python3 .github/scripts/analytics/tests_monitor.py --branch=${{ env.BASE_BRANCH }} --build_type=${{ env.BUILD_TYPE }} | |
| - name: Sync fast-unmute grace for re-muted tests | |
| run: | | |
| .github/scripts/tests/mute/create_new_muted_ya.py sync_fast_unmute_grace \ | |
| --branch=${{ env.BASE_BRANCH }} \ | |
| --build-type=${{ env.BUILD_TYPE }} | |
| - name: Generate new muted_ya.txt | |
| run: | | |
| .github/scripts/tests/mute/create_new_muted_ya.py update_muted_ya \ | |
| --branch=${{ env.BASE_BRANCH }} \ | |
| --muted_ya_file=base_muted_ya.txt \ | |
| --build-type=${{ env.BUILD_TYPE }} | |
| - name: Check if changes exist | |
| id: changes_check | |
| run: | | |
| # Compare base_muted_ya.txt with newly generated file | |
| if git diff --no-index --quiet base_muted_ya.txt mute_update/new_muted_ya.txt; then | |
| echo "No changes detected in muted_ya.txt" | |
| echo "changes=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Changes detected in muted_ya.txt" | |
| echo "changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Setup git config | |
| if: steps.changes_check.outputs.changes == 'true' | |
| run: | | |
| git config user.name YDBot | |
| git config user.email ydbot@ydb.tech | |
| - name: Create PR branch and apply changes | |
| if: steps.changes_check.outputs.changes == 'true' | |
| run: | | |
| set -euo pipefail | |
| # Путь к mute-файлу заранее в матрице (тот же dedicated_relative, что в mute_update_workflow_matrix). | |
| OUT='${{ matrix.MUTED_YA_RELATIVE }}' | |
| git checkout -B ${{ env.PR_BRANCH }} origin/${{ env.BASE_BRANCH }} | |
| mkdir -p "$(dirname "$OUT")" | |
| cp mute_update/new_muted_ya.txt "$OUT" | |
| git add "$OUT" | |
| git commit -m "Update muted_ya for ${{ env.BUILD_TYPE }} ${{ env.BASE_BRANCH }}" | |
| echo "✓ PR branch ${{ env.PR_BRANCH }} created with changes ($OUT)" | |
| - name: Collect PR description | |
| if: steps.changes_check.outputs.changes == 'true' | |
| run: | | |
| python3 - <<'PY' | |
| from pathlib import Path | |
| from urllib.parse import quote | |
| base_branch = "${{ env.BASE_BRANCH }}" | |
| build_type = "${{ env.BUILD_TYPE }}" | |
| max_tests = 50 | |
| def read_lines(path): | |
| p = Path(path) | |
| if not p.exists() or p.stat().st_size == 0: | |
| return [] | |
| return p.read_text(encoding="utf-8").splitlines() | |
| def append_debug_section(parts, title, lines, count_override=None): | |
| if not lines: | |
| return | |
| count = count_override if count_override is not None else len(lines) | |
| parts.append(f"**{title}: {count}**\n\n") | |
| parts.append("```\n") | |
| parts.append("\n".join(lines)) | |
| parts.append("\n```\n\n") | |
| to_delete = read_lines("mute_update/to_delete.txt") | |
| to_mute = read_lines("mute_update/to_mute.txt") | |
| to_unmute = read_lines("mute_update/to_unmute.txt") | |
| to_delete_debug = read_lines("mute_update/to_delete_debug.txt") | |
| to_mute_debug = read_lines("mute_update/to_mute_debug.txt") | |
| to_unmute_debug = read_lines("mute_update/to_unmute_debug.txt") | |
| body = [f"# Muted tests update for {base_branch} (build: {build_type})\n\n"] | |
| append_debug_section( | |
| body, | |
| "Removed from mute", | |
| to_delete_debug or to_delete, | |
| count_override=len(to_delete), | |
| ) | |
| append_debug_section( | |
| body, | |
| "Muted flaky", | |
| to_mute_debug or to_mute, | |
| count_override=len(to_mute), | |
| ) | |
| if to_mute: | |
| url = ( | |
| "https://datalens.yandex.cloud/34xnbsom67hcq-ydb-autotests-test-history-link" | |
| f"?branch={quote(base_branch, safe='')}" | |
| f"&build_type={quote(build_type, safe='')}" | |
| ) | |
| for test_name in to_mute[:max_tests]: | |
| url += f"&full_name={quote(test_name.replace(' ', '/'), safe='')}" | |
| body.append(f"[View history of muted flaky tests on Dashboard]({url})\n") | |
| append_debug_section( | |
| body, | |
| "Unmuted stable", | |
| to_unmute_debug or to_unmute, | |
| count_override=len(to_unmute), | |
| ) | |
| Path("pr_body_content.txt").write_text("".join(body), encoding="utf-8") | |
| PY | |
| echo "PR_BODY_PATH=pr_body_content.txt" >> "$GITHUB_ENV" | |
| - name: Upload all generated files as artifacts | |
| if: steps.changes_check.outputs.changes == 'true' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: muted-tests-all-files-${{ env.BASE_BRANCH }}-${{ env.BUILD_TYPE }} | |
| path: | | |
| mute_update/ | |
| retention-days: 7 | |
| - name: Push PR branch | |
| if: steps.changes_check.outputs.changes == 'true' | |
| run: | | |
| # Retry logic for transient GitHub API timeouts | |
| MAX_RETRIES=5 | |
| RETRY_DELAY=10 | |
| ATTEMPT=1 | |
| while [ $ATTEMPT -le $MAX_RETRIES ]; do | |
| echo "Attempt $ATTEMPT of $MAX_RETRIES to push branch ${{ env.PR_BRANCH }}" | |
| if git push origin ${{ env.PR_BRANCH }} --force; then | |
| echo "✓ Successfully pushed branch ${{ env.PR_BRANCH }}" | |
| exit 0 | |
| else | |
| EXIT_CODE=$? | |
| echo "⚠️ Push failed with exit code $EXIT_CODE" | |
| if [ $ATTEMPT -lt $MAX_RETRIES ]; then | |
| echo "Waiting ${RETRY_DELAY} seconds before retry..." | |
| sleep $RETRY_DELAY | |
| RETRY_DELAY=$((RETRY_DELAY * 2)) # Exponential backoff | |
| else | |
| echo "❌ Failed to push after $MAX_RETRIES attempts" | |
| exit $EXIT_CODE | |
| fi | |
| fi | |
| ATTEMPT=$((ATTEMPT + 1)) | |
| done | |
| - name: Create or update PR | |
| if: steps.changes_check.outputs.changes == 'true' | |
| id: create_or_update_pr | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.YDBOT_TOKEN }} | |
| run: | | |
| # Возвращаемся на WORKFLOW_CHECKOUT_REF для доступа к скриптам | |
| git checkout "origin/${{ env.WORKFLOW_CHECKOUT_REF }}" | |
| python .github/scripts/create_or_update_pr.py create_or_update \ | |
| --base_branch="${{ env.BASE_BRANCH }}" \ | |
| --branch_for_pr="${{ env.PR_BRANCH }}" \ | |
| --title="Update muted_ya (${{ env.BUILD_TYPE }}) in ${{ env.BASE_BRANCH }}" \ | |
| --body="${{ env.PR_BODY_PATH }}" | |
| - name: Prepare PR comment | |
| if: steps.changes_check.outputs.changes == 'true' | |
| id: comment_pr | |
| run: | | |
| # Читаем содержимое PR body | |
| PR_BODY_CONTENT=$(cat pr_body_content.txt) | |
| WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| COMPLETE_BODY=$(cat << EOF | |
| Collected in workflow [#${{ github.run_number }}](${WORKFLOW_URL}) for ${{ env.BASE_BRANCH }} (build: ${{ env.BUILD_TYPE }}) | |
| ${PR_BODY_CONTENT} | |
| EOF | |
| ) | |
| echo "$COMPLETE_BODY" > pr_comment_final.txt | |
| echo "PR_COMMENT_FILE=pr_comment_final.txt" >> $GITHUB_OUTPUT | |
| - name: Add comment, reviewers and labels | |
| if: steps.changes_check.outputs.changes == 'true' | |
| uses: actions/github-script@v8 | |
| env: | |
| LABELS: ${{ env.LABELS }} | |
| REVIEWERS: ${{ env.REVIEWERS }} | |
| with: | |
| github-token: ${{ secrets.YDBOT_TOKEN }} | |
| script: | | |
| const fs = require('fs'); | |
| // Читаем комментарий из файла | |
| const commentFile = '${{ steps.comment_pr.outputs.PR_COMMENT_FILE }}'; | |
| const commentBody = fs.readFileSync(commentFile, 'utf8'); | |
| const MAX_COMMENT_LENGTH = 65000; | |
| let finalCommentBody = commentBody; | |
| if (commentBody.length > MAX_COMMENT_LENGTH) { | |
| console.log(`Comment is too large (${commentBody.length} chars), creating gist fallback`); | |
| try { | |
| const gist = await github.rest.gists.create({ | |
| public: false, | |
| description: `Muted tests update details for PR #${{ steps.create_or_update_pr.outputs.pr_number }}`, | |
| files: { | |
| [`muted-tests-update-${{ env.BASE_BRANCH }}-${{ env.BUILD_TYPE }}.md`]: { | |
| content: commentBody | |
| } | |
| } | |
| }); | |
| finalCommentBody = [ | |
| `Collected in workflow [#${{ github.run_number }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for ${{ env.BASE_BRANCH }} (build: ${{ env.BUILD_TYPE }})`, | |
| '', | |
| `The full comment body is too large for GitHub issue comments (${commentBody.length.toLocaleString()} chars).`, | |
| `Full details: ${gist.data.html_url}` | |
| ].join('\n'); | |
| } catch (error) { | |
| console.log(`Failed to create gist fallback: ${error.message}`); | |
| const suffix = '\n\n[Comment truncated due to GitHub limit]'; | |
| finalCommentBody = commentBody.slice(0, MAX_COMMENT_LENGTH - suffix.length) + suffix; | |
| } | |
| } | |
| // Добавляем комментарий | |
| await github.rest.issues.createComment({ | |
| issue_number: ${{ steps.create_or_update_pr.outputs.pr_number }}, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: finalCommentBody | |
| }); | |
| // Добавляем лейблы | |
| const labelsToAdd = process.env.LABELS.split(',').map(label => label.trim()).filter(Boolean); | |
| await github.rest.issues.addLabels({ | |
| ...context.repo, | |
| issue_number: ${{ steps.create_or_update_pr.outputs.pr_number }}, | |
| labels: labelsToAdd | |
| }); | |
| // Добавляем ревьюеров | |
| const reviewers = JSON.parse(process.env.REVIEWERS); | |
| await github.rest.pulls.requestReviewers({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: ${{ steps.create_or_update_pr.outputs.pr_number }}, | |
| team_reviewers: reviewers | |
| }); | |
| - name: Enable auto-merge (squash) | |
| if: steps.changes_check.outputs.changes == 'true' | |
| uses: actions/github-script@v8 | |
| with: | |
| github-token: ${{ secrets.YDBOT_TOKEN }} | |
| script: | | |
| const pr = ${{ steps.create_or_update_pr.outputs.pr_number }}; | |
| const { data: pullRequest } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: pr | |
| }); | |
| const mutation = ` | |
| mutation EnableAutoMerge { | |
| enablePullRequestAutoMerge(input: { | |
| pullRequestId: "${pullRequest.node_id}", | |
| mergeMethod: SQUASH | |
| }) { | |
| clientMutationId | |
| } | |
| } | |
| `; | |
| try { | |
| await github.graphql(mutation); | |
| console.log("Auto-merge with squash successfully enabled"); | |
| } catch (error) { | |
| console.error("Failed to enable auto-merge with squash:", error); | |
| } | |