Bot - CI Failure Notifier #1936
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bot - CI Failure Notifier | |
| on: | |
| workflow_run: | |
| workflows: | |
| - "CI" | |
| - "Security" | |
| - "Benchmarks" | |
| - "Integration" | |
| types: [completed] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| actions: read | |
| jobs: | |
| notify: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install log-parser dep | |
| run: npm install --no-save --prefix /tmp/botci adm-zip | |
| - name: Process CI Result | |
| uses: actions/github-script@v9 | |
| env: | |
| NODE_PATH: /tmp/botci/node_modules | |
| with: | |
| github-token: ${{ secrets.HOMEBREW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} | |
| script: | | |
| const run = context.payload.workflow_run; | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const workflowName = run.name || "workflow"; | |
| const isMasterPush = run.event === 'push' && run.head_branch === 'master'; | |
| // Resolve PR | |
| let pr_number = null; | |
| if (run.pull_requests && run.pull_requests.length > 0) { | |
| pr_number = run.pull_requests[0].number; | |
| } else { | |
| try { | |
| const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| owner, repo, commit_sha: run.head_sha | |
| }); | |
| const open = prs.find(p => p.state === 'open'); | |
| if (open) pr_number = open.number; | |
| else if (prs.length > 0) pr_number = prs[0].number; | |
| } catch (e) { | |
| console.log(`PR lookup failed: ${e.message}`); | |
| } | |
| } | |
| if (!pr_number && !isMasterPush) { | |
| console.log(`No PR and not master push (event=${run.event} branch=${run.head_branch}). Exiting.`); | |
| return; | |
| } | |
| const fixHints = { | |
| lint: "Run `make lint` locally.", | |
| 'mod-tidy': "Run `go mod tidy` and commit `go.mod`/`go.sum`.", | |
| nix: "Run `nix flake check --no-build` locally.", | |
| snap: "Validate `snapcraft.yaml` with `snapcraft list-plugins`.", | |
| flatpak: "Validate the Flatpak manifest YAML structure.", | |
| website: "Run `cd docs && npm ci && npx docusaurus build`.", | |
| 'lua-plugins': "Run `luac -p plugins/*.lua` to check Lua syntax.", | |
| goreleaser: "Run `goreleaser check -f .goreleaser.yml`.", | |
| build: "Run `go build ./...` and `go test ./...` locally.", | |
| govulncheck: "Run `govulncheck ./...` and update vulnerable deps.", | |
| gosec: "Inspect gosec SARIF in the Security tab; fix or annotate with `//nolint:gosec` (justified).", | |
| trivy: "Run `trivy fs .` and address CRITICAL/HIGH findings.", | |
| codeql: "Inspect CodeQL findings in the Security tab.", | |
| benchmark: "Inspect benchstat report on the PR; look for >3% regressions.", | |
| 'imap + smtp e2e': "Run `docker compose -f tests/integration/docker-compose.yml up -d` then `go test -tags=integration ./tests/integration/...`.", | |
| 'cross-platform': "Reproduce with `go test -race ./...` on the failing OS.", | |
| fuzz: "Reproduce with `go test -fuzz=<Name> -fuzztime=30s ./<pkg>`.", | |
| }; | |
| function hintFor(jobName) { | |
| const lower = jobName.toLowerCase(); | |
| for (const key of Object.keys(fixHints)) { | |
| if (lower.includes(key)) return fixHints[key]; | |
| } | |
| return null; | |
| } | |
| async function downloadLogs() { | |
| try { | |
| const res = await github.rest.actions.downloadWorkflowRunLogs({ | |
| owner, repo, run_id: run.id | |
| }); | |
| return Buffer.from(res.data); | |
| } catch (e) { | |
| console.log(`could not download logs: ${e.message}`); | |
| return null; | |
| } | |
| } | |
| async function extractFailures(zipBuf) { | |
| if (!zipBuf) return []; | |
| let AdmZip; | |
| try { | |
| AdmZip = require('adm-zip'); | |
| } catch (_) { | |
| console.log("adm-zip not available; skipping log parse"); | |
| return []; | |
| } | |
| const zip = new AdmZip(zipBuf); | |
| const failures = []; | |
| const goFailRe = /^FAIL\s+(\S+)/; | |
| const testFailRe = /^\s*---\s+FAIL:\s+(\S+)/; | |
| const buildFailRe = /^(.+\.go):(\d+):(\d+):\s+(.+)$/; | |
| for (const entry of zip.getEntries()) { | |
| if (entry.isDirectory) continue; | |
| if (!entry.entryName.endsWith('.txt')) continue; | |
| const text = entry.getData().toString('utf8'); | |
| const pkgs = new Set(); | |
| const tests = new Set(); | |
| const buildErrs = []; | |
| for (const line of text.split('\n')) { | |
| let m; | |
| if ((m = line.match(testFailRe))) tests.add(m[1]); | |
| else if ((m = line.match(goFailRe))) { | |
| if (m[1] !== '[build') pkgs.add(m[1]); | |
| } else if ((m = line.match(buildFailRe))) { | |
| if (buildErrs.length < 5) buildErrs.push(`${m[1]}:${m[2]}: ${m[4]}`); | |
| } | |
| } | |
| if (pkgs.size || tests.size || buildErrs.length) { | |
| failures.push({ | |
| file: entry.entryName, | |
| packages: [...pkgs], | |
| tests: [...tests], | |
| buildErrors: buildErrs, | |
| }); | |
| } | |
| } | |
| return failures; | |
| } | |
| function renderFailuresSection(failures) { | |
| if (!failures.length) return ""; | |
| const seenPkgs = new Set(); | |
| const seenTests = new Set(); | |
| const seenBuild = new Set(); | |
| for (const f of failures) { | |
| f.packages.forEach(p => seenPkgs.add(p)); | |
| f.tests.forEach(t => seenTests.add(t)); | |
| f.buildErrors.forEach(b => seenBuild.add(b)); | |
| } | |
| const lines = ["\n#### Failure details\n"]; | |
| if (seenBuild.size) { | |
| lines.push("**Build errors:**"); | |
| for (const b of [...seenBuild].slice(0, 8)) lines.push(`- \`${b}\``); | |
| lines.push(""); | |
| } | |
| if (seenPkgs.size) { | |
| lines.push("**Failing packages:**"); | |
| for (const p of [...seenPkgs].slice(0, 20)) lines.push(`- \`${p}\``); | |
| lines.push(""); | |
| } | |
| if (seenTests.size) { | |
| lines.push("**Failing tests:**"); | |
| for (const t of [...seenTests].slice(0, 30)) lines.push(`- \`${t}\``); | |
| lines.push(""); | |
| } | |
| return lines.join("\n"); | |
| } | |
| const marker = ``; | |
| async function upsertPRComment(body) { | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner, repo, issue_number: pr_number, per_page: 100 | |
| }); | |
| const existing = comments.find(c => c.body && c.body.includes(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, repo, comment_id: existing.id, body | |
| }); | |
| console.log(`updated comment ${existing.id} on PR #${pr_number}`); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner, repo, issue_number: pr_number, body | |
| }); | |
| console.log(`created comment on PR #${pr_number}`); | |
| } | |
| } | |
| async function deletePRCommentIfExists() { | |
| if (!pr_number) return; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner, repo, issue_number: pr_number, per_page: 100 | |
| }); | |
| const existing = comments.find(c => c.body && c.body.includes(marker)); | |
| if (existing) { | |
| try { | |
| await github.rest.issues.deleteComment({ | |
| owner, repo, comment_id: existing.id | |
| }); | |
| console.log(`deleted stale comment ${existing.id}`); | |
| } catch (e) { | |
| // Fallback: edit instead of deleting if delete denied. | |
| await github.rest.issues.updateComment({ | |
| owner, repo, comment_id: existing.id, | |
| body: `${marker}\n${workflowName} now passing on \`${run.head_sha.substring(0, 7)}\` ([run](${run.html_url})).` | |
| }); | |
| } | |
| } | |
| } | |
| // ----- failure ----- | |
| if (run.conclusion === 'failure' || run.conclusion === 'timed_out' || run.conclusion === 'startup_failure') { | |
| const { data: jobsData } = await github.rest.actions.listJobsForWorkflowRun({ | |
| owner, repo, run_id: run.id | |
| }); | |
| const failedJobs = jobsData.jobs | |
| .filter(j => j.conclusion === 'failure' || j.conclusion === 'timed_out') | |
| .map(j => ({ name: j.name, url: j.html_url })); | |
| const zipBuf = await downloadLogs(); | |
| const failures = await extractFailures(zipBuf); | |
| let lines = []; | |
| lines.push(`### ❌ ${workflowName} failed`); | |
| lines.push(""); | |
| lines.push(`Commit: \`${run.head_sha.substring(0, 7)}\` · [run logs](${run.html_url})`); | |
| lines.push(""); | |
| if (failedJobs.length) { | |
| lines.push("**Failed jobs:**"); | |
| for (const j of failedJobs) { | |
| const hint = hintFor(j.name); | |
| lines.push(`- [${j.name}](${j.url})${hint ? ` — ${hint}` : ""}`); | |
| } | |
| } | |
| lines.push(renderFailuresSection(failures)); | |
| const body = `${marker}\n` + lines.join("\n"); | |
| if (pr_number) { | |
| await upsertPRComment(body); | |
| } else if (isMasterPush) { | |
| const title = `${workflowName} failed on master @ ${run.head_sha.substring(0, 7)}`; | |
| const { data: existing } = await github.rest.issues.listForRepo({ | |
| owner, repo, state: 'open', labels: 'ci-failure', per_page: 50 | |
| }); | |
| const dup = existing.find(i => i.title === title); | |
| if (dup) { | |
| await github.rest.issues.createComment({ | |
| owner, repo, issue_number: dup.number, | |
| body: `Re-run also failed: ${run.html_url}\n\n${lines.join("\n")}` | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| owner, repo, title, body, labels: ['ci-failure'] | |
| }); | |
| } | |
| try { | |
| await github.rest.repos.createCommitComment({ | |
| owner, repo, commit_sha: run.head_sha, body | |
| }); | |
| } catch (e) { | |
| console.log(`commit comment failed: ${e.message}`); | |
| } | |
| } | |
| return; | |
| } | |
| // ----- success ----- | |
| if (run.conclusion === 'success') { | |
| if (pr_number) { | |
| await deletePRCommentIfExists(); | |
| } | |
| if (isMasterPush) { | |
| let botLogin = null; | |
| try { | |
| const { data: botUser } = await github.rest.users.getAuthenticated(); | |
| botLogin = botUser.login; | |
| } catch (_) {} | |
| const { data: openIssues } = await github.rest.issues.listForRepo({ | |
| owner, repo, state: 'open', labels: 'ci-failure', per_page: 50 | |
| }); | |
| for (const issue of openIssues) { | |
| if (botLogin && issue.user && issue.user.login !== botLogin) continue; | |
| if (!issue.title.includes(workflowName)) continue; | |
| await github.rest.issues.createComment({ | |
| owner, repo, issue_number: issue.number, | |
| body: `Subsequent master ${workflowName} run passed: ${run.html_url}. Closing.` | |
| }); | |
| await github.rest.issues.update({ | |
| owner, repo, issue_number: issue.number, state: 'closed' | |
| }); | |
| } | |
| } | |
| } |