Continuous Fuzzing #132
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Fuzzing | |
| on: | |
| schedule: | |
| - cron: "0 1 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| discover: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v5 | |
| - name: Build matrix | |
| id: set-matrix | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import glob | |
| import os | |
| files = sorted(os.path.basename(p) for p in glob.glob("fuzz/fuzz_*.py")) | |
| matrix = json.dumps({"fuzzer": files}) | |
| with open(os.environ["GITHUB_OUTPUT"], "a") as output: | |
| output.write(f"matrix={matrix}\n") | |
| PY | |
| fuzz: | |
| needs: discover | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| cache: "pip" | |
| cache-dependency-path: requirements-fuzz.txt | |
| - name: Install fuzzing dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-fuzz.txt | |
| - name: Run fuzzer | |
| id: run_fuzzer | |
| continue-on-error: true | |
| run: | | |
| set -o pipefail | |
| # Stream stdout to the action log (visible) and keep only the last | |
| # ~5000 lines on disk (capturing libFuzzer's crash dump tail without | |
| # letting the file grow unbounded over a 5h run). | |
| python fuzz/${{ matrix.fuzzer }} -max_total_time=18000 -timeout=30 2>&1 \ | |
| | python3 -u -c " | |
| import sys | |
| from collections import deque | |
| buf = deque(maxlen=5000) | |
| for line in sys.stdin: | |
| sys.stdout.write(line) | |
| sys.stdout.flush() | |
| buf.append(line) | |
| with open('fuzz_output.txt', 'w') as f: | |
| f.writelines(buf) | |
| " | |
| exit_code=${PIPESTATUS[0]} | |
| echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| - name: Extract failure snippet | |
| if: ${{ steps.run_fuzzer.outputs.exit_code != '0' }} | |
| run: | | |
| python - <<'PY' | |
| import os | |
| import re | |
| from pathlib import Path | |
| fuzzer = os.environ.get("FUZZER_NAME", "unknown") | |
| command = os.environ.get("FUZZER_COMMAND", "") | |
| snippet_path = Path(os.environ.get("FUZZER_SNIPPET", "fuzz_failure_snippet.txt")) | |
| path = Path("fuzz_output.txt") | |
| if not path.exists(): | |
| snippet_path.write_text( | |
| f"Fuzzer: {fuzzer}\nCommand: {command}\n\nNo output captured.\n", | |
| encoding="utf-8", | |
| ) | |
| raise SystemExit(0) | |
| lines = path.read_text(errors="ignore").splitlines() | |
| include_idx = set() | |
| def add_range(start: int, end: int) -> None: | |
| for i in range(start, min(end, len(lines))): | |
| include_idx.add(i) | |
| exception_start = "=== Uncaught Python exception" | |
| for i, line in enumerate(lines): | |
| if line.startswith(exception_start): | |
| include_idx.add(i) | |
| j = i + 1 | |
| while j < len(lines): | |
| include_idx.add(j) | |
| if not lines[j].strip(): | |
| break | |
| if lines[j].startswith("Fuzzing for "): | |
| break | |
| if "ERROR: libFuzzer" in lines[j]: | |
| break | |
| j += 1 | |
| patterns = [ | |
| re.compile(r"^==\d+== ERROR: libFuzzer"), | |
| re.compile(r"^SUMMARY:"), | |
| re.compile(r"^Traceback"), | |
| re.compile(r"^ File "), | |
| re.compile(r"(Error|Exception|AssertionError|TimeoutError):"), | |
| re.compile(r"^artifact_prefix="), | |
| re.compile(r"^Test unit written to"), | |
| re.compile(r"^Base64:"), | |
| ] | |
| for i, line in enumerate(lines): | |
| if any(p.search(line) for p in patterns): | |
| include_idx.add(i) | |
| selected = [lines[i] for i in sorted(include_idx)] | |
| if len(selected) > 200: | |
| selected = selected[-200:] | |
| content = "\n".join(selected).strip() | |
| if not content: | |
| content = "No matching failure markers found in output." | |
| if len(content) > 4000: | |
| content = content[:4000].rstrip() + "\n... (truncated)" | |
| snippet = ( | |
| f"Fuzzer: {fuzzer}\n" | |
| f"Command: {command}\n\n" | |
| f"{content}\n" | |
| ) | |
| snippet_path.write_text(snippet, encoding="utf-8") | |
| PY | |
| env: | |
| FUZZER_NAME: ${{ matrix.fuzzer }} | |
| FUZZER_COMMAND: python fuzz/${{ matrix.fuzzer }} -max_total_time=18000 -timeout=30 | |
| FUZZER_SNIPPET: fuzz_failure_${{ matrix.fuzzer }}.txt | |
| - name: Upload failure snippet | |
| if: ${{ steps.run_fuzzer.outputs.exit_code != '0' }} | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: fuzz-failure-${{ matrix.fuzzer }} | |
| path: fuzz_failure_${{ matrix.fuzzer }}.txt | |
| - name: Fail job if fuzzer failed | |
| if: ${{ steps.run_fuzzer.outputs.exit_code != '0' }} | |
| run: exit 1 | |
| report_failure: | |
| needs: fuzz | |
| if: ${{ always() && (needs.fuzz.result == 'failure' || needs.fuzz.result == 'cancelled') }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| issues: write | |
| steps: | |
| - name: Download failure snippets | |
| uses: actions/download-artifact@v5 | |
| continue-on-error: true | |
| with: | |
| path: fuzz-failure | |
| pattern: fuzz-failure-* | |
| merge-multiple: true | |
| - name: Create or update failure issue | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const runId = context.runId; | |
| const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${runId}`; | |
| const title = "Fuzzing failure detected"; | |
| const fs = require("fs"); | |
| const path = require("path"); | |
| const snippetsDir = path.join(process.env.GITHUB_WORKSPACE, "fuzz-failure"); | |
| let snippets = []; | |
| if (fs.existsSync(snippetsDir)) { | |
| const entries = fs.readdirSync(snippetsDir); | |
| for (const entry of entries) { | |
| const fullPath = path.join(snippetsDir, entry); | |
| if (!fs.statSync(fullPath).isFile()) continue; | |
| const content = fs.readFileSync(fullPath, "utf8").trim(); | |
| if (!content) continue; | |
| snippets.push(`### ${entry}\n\`\`\`\n${content}\n\`\`\``); | |
| } | |
| } | |
| const snippetsSection = snippets.length | |
| ? ["", "Failure snippets:", "", snippets.join("\n\n")].join("\n") | |
| : "\nFailure snippets:\n\n(no snippets were captured)\n"; | |
| const jobs = await github.paginate( | |
| github.rest.actions.listJobsForWorkflowRun, | |
| { owner, repo, run_id: runId, per_page: 100 } | |
| ); | |
| // Distinguish a fuzzer-found crash (Run fuzzer step completed with | |
| // a non-zero exit code) from a runner kill (the step never | |
| // completed because the runner was terminated externally — OOM, | |
| // infrastructure issue, etc.). The latter is infrastructure noise, | |
| // not a code bug, so it should not file an issue. | |
| const crashedJobs = []; | |
| const killedJobs = []; | |
| for (const job of jobs) { | |
| if (job.conclusion !== "failure" && job.conclusion !== "cancelled") continue; | |
| const runFuzzer = (job.steps || []).find((s) => s.name === "Run fuzzer"); | |
| if (runFuzzer && runFuzzer.status !== "completed") { | |
| killedJobs.push(job.name); | |
| } else { | |
| crashedJobs.push(job.name); | |
| } | |
| } | |
| if (crashedJobs.length === 0 && killedJobs.length > 0) { | |
| core.notice( | |
| `Skipping issue creation: ${killedJobs.length} job(s) were terminated externally (likely runner kill), no fuzzer crashes detected. Killed: ${killedJobs.join(", ")}` | |
| ); | |
| return; | |
| } | |
| const sections = [ | |
| "Automated report: the scheduled fuzzing workflow failed.", | |
| "", | |
| `Run: ${runUrl}`, | |
| "", | |
| "Crashed jobs (fuzzer found a real failure):", | |
| crashedJobs.length | |
| ? crashedJobs.map((n) => `- ${n}`).join("\n") | |
| : "- (none)", | |
| ]; | |
| if (killedJobs.length) { | |
| sections.push( | |
| "", | |
| "Killed jobs (runner terminated externally, not a fuzzer crash):", | |
| killedJobs.map((n) => `- ${n}`).join("\n") | |
| ); | |
| } | |
| sections.push( | |
| snippetsSection, | |
| "", | |
| "Please review the workflow logs and reproduce locally if needed." | |
| ); | |
| const body = sections.join("\n"); | |
| const query = `repo:${owner}/${repo} is:issue is:open in:title "${title}"`; | |
| const search = await github.rest.search.issuesAndPullRequests({ q: query }); | |
| if (search.data.total_count > 0) { | |
| const issueNumber = search.data.items[0].number; | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: issueNumber, | |
| body, | |
| }); | |
| core.info(`Updated existing issue #${issueNumber}`); | |
| } else { | |
| const issue = await github.rest.issues.create({ | |
| owner, | |
| repo, | |
| title, | |
| body, | |
| labels: ["bug"], | |
| }); | |
| core.info(`Created issue #${issue.data.number}`); | |
| } | |