Continuous Fuzzing #56
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Fuzzing | |
| on: | |
| schedule: | |
| - cron: "0 1 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| discover: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v4 | |
| - name: Build matrix | |
| id: set-matrix | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import glob | |
| import os | |
| files = sorted(os.path.basename(p) for p in glob.glob("fuzz/fuzz_*.py")) | |
| matrix = json.dumps({"fuzzer": files}) | |
| with open(os.environ["GITHUB_OUTPUT"], "a") as output: | |
| output.write(f"matrix={matrix}\n") | |
| PY | |
| fuzz: | |
| needs: discover | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "pip" | |
| cache-dependency-path: requirements-fuzz.txt | |
| - name: Install fuzzing dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-fuzz.txt | |
| - name: Run fuzzer | |
| id: run_fuzzer | |
| continue-on-error: true | |
| run: | | |
| set -o pipefail | |
| python fuzz/${{ matrix.fuzzer }} -max_total_time=18000 -timeout=30 | tee fuzz_output.txt | |
| exit_code=${PIPESTATUS[0]} | |
| echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| - name: Extract failure snippet | |
| if: ${{ steps.run_fuzzer.outputs.exit_code != '0' }} | |
| run: | | |
| python - <<'PY' | |
| import os | |
| import re | |
| from pathlib import Path | |
| fuzzer = os.environ.get("FUZZER_NAME", "unknown") | |
| command = os.environ.get("FUZZER_COMMAND", "") | |
| snippet_path = Path(os.environ.get("FUZZER_SNIPPET", "fuzz_failure_snippet.txt")) | |
| path = Path("fuzz_output.txt") | |
| if not path.exists(): | |
| snippet_path.write_text( | |
| f"Fuzzer: {fuzzer}\nCommand: {command}\n\nNo output captured.\n", | |
| encoding="utf-8", | |
| ) | |
| raise SystemExit(0) | |
| lines = path.read_text(errors="ignore").splitlines() | |
| include_idx = set() | |
| def add_range(start: int, end: int) -> None: | |
| for i in range(start, min(end, len(lines))): | |
| include_idx.add(i) | |
| exception_start = "=== Uncaught Python exception" | |
| for i, line in enumerate(lines): | |
| if line.startswith(exception_start): | |
| include_idx.add(i) | |
| j = i + 1 | |
| while j < len(lines): | |
| include_idx.add(j) | |
| if not lines[j].strip(): | |
| break | |
| if lines[j].startswith("Fuzzing for "): | |
| break | |
| if "ERROR: libFuzzer" in lines[j]: | |
| break | |
| j += 1 | |
| patterns = [ | |
| re.compile(r"^==\d+== ERROR: libFuzzer"), | |
| re.compile(r"^SUMMARY:"), | |
| re.compile(r"^Traceback"), | |
| re.compile(r"^ File "), | |
| re.compile(r"(Error|Exception|AssertionError|TimeoutError):"), | |
| re.compile(r"^artifact_prefix="), | |
| re.compile(r"^Test unit written to"), | |
| re.compile(r"^Base64:"), | |
| ] | |
| for i, line in enumerate(lines): | |
| if any(p.search(line) for p in patterns): | |
| include_idx.add(i) | |
| selected = [lines[i] for i in sorted(include_idx)] | |
| if len(selected) > 200: | |
| selected = selected[-200:] | |
| content = "\n".join(selected).strip() | |
| if not content: | |
| content = "No matching failure markers found in output." | |
| if len(content) > 4000: | |
| content = content[:4000].rstrip() + "\n... (truncated)" | |
| snippet = ( | |
| f"Fuzzer: {fuzzer}\n" | |
| f"Command: {command}\n\n" | |
| f"{content}\n" | |
| ) | |
| snippet_path.write_text(snippet, encoding="utf-8") | |
| PY | |
| env: | |
| FUZZER_NAME: ${{ matrix.fuzzer }} | |
| FUZZER_COMMAND: python fuzz/${{ matrix.fuzzer }} -max_total_time=18000 -timeout=30 | |
| FUZZER_SNIPPET: fuzz_failure_${{ matrix.fuzzer }}.txt | |
| - name: Upload failure snippet | |
| if: ${{ steps.run_fuzzer.outputs.exit_code != '0' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fuzz-failure-${{ matrix.fuzzer }} | |
| path: fuzz_failure_${{ matrix.fuzzer }}.txt | |
| - name: Fail job if fuzzer failed | |
| if: ${{ steps.run_fuzzer.outputs.exit_code != '0' }} | |
| run: exit 1 | |
| report_failure: | |
| needs: fuzz | |
| if: ${{ always() && (needs.fuzz.result == 'failure' || needs.fuzz.result == 'cancelled') }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| issues: write | |
| steps: | |
| - name: Download failure snippets | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| path: fuzz-failure | |
| pattern: fuzz-failure-* | |
| merge-multiple: true | |
| - name: Create or update failure issue | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const runId = context.runId; | |
| const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${runId}`; | |
| const title = "Fuzzing failure detected"; | |
| const fs = require("fs"); | |
| const path = require("path"); | |
| const snippetsDir = path.join(process.env.GITHUB_WORKSPACE, "fuzz-failure"); | |
| let snippets = []; | |
| if (fs.existsSync(snippetsDir)) { | |
| const entries = fs.readdirSync(snippetsDir); | |
| for (const entry of entries) { | |
| const fullPath = path.join(snippetsDir, entry); | |
| if (!fs.statSync(fullPath).isFile()) continue; | |
| const content = fs.readFileSync(fullPath, "utf8").trim(); | |
| if (!content) continue; | |
| snippets.push(`### ${entry}\n\`\`\`\n${content}\n\`\`\``); | |
| } | |
| } | |
| const snippetsSection = snippets.length | |
| ? ["", "Failure snippets:", "", snippets.join("\n\n")].join("\n") | |
| : "\nFailure snippets:\n\n(no snippets were captured)\n"; | |
| const jobs = await github.paginate( | |
| github.rest.actions.listJobsForWorkflowRun, | |
| { owner, repo, run_id: runId, per_page: 100 } | |
| ); | |
| const failedJobs = jobs | |
| .filter((job) => job.conclusion === "failure" || job.conclusion === "cancelled") | |
| .map((job) => job.name); | |
| const failedList = failedJobs.length | |
| ? failedJobs.map((name) => `- ${name}`).join("\n") | |
| : "- (unknown)"; | |
| const body = [ | |
| "Automated report: the scheduled fuzzing workflow failed.", | |
| "", | |
| `Run: ${runUrl}`, | |
| "", | |
| "Failed jobs:", | |
| failedList, | |
| snippetsSection, | |
| "", | |
| "Please review the workflow logs and reproduce locally if needed." | |
| ].join("\n"); | |
| const query = `repo:${owner}/${repo} is:issue is:open in:title "${title}"`; | |
| const search = await github.rest.search.issuesAndPullRequests({ q: query }); | |
| if (search.data.total_count > 0) { | |
| const issueNumber = search.data.items[0].number; | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: issueNumber, | |
| body, | |
| }); | |
| core.info(`Updated existing issue #${issueNumber}`); | |
| } else { | |
| const issue = await github.rest.issues.create({ | |
| owner, | |
| repo, | |
| title, | |
| body, | |
| labels: ["bug"], | |
| }); | |
| core.info(`Created issue #${issue.data.number}`); | |
| } | |