Fix Snyk SARIF uploads so alerts track per-image categories (#775)

Merge Snyk's per-target SARIF runs into a single run and strip
automationDetails so GitHub uses our stable per-image categories again,
mark the kitchen-sink scan continue-on-error (Snyk exits 1 on findings,
so it failed nightly and never uploaded), and add a cleanup job that
dismisses alerts in categories no longer produced by the scan matrix.

Author: julienp

.github/scripts/dismiss-stale-alerts.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python
+# Dismiss open code scanning alerts whose analysis category is not produced by
+# the current scan matrix.
+#
+# GitHub only auto-closes an alert when a newer analysis is uploaded to the
+# *same* category. When an image is retired from the scan matrix, its
+# categories stop receiving uploads and their alerts stay open forever. This
+# script dismisses those orphaned alerts.
+#
+# The set of expected categories is derived from the same versions.py that
+# generates the scan matrix, so retiring an image automatically retires its
+# alerts on the next scheduled run.
+#
+# Pass --dry-run to only print what would be dismissed.
+
+import json
+import os
+import sys
+import urllib.request
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "matrix"))
+import versions
+
+DRY_RUN = "--dry-run" in sys.argv
+REPO = os.environ.get("GITHUB_REPOSITORY", "pulumi/pulumi-docker-containers")
+TOKEN = os.environ["GH_TOKEN"]
+ARCHS = ["amd64", "arm64"]
+
+DISMISS_COMMENT = (
+    "This alert belongs to an analysis category that is not produced by the "
+    "Snyk scan workflow (retired image or renamed category), so it can never "
+    "be closed automatically. Current results are tracked under the per-image "
+    "categories."
+)
+
+
+def expected_categories():
+    expected = set()
+    for suffix in ["", "-nonroot"]:
+        for arch in ARCHS:
+            expected.add(f"pulumi{suffix}-{arch}")
+    for arch in ARCHS:
+        expected.add(f"pulumi-provider-build-environment-{arch}")
+    for base_os in ["debian", "ubi"]:
+        for arch in ARCHS:
+            expected.add(f"pulumi-base-{base_os}-{arch}")
+    for sdk in versions.unversioned:
+        for arch in ARCHS:
+            expected.add(f"pulumi-{sdk}-debian-{arch}")
+    for sdk, info in versions.versioned.items():
+        for version in [info["default"]] + info["additional"]:
+            for arch in ARCHS:
+                expected.add(f"pulumi-{sdk}-{version}-debian-{arch}")
+    for sdk in ["nodejs", "python", "dotnet", "go"]:
+        expected.add(f"pulumi-{sdk}-ubi")
+    return expected
+
+
+def api(path, method="GET", body=None):
+    request = urllib.request.Request(
+        f"https://api.github.com{path}",
+        method=method,
+        data=json.dumps(body).encode() if body is not None else None,
+        headers={
+            "Authorization": f"Bearer {TOKEN}",
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    with urllib.request.urlopen(request) as response:
+        return json.load(response)
+
+
+def open_alerts():
+    alerts = []
+    page = 1
+    while True:
+        batch = api(f"/repos/{REPO}/code-scanning/alerts?state=open&per_page=100&page={page}")
+        alerts.extend(batch)
+        if len(batch) < 100:
+            return alerts
+        page += 1
+
+
+expected = expected_categories()
+stale = [
+    alert
+    for alert in open_alerts()
+    if alert.get("most_recent_instance", {}).get("category") not in expected
+]
+print(f"Found {len(stale)} open alerts in stale categories.")
+
+for alert in stale:
+    category = alert["most_recent_instance"]["category"]
+    label = f"alert #{alert['number']} ({alert['rule']['id']}) in category '{category}'"
+    if DRY_RUN:
+        print(f"Would dismiss {label}")
+        continue
+    api(
+        f"/repos/{REPO}/code-scanning/alerts/{alert['number']}",
+        method="PATCH",
+        body={
+            "state": "dismissed",
+            "dismissed_reason": "won't fix",
+            "dismissed_comment": DISMISS_COMMENT,
+        },
+    )
+    print(f"Dismissed {label}")

.github/scripts/filter-sarif.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python
+# Merge all runs in Snyk's snyk.sarif into a single run and write it to
+# out.sarif.
+#
+# Snyk emits one SARIF run per target it detects in the image (OS packages,
+# npm projects, go binaries, ...). GitHub code scanning treats every run as a
+# separate analysis whose category is derived from the run's
+# `automationDetails.id`, ignoring the `category` input of the upload-sarif
+# action:
+# https://github.blog/changelog/2025-07-21-code-scanning-will-stop-combining-multiple-sarif-runs-uploaded-in-the-same-sarif-file/
+#
+# Uploading a single run without `automationDetails` makes GitHub use the
+# per-image category the workflow passes to upload-sarif. A stable category
+# per image lets GitHub track alerts across scans and automatically close
+# alerts that no longer appear in the image's latest scan.
+#
+# snyk.sarif is deleted after a successful merge so that sequential scans
+# within the same job can never accidentally re-upload a previous scan's
+# results. If snyk.sarif is missing the script fails, failing the job without
+# uploading anything, so a broken scan leaves existing alerts untouched.
+
+import json
+import os
+import sys
+
+if not os.path.exists("snyk.sarif"):
+    print(
+        "error: snyk.sarif not found — the Snyk scan failed to produce output.",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+with open("snyk.sarif") as f:
+    sarif = json.load(f)
+
+runs = sarif.get("runs", [])
+if len(runs) == 0:
+    print("error: snyk.sarif contains no runs", file=sys.stderr)
+    sys.exit(1)
+
+# Merge the rules of all runs, deduplicating by rule id, and remember each
+# rule's index in the merged rules array so results can be re-pointed at it.
+merged_rules = []
+rule_index_by_id = {}
+for run in runs:
+    for rule in run["tool"]["driver"].get("rules", []):
+        if rule["id"] not in rule_index_by_id:
+            rule_index_by_id[rule["id"]] = len(merged_rules)
+            merged_rules.append(rule)
+
+# Merge the results of all runs, dropping exact duplicates (the same vuln
+# reported at the same location for multiple targets collapses into a single
+# alert in GitHub anyway).
+merged_results = []
+seen_results = set()
+for run in runs:
+    for result in run.get("results", []):
+        if "ruleId" in result:
+            result["ruleIndex"] = rule_index_by_id[result["ruleId"]]
+        key = json.dumps(result, sort_keys=True)
+        if key not in seen_results:
+            seen_results.add(key)
+            merged_results.append(result)
+
+merged_driver = {**runs[0]["tool"]["driver"], "name": "Snyk Container", "rules": merged_rules}
+merged_run = {**runs[0], "tool": {"driver": merged_driver}, "results": merged_results}
+merged_run.pop("automationDetails", None)
+
+with open("out.sarif", "w") as out:
+    json.dump({**sarif, "runs": [merged_run]}, out, indent=2)
+
+os.remove("snyk.sarif")
+print(f"Merged {len(runs)} runs into one: {len(merged_results)} results, {len(merged_rules)} rules.")