diff --git a/.github/workflows/validate-renovate-deps.yml b/.github/workflows/validate-renovate-deps.yml new file mode 100644 index 000000000000..cfa47d73e833 --- /dev/null +++ b/.github/workflows/validate-renovate-deps.yml @@ -0,0 +1,21 @@ +--- +name: "Validate Renovate coverage of Bazel native deps" +on: + pull_request: + paths: + - "deps/**" + - "renovate.json" + - "tasks/renovate.py" + - "tasks/BUILD.bazel" + - ".github/workflows/validate-renovate-deps.yml" + +permissions: {} + +jobs: + check_coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/bazel-cache + - name: Verify every http_archive/http_file in deps/ has Renovate coverage + run: bazel run //tasks:check_renovate_bazel_coverage diff --git a/deps/.renovate-untracked.json b/deps/.renovate-untracked.json new file mode 100644 index 000000000000..32e91eefc3d6 --- /dev/null +++ b/deps/.renovate-untracked.json @@ -0,0 +1,59 @@ +{ + "_comment": "Bazel http_archive and http_file deps in any deps/**/*.MODULE.bazel file that intentionally have no Renovate customManager. Every entry MUST include a non-empty rationale. The check is enforced by tasks/check_renovate_bazel_coverage.py and the validate-renovate-deps.yml workflow. As deps move from this allowlist to a customManager in renovate.json, remove the entry here in the same PR.", + "intentionally_untracked": { + "xz": "Pending tracking in Phase 2a (github-releases batch).", + "zlib": "Pending tracking in Phase 2a (github-releases batch).", + "openssl": "Pending tracking in Phase 2a (github-releases batch). Blocked by openssl drift cleanup (AIX 3.5.5, MSI 3.5).", + "libffi": "Pending tracking in Phase 2a (github-releases batch).", + "pcre2": "Pending tracking in Phase 2a (github-releases batch).", + "util-linux": "Pending tracking in Phase 2a (github-releases batch).", + "libseccomp": "Pending tracking in Phase 2a (github-releases batch).", + "patchelf": "Pending tracking in Phase 2a (github-releases batch).", + "nghttp2": "Pending tracking in Phase 2a (github-releases batch).", + "popt": "Pending tracking in Phase 2a (github-releases batch).", + "libyaml": "Pending tracking in Phase 2a (github-releases batch).", + "zstd": "Pending tracking in Phase 2a (github-releases batch).", + "lua": "Pending tracking in Phase 2a (github-releases batch).", + "xmlsec": "Pending tracking in Phase 2a (github-releases batch).", + "krb5": "Pending tracking in Phase 2b (tricky-tag batch — tag krb5-X.Y.Z-final differs from tarball).", + "rpm": "Pending tracking in Phase 2b (tricky-tag batch — strip_prefix is rpm-rpm-X.Y.Z-release).", + "systemd": "Pending tracking in Phase 2b (tricky-tag batch — single-integer versioning, loose).", + "libsepol": "Pending tracking in Phase 2b (SELinux monorepo).", + "libselinux": "Pending tracking in Phase 2b (SELinux monorepo).", + "openssl_fips": "Pending tracking in Phase 2b. Will be added with `enabled: false` — pinned to 3.0.9 for FIPS 140-2 validation, must NOT auto-update.", + "libxml2": "Pending tracking in Phase 2c (GitLab-tags batch).", + "libxslt": "Pending tracking in Phase 2c (GitLab-tags batch).", + "dbus": "Pending tracking in Phase 2c (GitLab-tags batch).", + "sqlite3": "Hard: version is split across a tuple (sqlite_ver = (\"3\", \"53\", \"00\")) and embeds a release-year directory in the URL. Needs a refactor to a single literal before a Renovate regex can target it cleanly. Phase 3.", + "bzip2": "Hard: sourceware.org tarballs; no native Renovate datasource. Needs a customDatasource. Phase 3.", + "attr": "Hard: hosted on Savannah (savannah.nongnu.org); no native Renovate datasource. Needs a customDatasource. Phase 3.", + "gpg-error": "Hard: hosted on gnupg.org FTP; no native Renovate datasource. Needs a customDatasource. Phase 3.", + "gcrypt": "Hard: hosted on gnupg.org FTP; no native Renovate datasource. Needs a customDatasource. Phase 3.", + "unixodbc": "Hard: hosted on unixodbc.org; no native Renovate datasource. Needs a customDatasource. Phase 3.", + "acl": "Pending tracking. Hosted on Savannah (download.savannah.nongnu.org); needs a customDatasource. Phase 3.", + "curl": "Pending tracking in Phase 2a (github-releases batch). Tarball is on curl.haxx.se but releases are tagged on github.com/curl/curl.", + "cpython": "Pending tracking in Phase 2a (github-releases batch). Released on python.org; corresponding GitHub tags exist at github.com/python/cpython.", + "freetds": "Pending tracking in Phase 2a (github-releases batch). Tarballs on freetds.org; releases tagged at github.com/FreeTDS/freetds.", + "libpcap": "Pending tracking in Phase 2a (github-releases batch). Tarball on tcpdump.org; releases tagged at github.com/the-tcpdump-group/libpcap.", + "nfsiostat": "Pending tracking in Phase 2c. Hosted on kernel.org mirrors (no GitHub mirror with reliable tags). Needs a customDatasource.", + "openscap": "Pending tracking in Phase 2a (github-releases batch). Released at github.com/OpenSCAP/openscap.", + "setuptools": "Pending tracking in Phase 2a (github-releases batch). Released at github.com/pypa/setuptools.", + "compile_policy_x86_64": "Pending tracking in Phase 2a. DataDog/dd-policy-engine GitHub release; will be tracked as one bundle with the arm64 and windows variants since they share VERSION.", + "compile_policy_arm64": "Pending tracking in Phase 2a. Shares VERSION with compile_policy_x86_64; tracked together.", + "compile_policy_windows_x86_64": "Pending tracking in Phase 2a. Shares VERSION with compile_policy_x86_64; tracked together.", + "gstatus_binary": "Pending tracking in Phase 2a (github-releases batch). Released at github.com/gluster/gstatus.", + "msodbcsql18_deb_amd64": "Pending tracking. Microsoft .deb package; no native datasource for packages.microsoft.com. Both arches share VERSION and will be tracked as one bundle when a customDatasource is added. Phase 3.", + "msodbcsql18_deb_arm64": "Pending tracking. Shares VERSION with msodbcsql18_deb_amd64; tracked together. Phase 3.", + "snmp_traps": "Intentionally untracked. Datadog-internal SNMP traps DB published to dd-agent-omnibus S3; no public upstream to watch. Bumped manually when the traps DB is regenerated.", + "cacert_pem": "Intentionally untracked. Bumped via a separate Datadog Synthetics-driven workflow (see deps/cacerts/cacerts.MODULE.bazel) that alerts team-agent-build on header changes; auto-PRs from Renovate would race that flow.", + "cacert_license": "Intentionally untracked. Mozilla MPL-2.0 license text accompanying cacert_pem; updated together with cacert_pem via the same out-of-band flow.", + "sqlite3_license": "Intentionally untracked here — version is pinned via the shared sqlite_ver tuple in deps/repos.MODULE.bazel and will be tracked together with sqlite3 once that refactor lands (see sqlite3 entry).", + "sqlite_win": "Intentionally untracked here. Windows-only SQLite amalgamation; pinned to the same release as sqlite3 (manually kept in sync, see TODO in deps/cpython/cpython.MODULE.bazel). Will be folded in with sqlite3 in Phase 3.", + "bzip2_win": "Pending tracking. Loop-emitted Windows variant in deps/cpython/cpython.MODULE.bazel; version pinned via the python_src_deps dict alongside the base bzip2. Will be tracked together with bzip2 (Phase 3 customDatasource).", + "mpdecimal_win": "Pending tracking. Loop-emitted Windows variant in deps/cpython/cpython.MODULE.bazel; version pinned via the python_src_deps dict. No upstream `mpdecimal` dep exists today — needs its own customManager.", + "xz_win": "Pending tracking. Loop-emitted Windows variant in deps/cpython/cpython.MODULE.bazel; version pinned via the python_src_deps dict alongside the base xz. Will be tracked together with xz (Phase 2a).", + "zlib_win": "Pending tracking. Loop-emitted Windows variant in deps/cpython/cpython.MODULE.bazel; version pinned via the python_src_deps dict alongside the base zlib. Will be tracked together with zlib (Phase 2a).", + "libffi_win": "Pending tracking. Loop-emitted Windows variant in deps/cpython/cpython.MODULE.bazel; version pinned via the python_bin_deps dict. Note: pinned to 3.4.4 (cpython-bin-deps), independent of the base libffi version — must be tracked separately.", + "tcltk_win": "Pending tracking. Loop-emitted Windows variant in deps/cpython/cpython.MODULE.bazel; version pinned via the python_bin_deps dict. No upstream `tcltk` dep — needs its own customManager (cpython-bin-deps tags)." + } +} diff --git a/tasks/BUILD.bazel b/tasks/BUILD.bazel index 402dc7ea0085..5e74afb0afff 100644 --- a/tasks/BUILD.bazel +++ b/tasks/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_library.bzl", "py_library") package(default_visibility = ["//visibility:private"]) @@ -13,3 +14,14 @@ py_library( ) exports_files(["core_checks.py"]) + +# No `data = [...]` for renovate.json / deps/ / .renovate-untracked.json on +# purpose: the script resolves them via $BUILD_WORKSPACE_DIRECTORY (set by +# `bazel run`) so it reads the live workspace, not the bazel sandbox. Adding +# them as data deps would silently shadow the real files. +py_binary( + name = "check_renovate_bazel_coverage", + srcs = ["renovate.py"], + main = "renovate.py", + deps = ["@py_dev_requirements//invoke"], +) diff --git a/tasks/__init__.py b/tasks/__init__.py index d940c8813b13..fa9109d70860 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -69,6 +69,7 @@ python_version, quality_gates, release, + renovate, rtloader, sbomgen, schema, @@ -240,6 +241,7 @@ ns.add_collection(setup) ns.add_collection(systray) ns.add_collection(release) +ns.add_collection(renovate) ns.add_collection(rtloader) ns.add_collection(system_probe) ns.add_collection(process_agent) diff --git a/tasks/renovate.py b/tasks/renovate.py new file mode 100644 index 000000000000..5b800902a5ad --- /dev/null +++ b/tasks/renovate.py @@ -0,0 +1,306 @@ +""" +Renovate maintenance tasks. + +This module exposes a check that verifies every native dep declared via +``http_archive`` or ``http_file`` in any ``*.MODULE.bazel`` file under ``deps/`` +is either tracked by a Renovate ``customManager`` in ``renovate.json`` or listed +in ``deps/.renovate-untracked.json`` with a rationale. It runs in CI via +``.github/workflows/validate-renovate-deps.yml``. +""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path + +from invoke import task +from invoke.context import Context +from invoke.exceptions import Exit + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def main(): + import sys + + # The canonical entry points are `bazel run //tasks:check_renovate_bazel_coverage` + # (used by CI) and `dda inv renovate.check-bazel-coverage`. BUILD_WORKSPACE_DIRECTORY + # is set by `bazel run`; the REPO_ROOT fallback only exists so the `dda inv` + # path (which doesn't set it) still resolves the workspace correctly. + _workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + _root = _workspace if _workspace else str(REPO_ROOT) + # Translate invoke's Exit to sys.exit so `bazel run` / direct python + # invocation produce the same stderr report + exit code as `dda inv`, + # whose runner catches Exit internally. + try: + check_bazel_coverage(Context(), _root) + except Exit as e: + if e.message: + print(e.message, file=sys.stderr) + sys.exit(e.code) + + +@task +def check_bazel_coverage(_: Context, root: str | None = None) -> None: + """ + Fail if any http_archive or http_file in deps/ lacks a Renovate customManager, + or if the allowlist contains stale entries that no longer match any dep. + + Scans every ``*.MODULE.bazel`` file under ``deps/`` for ``http_archive`` + and ``http_file`` calls. A dep is considered covered when either: + * its name appears as ``depNameTemplate`` in one of ``renovate.json``'s customManagers, or + * it is listed in ``deps/.renovate-untracked.json`` with a non-empty rationale. + + Also fails when ``deps/.renovate-untracked.json`` lists a dep that no longer + exists in ``deps/**/*.MODULE.bazel`` — without this, renamed or removed deps + leave dead entries behind and the file rots. + + Writes a markdown report to ``$GITHUB_STEP_SUMMARY`` when running in GitHub Actions. + """ + root_path = Path(root) if root is not None else Path(REPO_ROOT) + dep_names = _parse_deps_dir(root_path / "deps") + tracked_names = _parse_renovate_json(root_path / "renovate.json") + allowlist = _parse_allowlist(root_path / "deps" / ".renovate-untracked.json") + allowlist_keys = set(allowlist) + + untracked = dep_names - tracked_names - allowlist_keys + stale = allowlist_keys - dep_names + # A dep listed in BOTH renovate.json AND the allowlist means a tracking PR + # forgot to delete the allowlist entry. Left alone, the dead entry would + # silently mask a later removal of the Renovate manager — defeating the + # phase-out workflow this check enforces. + double_classified = tracked_names & allowlist_keys + + if untracked or stale or double_classified: + report = _emit_failure_report(untracked, stale, double_classified, allowlist) + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + # GITHUB_STEP_SUMMARY is a shared file for the whole step; append + # rather than overwrite to play nice with any other writes. + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write(report + "\n") + raise Exit(report, code=1) + + print( + f"OK: {len(dep_names)} native deps (http_archive + http_file), " + f"{len(dep_names) - len(allowlist)} tracked by Renovate, " + f"{len(allowlist)} intentionally untracked." + ) + + +def _parse_deps_dir(deps_dir: Path) -> set[str]: + names: set[str] = set() + for path in deps_dir.rglob("*.MODULE.bazel"): + text = path.read_text() + for call in ("http_archive", "http_file"): + names |= _extract_call_names(text, call, path) + return names + + +def _extract_call_names(text: str, call_name: str, path: Path) -> set[str]: + """Extract name = "..." from all call_name(...) blocks, regardless of arg order or comments. + + When the name is a Starlark template (e.g. `name = "{}_win".format(name)`), + resolve it against the enclosing list comprehension and the dict literal it + iterates. Fails loudly on any template we can't resolve — silent skips + would let loop-emitted deps escape the coverage check. + """ + names: set[str] = set() + # Starlark accepts both single- and double-quoted string literals; capture either. + name_re = re.compile(r"""\bname\s*=\s*(?:"([^"]+)"|'([^']+)')""") + marker = call_name + "(" + start = 0 + while True: + pos = text.find(marker, start) + if pos == -1: + break + depth = 1 + i = pos + len(marker) + while i < len(text) and depth: + c = text[i] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + elif c in ('"', "'"): + # Skip the matching string literal so parens inside it don't + # throw off the depth count. Handles both quote styles. + quote = c + i += 1 + while i < len(text) and text[i] != quote: + if text[i] == "\\": + i += 1 + i += 1 + i += 1 + block = text[pos + len(marker) : i - 1] + uncommented = "\n".join(line for line in block.splitlines() if not line.lstrip().startswith("#")) + m = name_re.search(uncommented) + if m: + literal = m.group(1) or m.group(2) + if "{" in literal: + names |= _resolve_templated_name(text, pos, literal, path) + else: + names.add(literal) + start = i + return names + + +_DICT_LITERAL_RE = re.compile( + r'^(\w+)\s*=\s*\{(.*?)\n\}', + re.DOTALL | re.MULTILINE, +) +_DICT_KEY_RE = re.compile(r"""^\s*(?:"([^"]+)"|'([^']+)')\s*:""", re.MULTILINE) +# `for , ... in .items()` — first target name, optional +# tuple-destructuring tail (which may contain parentheses for `(a, b)`), then +# the source dict. We anchor on the closing ` in ` token so the destructuring +# tail can be anything. +_COMPREHENSION_RE = re.compile( + r'\bfor\s+(\w+)[^\n]*?\s+in\s+(\w+)\.items\(\)', +) + + +def _resolve_templated_name(text: str, call_pos: int, template: str, path: Path) -> set[str]: + """Resolve a templated name like "{}_win" by finding the enclosing + comprehension and the dict it iterates. + + Supported pattern (the only one in-tree today): + + SOME_DICT = {"keyA": (...), "keyB": (...)} + [ + http_archive( + name = "{}_suffix".format(VAR), + ... + ) + for VAR, (...) in SOME_DICT.items() + ] + + Raises Exit if the template can't be resolved — a silent skip would defeat + the coverage guarantee. + """ + after = text[call_pos:] + comp = _COMPREHENSION_RE.search(after) + if not comp: + raise Exit( + f"{path}: templated name {template!r} has no enclosing " + "`for VAR, ... in DICT.items()` comprehension. Refactor to literal " + "names, or extend tasks/renovate.py to handle this pattern." + ) + iter_var, dict_var = comp.group(1), comp.group(2) + keys = _lookup_dict_keys(text, dict_var) + if keys is None: + raise Exit( + f"{path}: templated name {template!r} iterates {dict_var}.items() " + f"but no top-level `{dict_var} = {{...}}` literal was found. " + "Refactor to literal names, or extend tasks/renovate.py." + ) + # Substitute each key for the loop variable. Both `"{}".format(VAR)` and + # `"{name}".format(name = VAR)` reduce to a single `{...}` placeholder + # that we replace with the literal key. + if "{}" in template: + return {template.replace("{}", k) for k in keys} + placeholder = "{" + iter_var + "}" + if placeholder in template: + return {template.replace(placeholder, k) for k in keys} + raise Exit( + f"{path}: templated name {template!r} doesn't reference loop var " + f"{iter_var!r} via `{{}}` or `{{{iter_var}}}`. Refactor or extend the parser." + ) + + +def _lookup_dict_keys(text: str, dict_var: str) -> list[str] | None: + for m in _DICT_LITERAL_RE.finditer(text): + if m.group(1) == dict_var: + # _DICT_KEY_RE has alternation (double- vs single-quoted); findall + # returns tuples — flatten to whichever group matched. + return [dq or sq for dq, sq in _DICT_KEY_RE.findall(m.group(2))] + return None + + +def _parse_renovate_json(path: Path) -> set[str]: + # Assumption: renovate.json is plain JSON plus trailing commas only — no + # // or /* */ comments, no single-quoted strings. Renovate accepts the full + # JSON5 grammar but ours stays in this subset; if that changes, swap in a + # real JSON5 parser (e.g. the `json5` package) instead of extending this regex. + raw = path.read_text() + stripped = re.sub(r",(\s*[}\]])", r"\1", raw) + data = json.loads(stripped) + return {cm["depNameTemplate"] for cm in data.get("customManagers", []) if "depNameTemplate" in cm} + + +def _parse_allowlist(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + data = json.loads(path.read_text()) + entries = data.get("intentionally_untracked", {}) + bad = [k for k, v in entries.items() if not (isinstance(v, str) and v.strip())] + if bad: + raise Exit( + f"Allowlist {path} has empty rationale for: {', '.join(bad)}. " + "Every entry must include a non-empty justification string." + ) + return entries + + +def _emit_failure_report( + untracked: set[str], + stale: set[str], + double_classified: set[str], + allowlist: dict[str, str], +) -> str: + lines = ["## ❌ Renovate coverage check failed", ""] + if untracked: + lines += [ + "The following native deps (http_archive / http_file) in `deps/` have no " + "matching `customManager` in `renovate.json`:", + "", + "| dep | suggested fix |", + "|---|---|", + ] + for dep in sorted(untracked): + lines.append( + f"| `{dep}` | Add a `customManagers` entry with " + f'`depNameTemplate: "{dep}"`, or add to ' + "`deps/.renovate-untracked.json` with a rationale. |" + ) + lines += [ + "", + "See `renovate.json` for existing patterns (linux-images, windows-images, ...).", + "", + ] + if stale: + lines += [ + "The following entries in `deps/.renovate-untracked.json` no longer match " + "any `http_archive` / `http_file` name in `deps/**/*.MODULE.bazel` and " + "must be removed:", + "", + ] + for dep in sorted(stale): + lines.append(f"- `{dep}`") + lines += [ + "", + "If the dep was renamed, replace the old entry with the new name. If it " + "was removed, delete the entry. Stale allowlist entries silently weaken the check.", + "", + ] + if double_classified: + lines += [ + "The following deps are listed in BOTH `renovate.json` (as a " + "`customManagers` entry) AND `deps/.renovate-untracked.json`. Once a dep is " + "tracked by Renovate, its allowlist entry must be deleted in the same PR — " + "leaving it behind silently masks future regressions of the Renovate manager:", + "", + ] + for dep in sorted(double_classified): + lines.append(f"- `{dep}` — remove the entry from `deps/.renovate-untracked.json`") + lines.append("") + lines.append( + f"Currently allowlisted ({len(allowlist)}): " + + (", ".join(f"`{k}`" for k in sorted(allowlist)) if allowlist else "_none_") + ) + return "\n".join(lines) + + +if __name__ == "__main__": + main()