From 37bc24825e21346948cae98e0127b22a2f2fec7b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 15 May 2026 21:42:47 -0400 Subject: [PATCH] feat(changelogs): Use SBOMs for standardized package data extraction Refactors the changelog generation script to fetch package information from image Software Bill of Materials (SBOMs) instead of custom labels. This is split out of #4564 to derisk the change --- .github/changelogs.py | 127 +++++++++++++++++++++---- .github/renovate.json5 | 2 +- .github/workflows/generate-release.yml | 8 +- .gitignore | 1 + Justfile | 28 ++++++ 5 files changed, 143 insertions(+), 23 deletions(-) diff --git a/.github/changelogs.py b/.github/changelogs.py index 5ad48dc3938..63d702c4fa0 100644 --- a/.github/changelogs.py +++ b/.github/changelogs.py @@ -1,12 +1,14 @@ from itertools import product import subprocess import json +import os +import tempfile import time from typing import Any import re from collections import defaultdict -REGISTRY = "docker://ghcr.io/ublue-os/" +REGISTRY = "ghcr.io/ublue-os/" IMAGE_MATRIX_LATEST = { "experience": ["base", "dx"], @@ -22,6 +24,7 @@ RETRIES = 3 RETRY_WAIT = 5 FEDORA_PATTERN = re.compile(r"\.fc\d\d") +EPOCH_PATTERN = re.compile(r"^\d+:") START_PATTERN = lambda target: re.compile(rf"{target}-\d\d\d+") PATTERN_ADD = "\n| ✨ | {name} | | {version} |" @@ -125,7 +128,7 @@ def get_manifests(target: str): for i in range(RETRIES): try: output = subprocess.run( - ["skopeo", "inspect", REGISTRY + img + ":" + target], + ["skopeo", "inspect", f"docker://{REGISTRY}{img}:{target}"], check=True, stdout=subprocess.PIPE, ).stdout @@ -165,24 +168,104 @@ def get_tags(target: str, manifests: dict[str, Any]): return tags[-2], tags[-1] -def get_packages(manifests: dict[str, Any]): +def get_image_digest(image: str, tag: str) -> str: + """Get image digest using skopeo.""" + result = subprocess.run( + ["skopeo", "inspect", f"docker://{image}:{tag}"], + capture_output=True, + text=True, + check=True + ) + return json.loads(result.stdout)["Digest"] + + +def get_sbom(image: str, digest: str) -> dict: + """Fetch SBOM using ORAS.""" + full_ref = f"{image}@{digest}" + + # Find the SBOM referrer attached to this image + result = subprocess.run( + ["oras", "discover", "--format", "json", full_ref], + capture_output=True, + text=True, + check=True, + ) + discovered = json.loads(result.stdout) + + sbom_digest = None + for referrer in discovered.get("referrers", []): + if "spdx+json" in referrer.get("artifactType", ""): + sbom_digest = referrer["digest"] + break + + if sbom_digest is None: + raise RuntimeError(f"No SBOM referrer found for {full_ref}") + + sbom_ref = f"{image}@{sbom_digest}" + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run( + ["oras", "pull", sbom_ref], + capture_output=True, + check=True, + cwd=tmpdir, + ) + for fname in os.listdir(tmpdir): + fpath = os.path.join(tmpdir, fname) + if fname.endswith(".zst"): + result = subprocess.run( + ["zstd", "-d", fpath, "--stdout"], + capture_output=True, + check=True, + ) + return json.loads(result.stdout) + elif fname.endswith(".json"): + with open(fpath) as f: + return json.load(f) + + raise RuntimeError(f"No SBOM file found after pulling {sbom_ref}") + + +def parse_sbom_packages(sbom: dict) -> dict[str, str]: packages = {} - for img, manifest in manifests.items(): + for artifact in sbom.get("artifacts", []): + # Only process RPM packages + if artifact.get("type") != "rpm": + continue + name = artifact.get("name") + version = artifact.get("version") + if name and version: + # If we see the same package, keep the one with epoch (more specific) + if name not in packages or (":" in version and ":" not in packages[name]): + packages[name] = version + return packages + + +def get_packages(target: str, images: list[tuple[str, str, str, str]]): + packages = {} + for j, (img, _, _, _) in enumerate(images): + print(f"Getting packages for {img}:{target} via SBOM ({j+1}/{len(images)})") try: - packages[img] = json.loads(manifest["Labels"]["dev.hhd.rechunk.info"])[ - "packages" - ] + full_image = f"{REGISTRY}{img}" + digest = get_image_digest(full_image, target) + sbom = get_sbom(full_image, digest) + packages[img] = parse_sbom_packages(sbom) + print(f" Found {len(packages[img])} packages") except Exception as e: - print(f"Failed to get packages for {img}:\n{e}") + print(f" Failed to get packages for {img}:{target}: {e}") + raise e return packages -def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, Any]): +def get_package_groups(target: str, prev_tag: str, curr_tag: str): common = set() others = {k: set() for k in OTHER_NAMES.keys()} - npkg = get_packages(manifests) - ppkg = get_packages(prev) + images = list(get_images(target)) + + print(f"\nFetching current packages for {curr_tag}...") + npkg = get_packages(curr_tag, images) + print(f"\nFetching previous packages for {prev_tag}...") + ppkg = get_packages(prev_tag, images) keys = set(npkg.keys()) | set(ppkg.keys()) pkg = defaultdict(set) @@ -232,16 +315,16 @@ def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, A first = False - return sorted(common), {k: sorted(v) for k, v in others.items()} + return sorted(common), {k: sorted(v) for k, v in others.items()}, npkg, ppkg -def get_versions(manifests: dict[str, Any]): +def get_versions(packages: dict[str, dict[str, str]]): + """Extract version info from packages dict, stripping epoch prefix and Fedora suffix.""" versions = {} - pkgs = get_packages(manifests) - for img_pkgs in pkgs.values(): + for img_pkgs in packages.values(): for pkg, v in img_pkgs.items(): + v = re.sub(EPOCH_PATTERN, "", v) v = re.sub(FEDORA_PATTERN, "", v) - v = re.sub(r"\.switcheroo", "", v) versions[pkg] = v return versions @@ -338,14 +421,16 @@ def generate_changelog( target: str, pretty: str | None, workdir: str, + prev_tag: str, + curr_tag: str, prev_manifests, manifests, ): - common, others = get_package_groups(target, prev_manifests, manifests) - versions = get_versions(manifests) - prev_versions = get_versions(prev_manifests) + common, others, curr_packages, prev_packages = get_package_groups(target, prev_tag, curr_tag) + versions = get_versions(curr_packages) + prev_versions = get_versions(prev_packages) - prev, curr = get_tags(target, manifests) + prev, curr = prev_tag, curr_tag if not pretty: # Generate pretty version since we dont have it @@ -447,6 +532,8 @@ def main(): target, args.pretty, args.workdir, + prev, + curr, prev_manifests, manifests, ) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a396efec7fa..e3993079c78 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -54,7 +54,7 @@ }, { "matchUpdateTypes": ["pin", "digest", "pinDigest"], - "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer", "ghcr.io/hhd-dev/rechunk"], + "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer"], "automerge": false }, { diff --git a/.github/workflows/generate-release.yml b/.github/workflows/generate-release.yml index 048abc3f231..69b6feecef4 100644 --- a/.github/workflows/generate-release.yml +++ b/.github/workflows/generate-release.yml @@ -16,13 +16,14 @@ on: options: - '["stable"]' -permissions: - contents: write +permissions: {} name: Generate Release jobs: generate-release: runs-on: ubuntu-latest + permissions: + contents: write strategy: fail-fast: false matrix: @@ -44,6 +45,9 @@ jobs: run: | just check + - name: Install ORAS + uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0 + - name: Generate Release Text id: generate-release-text shell: bash diff --git a/.gitignore b/.gitignore index 9bf5d13c2e2..dba86ff7c3d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ previous.manifest.json changelog.md output.env version.txt +sbom_out/ devcontainer diff --git a/Justfile b/Justfile index 29e81b29be5..d89a21f0616 100644 --- a/Justfile +++ b/Justfile @@ -679,6 +679,34 @@ tag-images image_name="" default_tag="" tags="": # Show Images ${PODMAN} images +# Extract Container and generate SBOM +[group('Utility')] +gen-sbom $image="bluefin" $tag="latest" $flavor="main" $syft_cmd="syft": + #!/usr/bin/bash + set -eoux pipefail + + image_name=$({{ just }} image_name '{{ image }}' '{{ tag }}' '{{ flavor }}') + + OUT_DIR="sbom_out/${image_name}" + mkdir -p "${OUT_DIR}" + + # We have to do it this stupid way because we are OOMing on github runners + # https://github.com/anchore/syft/issues/3800 + ${PODMAN} container create --replace --name ${image_name} "${image_name}:${tag}" + + ROOTFS="${OUT_DIR}/rootfs" + mkdir -p "${ROOTFS}" + + ${PODMAN} export ${image_name} | tar -C "${ROOTFS}" -xf - + ${PODMAN} container rm ${image_name} + + SBOM="${OUT_DIR}/sbom.json" + + ${syft_cmd} --source-name "${image_name}:${tag}" "${OUT_DIR}" -o syft-json=${SBOM} + du -sh "${SBOM}" + + rm -rf "${ROOTFS}" + # DNF CI package cache [group('Utility')] setup-cache $image="bluefin" $tag="latest" $ghcr="0" $github_event="0":