feat(changelogs): Use SBOMs for standardized package data extraction

Refactors the changelog generation script to fetch package information
from image Software Bill of Materials (SBOMs) instead of custom labels.

This is split out of #4564 to derisk the change

Author: dylanmtaylor

.github/changelogs.py

@@ -1,12 +1,14 @@
 from itertools import product
 import subprocess
 import json
+import os
+import tempfile
 import time
 from typing import Any
 import re
 from collections import defaultdict
 
-REGISTRY = "docker://ghcr.io/ublue-os/"
+REGISTRY = "ghcr.io/ublue-os/"
 
 IMAGE_MATRIX_LATEST = {
     "experience": ["base", "dx"],
@@ -22,6 +24,7 @@
 RETRIES = 3
 RETRY_WAIT = 5
 FEDORA_PATTERN = re.compile(r"\.fc\d\d")
+EPOCH_PATTERN = re.compile(r"^\d+:")
 START_PATTERN = lambda target: re.compile(rf"{target}-\d\d\d+")
 
 PATTERN_ADD = "\n| ✨ | {name} | | {version} |"
@@ -125,7 +128,7 @@ def get_manifests(target: str):
         for i in range(RETRIES):
             try:
                 output = subprocess.run(
-                    ["skopeo", "inspect", REGISTRY + img + ":" + target],
+                    ["skopeo", "inspect", f"docker://{REGISTRY}{img}:{target}"],
                     check=True,
                     stdout=subprocess.PIPE,
                 ).stdout
@@ -165,24 +168,104 @@ def get_tags(target: str, manifests: dict[str, Any]):
     return tags[-2], tags[-1]
 
 
-def get_packages(manifests: dict[str, Any]):
+def get_image_digest(image: str, tag: str) -> str:
+    """Get image digest using skopeo."""
+    result = subprocess.run(
+        ["skopeo", "inspect", f"docker://{image}:{tag}"],
+        capture_output=True,
+        text=True,
+        check=True
+    )
+    return json.loads(result.stdout)["Digest"]
+
+
+def get_sbom(image: str, digest: str) -> dict:
+    """Fetch SBOM using ORAS."""
+    full_ref = f"{image}@{digest}"
+
+    # Find the SBOM referrer attached to this image
+    result = subprocess.run(
+        ["oras", "discover", "--format", "json", full_ref],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    discovered = json.loads(result.stdout)
+
+    sbom_digest = None
+    for referrer in discovered.get("referrers", []):
+        if "spdx+json" in referrer.get("artifactType", ""):
+            sbom_digest = referrer["digest"]
+            break
+
+    if sbom_digest is None:
+        raise RuntimeError(f"No SBOM referrer found for {full_ref}")
+
+    sbom_ref = f"{image}@{sbom_digest}"
+    with tempfile.TemporaryDirectory() as tmpdir:
+        subprocess.run(
+            ["oras", "pull", sbom_ref],
+            capture_output=True,
+            check=True,
+            cwd=tmpdir,
+        )
+        for fname in os.listdir(tmpdir):
+            fpath = os.path.join(tmpdir, fname)
+            if fname.endswith(".zst"):
+                result = subprocess.run(
+                    ["zstd", "-d", fpath, "--stdout"],
+                    capture_output=True,
+                    check=True,
+                )
+                return json.loads(result.stdout)
+            elif fname.endswith(".json"):
+                with open(fpath) as f:
+                    return json.load(f)
+
+    raise RuntimeError(f"No SBOM file found after pulling {sbom_ref}")
+
+
+def parse_sbom_packages(sbom: dict) -> dict[str, str]:
     packages = {}
-    for img, manifest in manifests.items():
+    for artifact in sbom.get("artifacts", []):
+        # Only process RPM packages
+        if artifact.get("type") != "rpm":
+            continue
+        name = artifact.get("name")
+        version = artifact.get("version")
+        if name and version:
+            # If we see the same package, keep the one with epoch (more specific)
+            if name not in packages or (":" in version and ":" not in packages[name]):
+                packages[name] = version
+    return packages
+
+
+def get_packages(target: str, images: list[tuple[str, str, str, str]]):
+    packages = {}
+    for j, (img, _, _, _) in enumerate(images):
+        print(f"Getting packages for {img}:{target} via SBOM ({j+1}/{len(images)})")
         try:
-            packages[img] = json.loads(manifest["Labels"]["dev.hhd.rechunk.info"])[
-                "packages"
-            ]
+            full_image = f"{REGISTRY}{img}"
+            digest = get_image_digest(full_image, target)
+            sbom = get_sbom(full_image, digest)
+            packages[img] = parse_sbom_packages(sbom)
+            print(f"  Found {len(packages[img])} packages")
         except Exception as e:
-            print(f"Failed to get packages for {img}:\n{e}")
+            print(f"  Failed to get packages for {img}:{target}: {e}")
+            raise e
     return packages
 
 
-def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, Any]):
+def get_package_groups(target: str, prev_tag: str, curr_tag: str):
     common = set()
     others = {k: set() for k in OTHER_NAMES.keys()}
 
-    npkg = get_packages(manifests)
-    ppkg = get_packages(prev)
+    images = list(get_images(target))
+
+    print(f"\nFetching current packages for {curr_tag}...")
+    npkg = get_packages(curr_tag, images)
+    print(f"\nFetching previous packages for {prev_tag}...")
+    ppkg = get_packages(prev_tag, images)
 
     keys = set(npkg.keys()) | set(ppkg.keys())
     pkg = defaultdict(set)
@@ -232,16 +315,16 @@ def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, A
 
             first = False
 
-    return sorted(common), {k: sorted(v) for k, v in others.items()}
+    return sorted(common), {k: sorted(v) for k, v in others.items()}, npkg, ppkg
 
 
-def get_versions(manifests: dict[str, Any]):
+def get_versions(packages: dict[str, dict[str, str]]):
+    """Extract version info from packages dict, stripping epoch prefix and Fedora suffix."""
     versions = {}
-    pkgs = get_packages(manifests)
-    for img_pkgs in pkgs.values():
+    for img_pkgs in packages.values():
         for pkg, v in img_pkgs.items():
+            v = re.sub(EPOCH_PATTERN, "", v)
             v = re.sub(FEDORA_PATTERN, "", v)
-            v = re.sub(r"\.switcheroo", "", v)
             versions[pkg] = v
     return versions
 
@@ -338,14 +421,16 @@ def generate_changelog(
     target: str,
     pretty: str | None,
     workdir: str,
+    prev_tag: str,
+    curr_tag: str,
     prev_manifests,
     manifests,
 ):
-    common, others = get_package_groups(target, prev_manifests, manifests)
-    versions = get_versions(manifests)
-    prev_versions = get_versions(prev_manifests)
+    common, others, curr_packages, prev_packages = get_package_groups(target, prev_tag, curr_tag)
+    versions = get_versions(curr_packages)
+    prev_versions = get_versions(prev_packages)
 
-    prev, curr = get_tags(target, manifests)
+    prev, curr = prev_tag, curr_tag
 
     if not pretty:
         # Generate pretty version since we dont have it
@@ -447,6 +532,8 @@ def main():
         target,
         args.pretty,
         args.workdir,
+        prev,
+        curr,
         prev_manifests,
         manifests,
     )

.github/renovate.json5

@@ -54,7 +54,7 @@
     },
     {
       "matchUpdateTypes": ["pin", "digest", "pinDigest"],
-      "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer", "ghcr.io/hhd-dev/rechunk"],
+      "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer"],
       "automerge": false
     },
     {

.github/workflows/generate-release.yml

@@ -16,13 +16,14 @@ on:
         options:
           - '["stable"]'
 
-permissions:
-  contents: write
+permissions: {}
 
 name: Generate Release
 jobs:
   generate-release:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     strategy:
       fail-fast: false
       matrix:
@@ -44,6 +45,9 @@ jobs:
         run: |
           just check
 
+      - name: Install ORAS
+        uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
+
       - name: Generate Release Text
         id: generate-release-text
         shell: bash

.gitignore

@@ -7,6 +7,7 @@ previous.manifest.json
 changelog.md
 output.env
 version.txt
+sbom_out/
 
 devcontainer
 

Justfile

@@ -679,6 +679,34 @@ tag-images image_name="" default_tag="" tags="":
     # Show Images
     ${PODMAN} images
 
+# Extract Container and generate SBOM
+[group('Utility')]
+gen-sbom $image="bluefin" $tag="latest" $flavor="main" $syft_cmd="syft":
+    #!/usr/bin/bash
+    set -eoux pipefail
+
+    image_name=$({{ just }} image_name '{{ image }}' '{{ tag }}' '{{ flavor }}')
+
+    OUT_DIR="sbom_out/${image_name}"
+    mkdir -p "${OUT_DIR}"
+
+    # We have to do it this stupid way because we are OOMing on github runners
+    # https://github.com/anchore/syft/issues/3800
+    ${PODMAN} container create --replace --name ${image_name} "${image_name}:${tag}"
+
+    ROOTFS="${OUT_DIR}/rootfs"
+    mkdir -p "${ROOTFS}"
+
+    ${PODMAN} export ${image_name} | tar -C "${ROOTFS}" -xf -
+    ${PODMAN} container rm ${image_name}
+
+    SBOM="${OUT_DIR}/sbom.json"
+
+    ${syft_cmd} --source-name "${image_name}:${tag}" "${OUT_DIR}" -o syft-json=${SBOM}
+    du -sh "${SBOM}"
+
+    rm -rf "${ROOTFS}"
+
 # DNF CI package cache
 [group('Utility')]
 setup-cache $image="bluefin" $tag="latest" $ghcr="0" $github_event="0":