feat(changelogs): Use SBOMs for standardized package data extraction (#4635)

Author: dylanmtaylor

.github/changelogs.py

@@ -1,12 +1,14 @@
 from itertools import product
 import subprocess
 import json
+import os
+import tempfile
 import time
 from typing import Any
 import re
 from collections import defaultdict
 
-REGISTRY = "docker://ghcr.io/ublue-os/"
+REGISTRY = "ghcr.io/ublue-os/"
 
 IMAGE_MATRIX_LATEST = {
     "experience": ["base", "dx"],
@@ -22,6 +24,7 @@
 RETRIES = 3
 RETRY_WAIT = 5
 FEDORA_PATTERN = re.compile(r"\.fc\d\d")
+EPOCH_PATTERN = re.compile(r"^\d+:")
 START_PATTERN = lambda target: re.compile(rf"{target}-\d\d\d+")
 
 PATTERN_ADD = "\n| ✨ | {name} | | {version} |"
@@ -125,7 +128,7 @@ def get_manifests(target: str):
         for i in range(RETRIES):
             try:
                 output = subprocess.run(
-                    ["skopeo", "inspect", REGISTRY + img + ":" + target],
+                    ["skopeo", "inspect", f"docker://{REGISTRY}{img}:{target}"],
                     check=True,
                     stdout=subprocess.PIPE,
                 ).stdout
@@ -165,24 +168,104 @@ def get_tags(target: str, manifests: dict[str, Any]):
     return tags[-2], tags[-1]
 
 
-def get_packages(manifests: dict[str, Any]):
+def get_image_digest(image: str, tag: str) -> str:
+    """Get image digest using skopeo."""
+    result = subprocess.run(
+        ["skopeo", "inspect", f"docker://{image}:{tag}"],
+        capture_output=True,
+        text=True,
+        check=True
+    )
+    return json.loads(result.stdout)["Digest"]
+
+
+def get_sbom(image: str, digest: str) -> dict:
+    """Fetch SBOM using ORAS."""
+    full_ref = f"{image}@{digest}"
+
+    # Find the SBOM referrer attached to this image
+    result = subprocess.run(
+        ["oras", "discover", "--format", "json", full_ref],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    discovered = json.loads(result.stdout)
+
+    sbom_digest = None
+    for referrer in discovered.get("referrers", []):
+        if "spdx+json" in referrer.get("artifactType", ""):
+            sbom_digest = referrer["digest"]
+            break
+
+    if sbom_digest is None:
+        raise RuntimeError(f"No SBOM referrer found for {full_ref}")
+
+    sbom_ref = f"{image}@{sbom_digest}"
+    with tempfile.TemporaryDirectory() as tmpdir:
+        subprocess.run(
+            ["oras", "pull", sbom_ref],
+            capture_output=True,
+            check=True,
+            cwd=tmpdir,
+        )
+        for fname in os.listdir(tmpdir):
+            fpath = os.path.join(tmpdir, fname)
+            if fname.endswith(".zst"):
+                result = subprocess.run(
+                    ["zstd", "-d", fpath, "--stdout"],
+                    capture_output=True,
+                    check=True,
+                )
+                return json.loads(result.stdout)
+            elif fname.endswith(".json"):
+                with open(fpath) as f:
+                    return json.load(f)
+
+    raise RuntimeError(f"No SBOM file found after pulling {sbom_ref}")
+
+
+def parse_sbom_packages(sbom: dict) -> dict[str, str]:
     packages = {}
-    for img, manifest in manifests.items():
+    for artifact in sbom.get("artifacts", []):
+        # Only process RPM packages
+        if artifact.get("type") != "rpm":
+            continue
+        name = artifact.get("name")
+        version = artifact.get("version")
+        if name and version:
+            # If we see the same package, keep the one with epoch (more specific)
+            if name not in packages or (":" in version and ":" not in packages[name]):
+                packages[name] = version
+    return packages
+
+
+def get_packages(target: str, images: list[tuple[str, str, str, str]]):
+    packages = {}
+    for j, (img, _, _, _) in enumerate(images):
+        print(f"Getting packages for {img}:{target} via SBOM ({j+1}/{len(images)})")
         try:
-            packages[img] = json.loads(manifest["Labels"]["dev.hhd.rechunk.info"])[
-                "packages"
-            ]
+            full_image = f"{REGISTRY}{img}"
+            digest = get_image_digest(full_image, target)
+            sbom = get_sbom(full_image, digest)
+            packages[img] = parse_sbom_packages(sbom)
+            print(f"  Found {len(packages[img])} packages")
         except Exception as e:
-            print(f"Failed to get packages for {img}:\n{e}")
+            print(f"  Failed to get packages for {img}:{target}: {e}")
+            raise e
     return packages
 
 
-def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, Any]):
+def get_package_groups(target: str, prev_tag: str, curr_tag: str):
     common = set()
     others = {k: set() for k in OTHER_NAMES.keys()}
 
-    npkg = get_packages(manifests)
-    ppkg = get_packages(prev)
+    images = list(get_images(target))
+
+    print(f"\nFetching current packages for {curr_tag}...")
+    npkg = get_packages(curr_tag, images)
+    print(f"\nFetching previous packages for {prev_tag}...")
+    ppkg = get_packages(prev_tag, images)
 
     keys = set(npkg.keys()) | set(ppkg.keys())
     pkg = defaultdict(set)
@@ -232,16 +315,16 @@ def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, A
 
             first = False
 
-    return sorted(common), {k: sorted(v) for k, v in others.items()}
+    return sorted(common), {k: sorted(v) for k, v in others.items()}, npkg, ppkg
 
 
-def get_versions(manifests: dict[str, Any]):
+def get_versions(packages: dict[str, dict[str, str]]):
+    """Extract version info from packages dict, stripping epoch prefix and Fedora suffix."""
     versions = {}
-    pkgs = get_packages(manifests)
-    for img_pkgs in pkgs.values():
+    for img_pkgs in packages.values():
         for pkg, v in img_pkgs.items():
+            v = re.sub(EPOCH_PATTERN, "", v)
             v = re.sub(FEDORA_PATTERN, "", v)
-            v = re.sub(r"\.switcheroo", "", v)
             versions[pkg] = v
     return versions
 
@@ -338,14 +421,16 @@ def generate_changelog(
     target: str,
     pretty: str | None,
     workdir: str,
+    prev_tag: str,
+    curr_tag: str,
     prev_manifests,
     manifests,
 ):
-    common, others = get_package_groups(target, prev_manifests, manifests)
-    versions = get_versions(manifests)
-    prev_versions = get_versions(prev_manifests)
+    common, others, curr_packages, prev_packages = get_package_groups(target, prev_tag, curr_tag)
+    versions = get_versions(curr_packages)
+    prev_versions = get_versions(prev_packages)
 
-    prev, curr = get_tags(target, manifests)
+    prev, curr = prev_tag, curr_tag
 
     if not pretty:
         # Generate pretty version since we dont have it
@@ -447,6 +532,8 @@ def main():
         target,
         args.pretty,
         args.workdir,
+        prev,
+        curr,
         prev_manifests,
         manifests,
     )

.github/renovate.json5

@@ -54,7 +54,7 @@
     },
     {
       "matchUpdateTypes": ["pin", "digest", "pinDigest"],
-      "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer", "ghcr.io/hhd-dev/rechunk"],
+      "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer"],
       "automerge": false
     },
     {

.github/workflows/generate-release.yml

@@ -16,13 +16,14 @@ on:
         options:
           - '["stable"]'
 
-permissions:
-  contents: write
+permissions: {}
 
 name: Generate Release
 jobs:
   generate-release:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     strategy:
       fail-fast: false
       matrix:
@@ -44,6 +45,9 @@ jobs:
         run: |
           just check
 
+      - name: Install ORAS
+        uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
+
       - name: Generate Release Text
         id: generate-release-text
         shell: bash

.github/workflows/reusable-build.yml

@@ -146,22 +146,15 @@ jobs:
         if: github.event_name != 'pull_request'
         id: generate-sbom
         env:
-          IMAGE: ${{ env.IMAGE_NAME }}
-          STREAM_NAME: ${{ matrix.stream_name }}
+          MATRIX_BASE_NAME: ${{ matrix.base_name }}
+          MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
+          MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
           SYFT_CMD: ${{ steps.setup-syft.outputs.cmd }}
-          OCI_DIR: "/tmp/image-oci-dir"
         run: |
-          mkdir -p ${OCI_DIR}/rootfs
-          sudo podman container create --replace --name "${IMAGE}" "localhost/${IMAGE}:${STREAM_NAME}"
-          sudo podman export "${IMAGE}" | sudo tar -C ${OCI_DIR}/rootfs -xf -
-          sudo podman container rm "${IMAGE}"
-
-          SBOM="$(mktemp -d)/sbom.json"
-          export SYFT_PARALLELISM=$(($(nproc)*2))
-          sudo $SYFT_CMD --source-name "${IMAGE}"-"${STREAM_NAME}" ${OCI_DIR} -o syft-json=${SBOM}
-          du -sh ${SBOM}
-          echo "SBOM=${SBOM}" >> $GITHUB_OUTPUT
-          sudo rm -rf ${OCI_DIR}
+          sudo -E $(command -v just) gen-sbom "${MATRIX_BASE_NAME}" \
+                            "${MATRIX_STREAM_NAME}" \
+                            "${MATRIX_IMAGE_FLAVOR}" \
+                            "${SYFT_CMD}"
 
       - name: Rechunk Image
         id: rechunk-image
@@ -288,8 +281,10 @@ jobs:
         env:
           IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}
           DIGEST: ${{ steps.push.outputs.digest }}
-          SBOM: ${{ steps.generate-sbom.outputs.SBOM }}
+          IMAGE_NAME: ${{ env.IMAGE_NAME }}
         run: |
+          SBOM="sbom_out/${IMAGE_NAME}/sbom.json"
+
           cd "$(dirname "${SBOM}")"
           oras attach \
             --artifact-type application/vnd.spdx+json \

.gitignore

@@ -7,6 +7,7 @@ previous.manifest.json
 changelog.md
 output.env
 version.txt
+sbom_out/
 
 devcontainer
 

Justfile

@@ -679,6 +679,34 @@ tag-images image_name="" default_tag="" tags="":
     # Show Images
     ${PODMAN} images
 
+# Extract Container and generate SBOM
+[group('Utility')]
+gen-sbom $image="bluefin" $tag="latest" $flavor="main" $syft_cmd="syft":
+    #!/usr/bin/bash
+    set -eoux pipefail
+
+    image_name=$({{ just }} image_name '{{ image }}' '{{ tag }}' '{{ flavor }}')
+
+    OUT_DIR="sbom_out/${image_name}"
+    mkdir -p "${OUT_DIR}"
+
+    # We have to do it this stupid way because we are OOMing on github runners
+    # https://github.com/anchore/syft/issues/3800
+    ${PODMAN} container create --replace --name ${image_name} "${image_name}:${tag}"
+
+    ROOTFS="${OUT_DIR}/rootfs"
+    mkdir -p "${ROOTFS}"
+
+    ${PODMAN} export ${image_name} | tar -C "${ROOTFS}" -xf -
+    ${PODMAN} container rm ${image_name}
+
+    SBOM="${OUT_DIR}/sbom.json"
+
+    ${syft_cmd} --source-name "${image_name}:${tag}" "${OUT_DIR}" -o syft-json=${SBOM}
+    du -sh "${SBOM}"
+
+    rm -rf "${ROOTFS}"
+
 # DNF CI package cache
 [group('Utility')]
 setup-cache $image="bluefin" $tag="latest" $ghcr="0" $github_event="0":