diff --git a/.github/changelogs.py b/.github/changelogs.py index 5ad48dc3938..63d702c4fa0 100644 --- a/.github/changelogs.py +++ b/.github/changelogs.py @@ -1,12 +1,14 @@ from itertools import product import subprocess import json +import os +import tempfile import time from typing import Any import re from collections import defaultdict -REGISTRY = "docker://ghcr.io/ublue-os/" +REGISTRY = "ghcr.io/ublue-os/" IMAGE_MATRIX_LATEST = { "experience": ["base", "dx"], @@ -22,6 +24,7 @@ RETRIES = 3 RETRY_WAIT = 5 FEDORA_PATTERN = re.compile(r"\.fc\d\d") +EPOCH_PATTERN = re.compile(r"^\d+:") START_PATTERN = lambda target: re.compile(rf"{target}-\d\d\d+") PATTERN_ADD = "\n| ✨ | {name} | | {version} |" @@ -125,7 +128,7 @@ def get_manifests(target: str): for i in range(RETRIES): try: output = subprocess.run( - ["skopeo", "inspect", REGISTRY + img + ":" + target], + ["skopeo", "inspect", f"docker://{REGISTRY}{img}:{target}"], check=True, stdout=subprocess.PIPE, ).stdout @@ -165,24 +168,104 @@ def get_tags(target: str, manifests: dict[str, Any]): return tags[-2], tags[-1] -def get_packages(manifests: dict[str, Any]): +def get_image_digest(image: str, tag: str) -> str: + """Get image digest using skopeo.""" + result = subprocess.run( + ["skopeo", "inspect", f"docker://{image}:{tag}"], + capture_output=True, + text=True, + check=True + ) + return json.loads(result.stdout)["Digest"] + + +def get_sbom(image: str, digest: str) -> dict: + """Fetch SBOM using ORAS.""" + full_ref = f"{image}@{digest}" + + # Find the SBOM referrer attached to this image + result = subprocess.run( + ["oras", "discover", "--format", "json", full_ref], + capture_output=True, + text=True, + check=True, + ) + discovered = json.loads(result.stdout) + + sbom_digest = None + for referrer in discovered.get("referrers", []): + if "spdx+json" in referrer.get("artifactType", ""): + sbom_digest = referrer["digest"] + break + + if sbom_digest is None: + raise RuntimeError(f"No SBOM referrer found for {full_ref}") + + sbom_ref = f"{image}@{sbom_digest}" + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run( + ["oras", "pull", sbom_ref], + capture_output=True, + check=True, + cwd=tmpdir, + ) + for fname in os.listdir(tmpdir): + fpath = os.path.join(tmpdir, fname) + if fname.endswith(".zst"): + result = subprocess.run( + ["zstd", "-d", fpath, "--stdout"], + capture_output=True, + check=True, + ) + return json.loads(result.stdout) + elif fname.endswith(".json"): + with open(fpath) as f: + return json.load(f) + + raise RuntimeError(f"No SBOM file found after pulling {sbom_ref}") + + +def parse_sbom_packages(sbom: dict) -> dict[str, str]: packages = {} - for img, manifest in manifests.items(): + for artifact in sbom.get("artifacts", []): + # Only process RPM packages + if artifact.get("type") != "rpm": + continue + name = artifact.get("name") + version = artifact.get("version") + if name and version: + # If we see the same package, keep the one with epoch (more specific) + if name not in packages or (":" in version and ":" not in packages[name]): + packages[name] = version + return packages + + +def get_packages(target: str, images: list[tuple[str, str, str, str]]): + packages = {} + for j, (img, _, _, _) in enumerate(images): + print(f"Getting packages for {img}:{target} via SBOM ({j+1}/{len(images)})") try: - packages[img] = json.loads(manifest["Labels"]["dev.hhd.rechunk.info"])[ - "packages" - ] + full_image = f"{REGISTRY}{img}" + digest = get_image_digest(full_image, target) + sbom = get_sbom(full_image, digest) + packages[img] = parse_sbom_packages(sbom) + print(f" Found {len(packages[img])} packages") except Exception as e: - print(f"Failed to get packages for {img}:\n{e}") + print(f" Failed to get packages for {img}:{target}: {e}") + raise e return packages -def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, Any]): +def get_package_groups(target: str, prev_tag: str, curr_tag: str): common = set() others = {k: set() for k in OTHER_NAMES.keys()} - npkg = get_packages(manifests) - ppkg = get_packages(prev) + images = list(get_images(target)) + + print(f"\nFetching current packages for {curr_tag}...") + npkg = get_packages(curr_tag, images) + print(f"\nFetching previous packages for {prev_tag}...") + ppkg = get_packages(prev_tag, images) keys = set(npkg.keys()) | set(ppkg.keys()) pkg = defaultdict(set) @@ -232,16 +315,16 @@ def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, A first = False - return sorted(common), {k: sorted(v) for k, v in others.items()} + return sorted(common), {k: sorted(v) for k, v in others.items()}, npkg, ppkg -def get_versions(manifests: dict[str, Any]): +def get_versions(packages: dict[str, dict[str, str]]): + """Extract version info from packages dict, stripping epoch prefix and Fedora suffix.""" versions = {} - pkgs = get_packages(manifests) - for img_pkgs in pkgs.values(): + for img_pkgs in packages.values(): for pkg, v in img_pkgs.items(): + v = re.sub(EPOCH_PATTERN, "", v) v = re.sub(FEDORA_PATTERN, "", v) - v = re.sub(r"\.switcheroo", "", v) versions[pkg] = v return versions @@ -338,14 +421,16 @@ def generate_changelog( target: str, pretty: str | None, workdir: str, + prev_tag: str, + curr_tag: str, prev_manifests, manifests, ): - common, others = get_package_groups(target, prev_manifests, manifests) - versions = get_versions(manifests) - prev_versions = get_versions(prev_manifests) + common, others, curr_packages, prev_packages = get_package_groups(target, prev_tag, curr_tag) + versions = get_versions(curr_packages) + prev_versions = get_versions(prev_packages) - prev, curr = get_tags(target, manifests) + prev, curr = prev_tag, curr_tag if not pretty: # Generate pretty version since we dont have it @@ -447,6 +532,8 @@ def main(): target, args.pretty, args.workdir, + prev, + curr, prev_manifests, manifests, ) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a396efec7fa..e3993079c78 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -54,7 +54,7 @@ }, { "matchUpdateTypes": ["pin", "digest", "pinDigest"], - "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer", "ghcr.io/hhd-dev/rechunk"], + "matchPackageNames": ["ghcr.io/jasonn3/build-container-installer"], "automerge": false }, { diff --git a/.github/workflows/generate-release.yml b/.github/workflows/generate-release.yml index 048abc3f231..69b6feecef4 100644 --- a/.github/workflows/generate-release.yml +++ b/.github/workflows/generate-release.yml @@ -16,13 +16,14 @@ on: options: - '["stable"]' -permissions: - contents: write +permissions: {} name: Generate Release jobs: generate-release: runs-on: ubuntu-latest + permissions: + contents: write strategy: fail-fast: false matrix: @@ -44,6 +45,9 @@ jobs: run: | just check + - name: Install ORAS + uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0 + - name: Generate Release Text id: generate-release-text shell: bash diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index b1cc9835100..000c37f4a47 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -146,22 +146,15 @@ jobs: if: github.event_name != 'pull_request' id: generate-sbom env: - IMAGE: ${{ env.IMAGE_NAME }} - STREAM_NAME: ${{ matrix.stream_name }} + MATRIX_BASE_NAME: ${{ matrix.base_name }} + MATRIX_STREAM_NAME: ${{ matrix.stream_name }} + MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }} SYFT_CMD: ${{ steps.setup-syft.outputs.cmd }} - OCI_DIR: "/tmp/image-oci-dir" run: | - mkdir -p ${OCI_DIR}/rootfs - sudo podman container create --replace --name "${IMAGE}" "localhost/${IMAGE}:${STREAM_NAME}" - sudo podman export "${IMAGE}" | sudo tar -C ${OCI_DIR}/rootfs -xf - - sudo podman container rm "${IMAGE}" - - SBOM="$(mktemp -d)/sbom.json" - export SYFT_PARALLELISM=$(($(nproc)*2)) - sudo $SYFT_CMD --source-name "${IMAGE}"-"${STREAM_NAME}" ${OCI_DIR} -o syft-json=${SBOM} - du -sh ${SBOM} - echo "SBOM=${SBOM}" >> $GITHUB_OUTPUT - sudo rm -rf ${OCI_DIR} + sudo -E $(command -v just) gen-sbom "${MATRIX_BASE_NAME}" \ + "${MATRIX_STREAM_NAME}" \ + "${MATRIX_IMAGE_FLAVOR}" \ + "${SYFT_CMD}" - name: Rechunk Image id: rechunk-image @@ -288,8 +281,10 @@ jobs: env: IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }} DIGEST: ${{ steps.push.outputs.digest }} - SBOM: ${{ steps.generate-sbom.outputs.SBOM }} + IMAGE_NAME: ${{ env.IMAGE_NAME }} run: | + SBOM="sbom_out/${IMAGE_NAME}/sbom.json" + cd "$(dirname "${SBOM}")" oras attach \ --artifact-type application/vnd.spdx+json \ diff --git a/.gitignore b/.gitignore index 9bf5d13c2e2..dba86ff7c3d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ previous.manifest.json changelog.md output.env version.txt +sbom_out/ devcontainer diff --git a/Justfile b/Justfile index 29e81b29be5..d89a21f0616 100644 --- a/Justfile +++ b/Justfile @@ -679,6 +679,34 @@ tag-images image_name="" default_tag="" tags="": # Show Images ${PODMAN} images +# Extract Container and generate SBOM +[group('Utility')] +gen-sbom $image="bluefin" $tag="latest" $flavor="main" $syft_cmd="syft": + #!/usr/bin/bash + set -eoux pipefail + + image_name=$({{ just }} image_name '{{ image }}' '{{ tag }}' '{{ flavor }}') + + OUT_DIR="sbom_out/${image_name}" + mkdir -p "${OUT_DIR}" + + # We have to do it this stupid way because we are OOMing on github runners + # https://github.com/anchore/syft/issues/3800 + ${PODMAN} container create --replace --name ${image_name} "${image_name}:${tag}" + + ROOTFS="${OUT_DIR}/rootfs" + mkdir -p "${ROOTFS}" + + ${PODMAN} export ${image_name} | tar -C "${ROOTFS}" -xf - + ${PODMAN} container rm ${image_name} + + SBOM="${OUT_DIR}/sbom.json" + + ${syft_cmd} --source-name "${image_name}:${tag}" "${OUT_DIR}" -o syft-json=${SBOM} + du -sh "${SBOM}" + + rm -rf "${ROOTFS}" + # DNF CI package cache [group('Utility')] setup-cache $image="bluefin" $tag="latest" $ghcr="0" $github_event="0":