feat: replace legacy rechunker with rpm-ostree compose build-chunked-oci

Author: dylanmtaylor

.github/workflows/reusable-build.yml

@@ -163,25 +163,19 @@ jobs:
           echo "SBOM=${SBOM}" >> $GITHUB_OUTPUT
           sudo rm -rf ${OCI_DIR}
 
-      - name: Rechunk Image
-        id: rechunk-image
-        shell: bash
+      - name: Rechunk Image with rpm-ostree
+        id: rechunker
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MATRIX_BASE_NAME: ${{ matrix.base_name }}
+          MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
+          MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
+          DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
         run: |
-          sudo -E $(command -v just) rechunk "${{ matrix.base_name }}" \
-                            "${{ matrix.stream_name }}" \
-                            "${{ matrix.image_flavor }}" \
+          sudo -E $(command -v just) rechunk "${MATRIX_BASE_NAME}" \
+                            "${MATRIX_STREAM_NAME}" \
+                            "${MATRIX_IMAGE_FLAVOR}" \
                             "1"
 
-      - name: Load Image into Podman
-        id: load-rechunk
-        shell: bash
-        run: |
-          sudo -E $(command -v just) load-rechunk "${{ matrix.base_name }}" \
-                            "${{ env.DEFAULT_TAG }}" \
-                            "${{ matrix.image_flavor }}"
-
       - name: Secureboot Check
         id: secureboot
         shell: bash
@@ -190,6 +184,38 @@ jobs:
                           "${{ env.DEFAULT_TAG }}" \
                           "${{ matrix.image_flavor }}"
 
+      - name: Export to OCI Archive
+        if: github.event_name == 'pull_request'
+        id: oci-archive
+        env:
+          IMAGE_NAME: ${{ env.IMAGE_NAME }}
+          DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
+        run: |
+          sudo -E $(command -v just) export-oci "${{ matrix.base_name }}" \
+                            "${{ env.DEFAULT_TAG }}" \
+                            "${{ matrix.image_flavor }}"
+
+      - name: Upload OCI dir as Artifact
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+        with:
+          name: ${{ env.IMAGE_NAME }}.oci
+          path: ${{ env.IMAGE_NAME }}.oci
+          archive: false
+          if-no-files-found: error
+          retention-days: 1
+
+      - name: PR Testing Instructions
+        if: github.event_name == 'pull_request'
+        id: pr-summary
+        env:
+          IMAGE_NAME: ${{ env.IMAGE_NAME }}
+          MATRIX_STREAM_NAME: "${{ matrix.stream_name }}"
+        run: |
+          echo "Download the .oci file" >> $GITHUB_STEP_SUMMARY
+          echo "Rebase: sudo bootc switch --transport oci-archive /path/to/${IMAGE_NAME}.oci" >> $GITHUB_STEP_SUMMARY
+          echo "Go back to the production image e.g.: sudo bootc switch --enforce-container-sigpolicy ghcr.io/ublue-os/${IMAGE_NAME}:${MATRIX_STREAM_NAME}" >> $GITHUB_STEP_SUMMARY
+
       - name: Generate tags
         id: generate-tags
         shell: bash
@@ -225,10 +251,18 @@ jobs:
         with:
           string: ${{ env.IMAGE_REGISTRY }}
 
+      # TODO: remove me when we have a new podman in 26.04 runners
+      # needed because old podman doesn't push layer annotations for
+      # the rpm-ostree rechunker at all
+      - name: install podman from brew
+        if: github.event_name != 'pull_request'
+        run: |
+          /home/linuxbrew/.linuxbrew/bin/brew install podman
+
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
         run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | podman login ghcr.io -u ${{ github.actor }} --password-stdin
+          echo ${{ secrets.GITHUB_TOKEN }} | /home/linuxbrew/.linuxbrew/bin/podman login ghcr.io -u ${{ github.actor }} --password-stdin
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
       - name: Push to GHCR
@@ -241,9 +275,15 @@ jobs:
           timeout_minutes: 30
           command: |
             set -euox pipefail
+            # HACK: push a second time so layer annotations are pushed
+            # TODO: remove me when https://github.com/containers/podman/issues/27796 fixed
+
+            for tag in ${{ steps.generate-tags.outputs.alias_tags }}; do
+              sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${{ env.IMAGE_NAME }}:${tag} ${{ steps.registry_case.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${tag}
+            done
 
             for tag in ${{ steps.generate-tags.outputs.alias_tags }}; do
-              sudo -E podman push ${{ env.IMAGE_NAME }}:${tag} ${{ steps.registry_case.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${tag}
+              sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${{ env.IMAGE_NAME }}:${tag} ${{ steps.registry_case.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${tag}
             done
 
             digest=$(skopeo inspect docker://${{ steps.registry_case.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }} --format '{{.Digest}}')

.gitignore

@@ -1,8 +1,7 @@
 flatpaks_with_deps
 flatpak.*
 
-*_build
-*_build.*
+bluefin*.oci
 previous.manifest.json
 changelog.md
 output.env

Justfile

@@ -1,5 +1,4 @@
 repo_organization := "ublue-os"
-rechunker_image := "ghcr.io/ublue-os/legacy-rechunk:v1.0.1-x86_64@sha256:2627cbf92ca60ab7372070dcf93b40f457926f301509ffba47a04d6a9e1ddaf7"
 common_image := "ghcr.io/projectbluefin/common:latest"
 brew_image := "ghcr.io/ublue-os/brew:latest"
 images := '(
@@ -235,7 +234,7 @@ build $image="bluefin" $tag="latest" $flavor="main" rechunk="0" ghcr="0" pipelin
     elif [[ "{{ rechunk }}" == "1" && "{{ ghcr }}" == "1" ]]; then
         ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}" 1
     elif [[ "{{ rechunk }}" == "1" ]]; then
-        ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}"
+        {{ just }} rechunk "${image}" "${tag}" "${flavor}"
     fi
 
 # Build Image and Rechunk
@@ -259,183 +258,86 @@ build-pipeline image="bluefin" tag="latest" flavor="main" kernel_pin="":
     #!/usr/bin/bash
     ${SUDOIF} {{ just }} build {{ image }} {{ tag }} {{ flavor }} 1 1 1 {{ kernel_pin }}
 
-# Rechunk Image
+# Rechunk Image with rpm-ostree
 [group('Image')]
 [private]
 rechunk $image="bluefin" $tag="latest" $flavor="main" ghcr="0" pipeline="0":
     #!/usr/bin/bash
 
-    echo "::group:: Rechunk Prep"
     set -eoux pipefail
 
     # Validate
     {{ just }} validate "${image}" "${tag}" "${flavor}"
 
     # Image Name
     image_name=$({{ just }} image_name {{ image }} {{ tag }} {{ flavor }})
+    fedora_version=$({{ just }} fedora_version {{ image }} {{ tag }} {{ flavor }})
+    DEFAULT_TAG=$({{ just }} generate-default-tag {{ tag }} {{ ghcr }})
 
-    # Check if image is already built
-    ID=$(${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'")
-    if [[ -z "$ID" ]]; then
-        {{ just }} build "${image}" "${tag}" "${flavor}"
-    fi
-
-    # Load into Rootful Podman
-    ID=$(${SUDOIF} ${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'")
-    if [[ -z "$ID" && ! ${PODMAN} =~ docker ]]; then
-        COPYTMP=$(mktemp -p "${PWD}" -d -t podman_scp.XXXXXXXXXX)
-        ${SUDOIF} TMPDIR=${COPYTMP} ${PODMAN} image scp ${UID}@localhost::localhost/"${image_name}":"${tag}" root@localhost::localhost/"${image_name}":"${tag}"
-        rm -rf "${COPYTMP}"
+    if [[ "{{ ghcr }}" == "0" ]]; then
+      {{ just }} load-rootful "${image}" "${tag}" "${flavor}"
     fi
 
-    # Prep Container
-    CREF=$(${SUDOIF} ${PODMAN} create localhost/"${image_name}":"${tag}" bash)
-    OLD_IMAGE=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Image')
-    OUT_NAME="${image_name}_build"
-    MOUNT=$(${SUDOIF} ${PODMAN} mount "${CREF}")
-
-    # Fedora Version
-    fedora_version=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]' | grep -oP 'fc\K[0-9]+')
-
-    # Label Version
-    VERSION=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["org.opencontainers.image.version"]')
-
-    # Git SHA
-    SHA="dedbeef"
-    if [[ -z "$(git status -s)" ]]; then
-        SHA=$(git rev-parse HEAD)
-    fi
-
-    # Rest of Labels
-    LABELS="
-        io.artifacthub.package.deprecated=false
-        io.artifacthub.package.keywords=bootc,fedora,bluefin,ublue,universal-blue
-        io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4
-        io.artifacthub.package.maintainers=[{\"name\": \"castrojo\", \"email\": \"jorge.castro@gmail.com\"}]
-        io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/bluefin/refs/heads/main/README.md
-        org.opencontainers.image.created=$(date -u +%Y\-%m\-%d\T%H\:%M\:%S\Z)
-        org.opencontainers.image.license=Apache-2.0
-        org.opencontainers.image.source=https://raw.githubusercontent.com/ublue-os/bluefin/refs/heads/main/Containerfile
-        org.opencontainers.image.title=${image_name}
-        org.opencontainers.image.url=https://projectbluefin.io
-        org.opencontainers.image.vendor={{ repo_organization }}
-        ostree.linux=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]')
-        containers.bootc=1
-    "
-
-    # Cleanup Space during Github Action
+    # In CI this will replace the unrechunked image
     if [[ "{{ ghcr }}" == "1" ]]; then
-        base_image_name=silverblue-main
-        if [[ "${tag}" =~ stable ]]; then
-            tag="stable-daily"
-        fi
-        ID=$(${SUDOIF} ${PODMAN} images --filter reference=ghcr.io/{{ repo_organization }}/"${base_image_name}":${fedora_version} --format "{{ '{{.ID}}' }}")
-        if [[ -n "$ID" ]]; then
-            ${PODMAN} rmi "$ID"
-        fi
+      CHUNKED_IMAGE="localhost/${image_name}:${DEFAULT_TAG}"
+    else
+      CHUNKED_IMAGE="localhost/${image_name}:${DEFAULT_TAG}-chunked"
     fi
 
-    # Rechunk Container
-    rechunker="{{ rechunker_image }}"
-
-    echo "::endgroup::"
-    echo "::group:: Prune"
-
-    # Run Rechunker's Prune
+    # 128 layers, conservative default
+    # In CI this renames stable to stable-daily
     ${SUDOIF} ${PODMAN} run --rm \
         --pull=${PULL_POLICY} \
-        --security-opt label=disable \
-        --volume "$MOUNT":/var/tree \
-        --env TREE=/var/tree \
-        --user 0:0 \
-        "${rechunker}" \
-        /sources/rechunk/1_prune.sh
-
-    echo "::endgroup::"
-    echo "::group:: Create ostree tree"
-
-    # Run Rechunker's Create
-    ${SUDOIF} ${PODMAN} run --rm \
-        --security-opt label=disable \
-        --volume "$MOUNT":/var/tree \
-        --volume "cache_ostree:/var/ostree" \
-        --env TREE=/var/tree \
-        --env REPO=/var/ostree/repo \
-        --env RESET_TIMESTAMP=1 \
-        --user 0:0 \
-        "${rechunker}" \
-        /sources/rechunk/2_create.sh
-
-    # Cleanup Temp Container Reference
-    ${SUDOIF} ${PODMAN} unmount "$CREF"
-    ${SUDOIF} ${PODMAN} rm "$CREF"
-    ${SUDOIF} ${PODMAN} rmi "$OLD_IMAGE"
-
-    echo "::endgroup::"
-    echo "::group:: Rechunker"
-
-    # Run Rechunker
-    ${SUDOIF} ${PODMAN} run --rm \
-        --pull=${PULL_POLICY} \
-        --security-opt label=disable \
-        --volume "$PWD:/workspace" \
-        --volume "$PWD:/var/git" \
-        --volume cache_ostree:/var/ostree \
-        --env REPO=/var/ostree/repo \
-        --env PREV_REF=ghcr.io/ublue-os/"${image_name}":"${tag}" \
-        --env OUT_NAME="$OUT_NAME" \
-        --env LABELS="${LABELS}" \
-        --env "DESCRIPTION='An interpretation of the Ubuntu spirit built on Fedora technology'" \
-        --env "VERSION=${VERSION}" \
-        --env VERSION_FN=/workspace/version.txt \
-        --env OUT_REF="oci:$OUT_NAME" \
-        --env GIT_DIR="/var/git" \
-        --env REVISION="$SHA" \
-        --user 0:0 \
-        "${rechunker}" \
-        /sources/rechunk/3_chunk.sh
-
-    # Fix Permissions of OCI
-    ${SUDOIF} find ${OUT_NAME} -type d -exec chmod 0755 {} \; || true
-    ${SUDOIF} find ${OUT_NAME}* -type f -exec chmod 0644 {} \; || true
-
-    if [[ "${UID}" -gt "0" ]]; then
-        ${SUDOIF} chown "${UID}:${GROUPS}" -R "${PWD}"
-    elif [[ -n "${SUDO_UID:-}" ]]; then
-        chown "${SUDO_UID}":"${SUDO_GID}" -R "${PWD}"
-    fi
-
-    # Remove cache_ostree
-    ${SUDOIF} ${PODMAN} volume rm cache_ostree
-
-    echo "::endgroup::"
+        --privileged \
+        -v "/var/lib/containers:/var/lib/containers" \
+        --entrypoint /usr/bin/rpm-ostree \
+        "quay.io/fedora-ostree-desktops/silverblue:${fedora_version}" \
+        compose build-chunked-oci \
+        --max-layers 128 \
+        --format-version=2 \
+        --bootc \
+        --from "localhost/${image_name}:${tag}" \
+        --output containers-storage:${CHUNKED_IMAGE}
 
     # Pipeline Checks
     if [[ {{ pipeline }} == "1" && -n "${SUDO_USER:-}" ]]; then
-        sudo -u "${SUDO_USER}" {{ just }} load-rechunk "${image}" "${tag}" "${flavor}"
         sudo -u "${SUDO_USER}" {{ just }} secureboot "${image}" "${tag}" "${flavor}"
     fi
 
-# Load OCI into Podman Store
+# For Rechunk
 [group('Image')]
-load-rechunk image="bluefin" tag="latest" flavor="main":
+load-rootful $image="bluefin" $tag="latest" $flavor="main":
     #!/usr/bin/bash
-    set -eou pipefail
+    set -eoux pipefail
 
     # Validate
     {{ just }} validate {{ image }} {{ tag }} {{ flavor }}
 
     # Image Name
     image_name=$({{ just }} image_name {{ image }} {{ tag }} {{ flavor }})
 
-    # Load Image
-    OUT_NAME="${image_name}_build"
-    IMAGE=$(${PODMAN} pull oci:"${PWD}"/"${OUT_NAME}")
-    ${PODMAN} tag ${IMAGE} localhost/"${image_name}":{{ tag }}
+    if [[ ! "$(id -u)" == 0 && ! ${PODMAN} =~ docker ]]; then
+      ID=$(${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'")
+      if [[ -z "$ID" ]]; then
+          {{ just }} build "$image" "$tag" "$flavor"
+      fi
+      ${PODMAN} image scp localhost/"${image_name}":"${tag}" root@localhost::
+    fi
+
+# Generate OCI Archive for PR Testing
+[group('Image')]
+export-oci $image="bluefin" $tag="latest" $flavor="main":
+    #!/usr/bin/bash
+    set -eoux pipefail
 
-    # Cleanup
-    rm -rf "${OUT_NAME}*"
-    rm -f previous.manifest.json
+    # Validate
+    {{ just }} validate {{ image }} {{ tag }} {{ flavor }}
+
+    # Image Name
+    image_name=$({{ just }} image_name {{ image }} {{ tag }} {{ flavor }})
+
+    ${PODMAN} push localhost/"${image_name}":"${tag}" oci-archive:"${image_name}".oci
 
 # Run Container
 [group('Image')]

build_files/base/17-cleanup.sh

@@ -19,6 +19,10 @@ systemctl enable rpm-ostree-countme.service
 systemctl enable tailscaled.service
 systemctl enable ublue-system-setup.service
 
+# see /usr/bin/rechunker-group-fix
+# DO NOT REMOVE THIS
+systemctl enable rechunker-group-fix.service
+
 # run flatpak preinstall once at startup
 if [[ "$(rpm -E %fedora)" -ge "42" ]]; then
   systemctl enable flatpak-preinstall.service

system_files/shared/usr/bin/rechunker-group-fix

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# To use this script, you'll want to put this in your systemd service:
+# rm /etc/gshadow
+# systemd-sysusers
+# (run this script)
+# systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev
+# This will populate /etc/group successfully, and then populate /etc/gshadow
+# with any missing groups that we nuked when we removed /etc/gshadow
+
+GSHADOW_FILE="/etc/gshadow"
+GROUP_FILE="/etc/group"
+
+for f in $(cat $GROUP_FILE); do
+   cut -f1 -d':' <(echo $f) | xargs -I{} grep ^"{}:" $GSHADOW_FILE &>/dev/null || \
+     echo $(cut -f1 -d':' <(echo $f)):'!*::' >> $GSHADOW_FILE
+done