feat: integrate zstd:chunked and chunkah for content-based layers

.github/workflows/build-boxkit.yml

@@ -45,17 +45,26 @@ jobs:
           labels: |
             io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/boxkit/main/README.md
 
+      # Compute SOURCE_DATE_EPOCH for reproducible builds
+      - name: Compute SOURCE_DATE_EPOCH
+        id: epoch
+        run: echo "epoch=$(date -d '${{ github.event.head_commit.timestamp }}' +%s)" >> $GITHUB_OUTPUT
+
       # Build image using Buildah action
       - name: Build Image
         id: build_image
         uses: redhat-actions/buildah-build@7a95fa7ee0f02d552a32753e7414641a04307056 # v2
         with:
           containerfiles: |
-            ./ContainerFiles/${{ matrix.containerfile }}
+            ./Containerfiles/${{ matrix.containerfile }}
           image: ${{ matrix.containerfile }}
           tags: ${{ env.IMAGE_TAGS }}
           labels: ${{ steps.meta.outputs.labels }}
-          oci: true
+          oci: false
+          extra-args: |
+            --skip-unused-stages=false
+          build-args: |
+            SOURCE_DATE_EPOCH=${{ steps.epoch.outputs.epoch }}
 
       # Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
       # https://github.com/macbre/push-to-ghcr/issues/12
@@ -79,6 +88,8 @@ jobs:
           registry: ${{ steps.registry_case.outputs.lowercase }}
           username: ${{ env.REGISTRY_USER }}
           password: ${{ env.REGISTRY_PASSWORD }}
+          extra-args: |
+            --compression-format=zstd:chunked
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3

Containerfiles/boxkit

@@ -0,0 +1,35 @@
+FROM quay.io/toolbx-images/alpine-toolbox:edge AS builder
+
+# Copy the setup scripts and package list
+COPY scripts/boxkit.sh /
+COPY scripts/distrobox-shims.sh /
+COPY scripts/chunkah-tag.sh /
+COPY packages/boxkit.packages /
+
+# Run the setup scripts and tag files for chunkah
+RUN chmod +x boxkit.sh distrobox-shims.sh chunkah-tag.sh && \
+    /boxkit.sh && \
+    /chunkah-tag.sh && \
+    rm /boxkit.sh /distrobox-shims.sh /chunkah-tag.sh /boxkit.packages
+
+# For CI builds with buildah, use chunkah rechunking
+# For local podman builds, use: podman build --target=builder -f Containerfiles/boxkit -t boxkit:local .
+
+# https://github.com/coreos/chunkah
+FROM quay.io/jlebon/chunkah AS chunkah
+
+# Rechunk with content-based layers
+ARG SOURCE_DATE_EPOCH
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-0}
+RUN --mount=from=builder,src=/,target=/chunkah,ro \
+    --mount=type=bind,target=/run/src,rw \
+    chunkah build \
+    --max-layers 64 \
+    --label=com.github.containers.toolbox=true \
+    --label=usage="This image is meant to be used with the toolbox or distrobox command" \
+    --label=summary="A cloud-native terminal experience" \
+    --label=maintainer=jorge.castro@gmail.com \
+    > /run/src/out.ociarchive
+
+# Deploy final chunked image
+FROM oci-archive:out.ociarchive

Containerfiles/fedora-example

@@ -0,0 +1,36 @@
+FROM quay.io/fedora/fedora-toolbox:latest AS builder
+
+# Copy the setup scripts and package list
+COPY scripts/fedora-example.sh /
+COPY scripts/distrobox-shims.sh /
+COPY scripts/chunkah-tag.sh /
+COPY packages/fedora-example.packages /
+
+# Run the setup scripts and tag files for chunkah
+RUN chmod +x fedora-example.sh distrobox-shims.sh chunkah-tag.sh && \
+    /fedora-example.sh && \
+    /chunkah-tag.sh && \
+    rm /fedora-example.sh /distrobox-shims.sh /chunkah-tag.sh /fedora-example.packages
+
+# For CI builds with buildah, use chunkah rechunking
+# For local podman builds, use: podman build --target=builder -f Containerfiles/fedora-example -t fedora-example:local .
+
+# https://github.com/coreos/chunkah
+FROM quay.io/jlebon/chunkah AS chunkah
+
+# Rechunk with content-based layers
+ARG SOURCE_DATE_EPOCH
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-0}
+RUN --mount=from=builder,src=/,target=/chunkah,ro \
+    --mount=type=bind,target=/run/src,rw \
+    chunkah build \
+    --max-layers 64 \
+    --label=com.github.containers.toolbox=true \
+    --label=name=fedora-toolbox \
+    --label=usage="This image is meant to be used with the toolbox or distrobox command" \
+    --label=summary="An example ContainerFile to demonstrate multiple image builds." \
+    --label=maintainer=faeizmahrus@outlook.com \
+    > /run/src/out.ociarchive
+
+# Deploy final chunked image
+FROM oci-archive:out.ociarchive

Justfile

@@ -0,0 +1,38 @@
+#!/usr/bin/env -S just --justfile
+
+# boxkit - Justfile for local development and testing
+# Container provisioning logic lives in scripts/
+
+# Build boxkit image locally (without chunkah - chunkah requires buildah)
+build-boxkit:
+    podman build --target=builder -f Containerfiles/boxkit -t boxkit:local .
+
+# Build fedora-example image locally (without chunkah - chunkah requires buildah)
+build-fedora-example:
+    podman build --target=builder -f Containerfiles/fedora-example -t fedora-example:local .
+
+# Build all images
+build-all: build-boxkit build-fedora-example
+
+# Run boxkit in distrobox
+run-boxkit:
+    @distrobox create -i boxkit:local -n boxkit 2>/dev/null || true
+    distrobox enter boxkit
+
+# Run fedora-example in distrobox
+run-fedora-example:
+    @distrobox create -i fedora-example:local -n fedora-example 2>/dev/null || true
+    distrobox enter fedora-example
+
+# Clean built images
+clean:
+    podman rmi -f boxkit:local fedora-example:local 2>/dev/null || true
+    @echo "✓ Local images cleaned"
+
+# Clean distrobox containers
+clean-distrobox:
+    @distrobox rm -f boxkit fedora-example 2>/dev/null || true
+    @echo "✓ Distrobox containers removed"
+
+# Full cleanup
+clean-all: clean clean-distrobox

README.md

@@ -24,18 +24,19 @@ Tag your image with `boxkit` to share with others!
 
 ### How everything is organized
 
-- The ContainerFiles for the custom images are stored in the `ContainerFiles/` folder.
+- The Containerfiles for the custom images are stored in the `Containerfiles/` folder.
 - The setup scripts for the custom images (if needed) are stored in the `scripts/` folder.
 - The package lists for the setup scripts (if needed) are stored in the `packages/` folder.
+- The Justfile provides local build commands (`just build-boxkit`, `just build-all`).
 - The Github workflow that generates the images is `.github/workflows/build-boxkit.yml`
 
 ### How to make your own images
 
 1. Fork this repo.
-2. Add the ContainerFiles for your custom images to the `ContainerFiles/` folder.
+2. Add the Containerfiles for your custom images to the `Containerfiles/` folder.
 3. Add the setup scripts you want to use for your custom images (if needed) to the `scripts/` folder.
 4. Add the package list you want to use for your custom images (if needed) to the `packages/` folder.
-5. Add the name of the ContainerFiles of your custom images to the following section in `build-boxkit.yml`:
+5. Add the name of the Containerfiles of your custom images to the following section in `build-boxkit.yml`:
 
 ```yaml
 jobs:
@@ -50,11 +51,34 @@ jobs:
 - You can choose to only generate a single custom image if you want. 
 - You can remove the boxkit and fedora-example images provided in the boxkit repo and only generate your own custom images.
 - The `scripts/` and `packages/` folders are optional, you can generate your custom images without them, but they are highly recommended to use.
-- The name of your custom image and ContainerFile **MUST** be the same. <br>
+- The name of your custom image and Containerfile **MUST** be the same. <br>
 
-  e.g. If you want to create a custom image named *appbox-debian*, the corresponding ContainerFile must be named `appbox-debian` and must be stored inside the `ContainerFiles/` folder.
+  e.g. If you want to create a custom image named *appbox-debian*, the corresponding Containerfile must be named `appbox-debian` and must be stored inside the `Containerfiles/` folder.
 - The URL for the generated images will be `ghcr.io/<username>/<image_name>` by default.
 
+## zstd:chunked and Content-Based Layers
+
+boxkit images use [chunkah](https://github.com/coreos/chunkah) for content-based layer splitting and `zstd:chunked` compression. This significantly reduces bandwidth when pulling updated images by only fetching changed content.
+
+**Benefits:**
+- **Faster image pulls** - Only changed layers are downloaded
+- **Reduced bandwidth usage** - Efficient delta transfers
+- **Better layer reuse** - Package-based layers maximize caching across updates
+
+**How it works:**
+- The `scripts/chunkah-tag.sh` script tags all installed files with their package names using extended file attributes
+- The `chunkah` tool uses these tags to create intelligent package-based layers (up to 64 layers max)
+- Images are compressed with `zstd:chunked` for efficient partial pulls
+- Works with any OCI-compliant registry and requires no special client support
+
+**Supported distributions:**
+- Alpine Linux (apk)
+- Fedora/RHEL/CentOS (rpm/dnf/yum)
+- Debian/Ubuntu (dpkg/apt)
+- Arch Linux (pacman)
+
+The script auto-detects your package manager and tags files accordingly. If using an unsupported distribution, chunkah will still work but layer splitting will be less optimal.
+
 ### Signing your images
 Although optional, it is **Highly recommended** you use container signing for your images.
 To sign your images, follow the steps below:

scripts/chunkah-tag.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+# Tag files with package component names for chunkah content-based layering
+# Supports: apk (Alpine), rpm (Fedora/RHEL), dpkg (Debian/Ubuntu), pacman (Arch)
+
+set -euo pipefail
+
+tag_files() {
+    pkgname="$1"
+    shift
+    for filepath in "$@"; do
+        if [ -f "$filepath" ]; then
+            setfattr -n user.component -v "$pkgname" "$filepath" 2>/dev/null || true
+        fi
+    done
+}
+
+if command -v apk >/dev/null 2>&1; then
+    echo "==> Detected Alpine (apk) - tagging files for chunkah"
+    apk add --no-cache attr
+    apk info -q | while read pkgname; do
+        tag_files "$pkgname" $(apk info -L "$pkgname" 2>/dev/null | sed 's|^|/|')
+    done
+    apk del attr
+    echo "✓ Alpine file tagging complete"
+
+elif command -v rpm >/dev/null 2>&1; then
+    echo "==> Detected RPM-based (dnf/yum) - tagging files for chunkah"
+    if command -v dnf5 >/dev/null 2>&1; then
+        dnf5 install -y attr
+    elif command -v dnf >/dev/null 2>&1; then
+        dnf install -y attr
+    else
+        yum install -y attr
+    fi
+
+    rpm -qa --qf '%{NAME}\n' | while read pkgname; do
+        tag_files "$pkgname" $(rpm -ql "$pkgname" 2>/dev/null)
+    done
+
+    if command -v dnf5 >/dev/null 2>&1; then
+        dnf5 remove -y attr
+    elif command -v dnf >/dev/null 2>&1; then
+        dnf remove -y attr
+    else
+        yum remove -y attr
+    fi
+    echo "✓ RPM file tagging complete"
+
+elif command -v dpkg >/dev/null 2>&1; then
+    echo "==> Detected Debian-based (dpkg/apt) - tagging files for chunkah"
+    apt-get update && apt-get install -y attr
+    dpkg-query -W -f='${Package}\n' | while read pkgname; do
+        tag_files "$pkgname" $(dpkg -L "$pkgname" 2>/dev/null)
+    done
+    apt-get remove -y attr
+    echo "✓ Debian file tagging complete"
+
+elif command -v pacman >/dev/null 2>&1; then
+    echo "==> Detected Arch (pacman) - tagging files for chunkah"
+    pacman -Sy --noconfirm attr
+    pacman -Qq | while read pkgname; do
+        tag_files "$pkgname" $(pacman -Ql "$pkgname" 2>/dev/null | awk '{print $2}')
+    done
+    pacman -R --noconfirm attr
+    echo "✓ Arch file tagging complete"
+
+else
+    echo "WARNING: Unknown package manager detected"
+    echo "Chunkah will still work but layer splitting may be less optimal"
+    exit 0
+fi