diff --git a/.github/workflows/build-boxkit.yml b/.github/workflows/build-boxkit.yml index b104b247..da671e57 100644 --- a/.github/workflows/build-boxkit.yml +++ b/.github/workflows/build-boxkit.yml @@ -45,17 +45,32 @@ jobs: labels: | io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/boxkit/main/README.md + # Compute SOURCE_DATE_EPOCH for reproducible builds + - name: Compute SOURCE_DATE_EPOCH + id: epoch + run: | + commit_ts=$(git show -s --format=%ct "$GITHUB_SHA" 2>/dev/null || echo '') + if [ -z "$commit_ts" ]; then + echo "Warning: could not determine commit timestamp, falling back to current time" + commit_ts=$(date +%s) + fi + echo "epoch=$commit_ts" >> "$GITHUB_OUTPUT" + # Build image using Buildah action - name: Build Image id: build_image uses: redhat-actions/buildah-build@7a95fa7ee0f02d552a32753e7414641a04307056 # v2 with: containerfiles: | - ./ContainerFiles/${{ matrix.containerfile }} + ./Containerfiles/${{ matrix.containerfile }} image: ${{ matrix.containerfile }} tags: ${{ env.IMAGE_TAGS }} labels: ${{ steps.meta.outputs.labels }} - oci: true + oci: false + extra-args: | + --skip-unused-stages=false + build-args: | + SOURCE_DATE_EPOCH=${{ steps.epoch.outputs.epoch }} # Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR. # https://github.com/macbre/push-to-ghcr/issues/12 @@ -79,6 +94,8 @@ jobs: registry: ${{ steps.registry_case.outputs.lowercase }} username: ${{ env.REGISTRY_USER }} password: ${{ env.REGISTRY_PASSWORD }} + extra-args: | + --compression-format=zstd:chunked - name: Login to GitHub Container Registry uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3 diff --git a/ContainerFiles/boxkit b/ContainerFiles/boxkit deleted file mode 100644 index ead1ed5e..00000000 --- a/ContainerFiles/boxkit +++ /dev/null @@ -1,15 +0,0 @@ -FROM quay.io/toolbx-images/alpine-toolbox:edge - -LABEL com.github.containers.toolbox="true" \ - usage="This image is meant to be used with the toolbox or distrobox command" \ - summary="A cloud-native terminal experience" \ - maintainer="jorge.castro@gmail.com" - -# Copy the setup scripts and package list -COPY ../scripts/boxkit.sh / -COPY ../scripts/distrobox-shims.sh / -COPY ../packages/boxkit.packages / - -# Run the setup scripts -RUN chmod +x boxkit.sh distrobox-shims.sh && /boxkit.sh -RUN rm /boxkit.sh /distrobox-shims.sh /boxkit.packages diff --git a/ContainerFiles/fedora-example b/ContainerFiles/fedora-example deleted file mode 100644 index 08c7c4af..00000000 --- a/ContainerFiles/fedora-example +++ /dev/null @@ -1,15 +0,0 @@ -FROM quay.io/fedora/fedora-toolbox:latest - -LABEL com.github.containers.toolbox="true" \ - usage="This image is meant to be used with the toolbox or distrobox command" \ - summary="An example ContainerFile to demonstrate multiple image builds." \ - maintainer="faeizmahrus@outlook.com" - -# Copy the setup scripts and package list -COPY ../scripts/fedora-example.sh / -COPY ../scripts/distrobox-shims.sh / -COPY ../packages/fedora-example.packages / - -# Run the setup scripts -RUN chmod +x fedora-example.sh distrobox-shims.sh && /fedora-example.sh -RUN rm /fedora-example.sh /distrobox-shims.sh /fedora-example.packages diff --git a/Containerfiles/boxkit b/Containerfiles/boxkit new file mode 100644 index 00000000..0dcf1251 --- /dev/null +++ b/Containerfiles/boxkit @@ -0,0 +1,35 @@ +FROM quay.io/toolbx-images/alpine-toolbox:edge AS builder + +# Copy the setup scripts and package list +COPY scripts/boxkit.sh / +COPY scripts/distrobox-shims.sh / +COPY scripts/chunkah-tag.sh / +COPY packages/boxkit.packages / + +# Run the setup scripts and tag files for chunkah +RUN chmod +x boxkit.sh distrobox-shims.sh chunkah-tag.sh && \ + /boxkit.sh && \ + /chunkah-tag.sh && \ + rm /boxkit.sh /distrobox-shims.sh /chunkah-tag.sh /boxkit.packages + +# For CI builds with buildah, use chunkah rechunking +# For local podman builds, use: podman build --target=builder -f Containerfiles/boxkit -t boxkit:local . + +# https://github.com/coreos/chunkah +FROM quay.io/jlebon/chunkah AS chunkah + +# Rechunk with content-based layers +ARG SOURCE_DATE_EPOCH +ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-0} +RUN --mount=from=builder,src=/,target=/chunkah,ro \ + --mount=type=bind,target=/run/src,rw \ + chunkah build \ + --max-layers 64 \ + --label=com.github.containers.toolbox=true \ + --label=usage="This image is meant to be used with the toolbox or distrobox command" \ + --label=summary="A cloud-native terminal experience" \ + --label=maintainer=jorge.castro@gmail.com \ + > /run/src/out.ociarchive + +# Deploy final chunked image +FROM oci-archive:out.ociarchive diff --git a/Containerfiles/fedora-example b/Containerfiles/fedora-example new file mode 100644 index 00000000..4bfaf699 --- /dev/null +++ b/Containerfiles/fedora-example @@ -0,0 +1,36 @@ +FROM quay.io/fedora/fedora-toolbox:latest AS builder + +# Copy the setup scripts and package list +COPY scripts/fedora-example.sh / +COPY scripts/distrobox-shims.sh / +COPY scripts/chunkah-tag.sh / +COPY packages/fedora-example.packages / + +# Run the setup scripts and tag files for chunkah +RUN chmod +x fedora-example.sh distrobox-shims.sh chunkah-tag.sh && \ + /fedora-example.sh && \ + /chunkah-tag.sh && \ + rm /fedora-example.sh /distrobox-shims.sh /chunkah-tag.sh /fedora-example.packages + +# For CI builds with buildah, use chunkah rechunking +# For local podman builds, use: podman build --target=builder -f Containerfiles/fedora-example -t fedora-example:local . + +# https://github.com/coreos/chunkah +FROM quay.io/jlebon/chunkah AS chunkah + +# Rechunk with content-based layers +ARG SOURCE_DATE_EPOCH +ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-0} +RUN --mount=from=builder,src=/,target=/chunkah,ro \ + --mount=type=bind,target=/run/src,rw \ + chunkah build \ + --max-layers 64 \ + --label=com.github.containers.toolbox=true \ + --label=name=fedora-toolbox \ + --label=usage="This image is meant to be used with the toolbox or distrobox command" \ + --label=summary="An example ContainerFile to demonstrate multiple image builds." \ + --label=maintainer=faeizmahrus@outlook.com \ + > /run/src/out.ociarchive + +# Deploy final chunked image +FROM oci-archive:out.ociarchive diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..73c4ec3a --- /dev/null +++ b/Justfile @@ -0,0 +1,38 @@ +#!/usr/bin/env -S just --justfile + +# boxkit - Justfile for local development and testing +# Container provisioning logic lives in scripts/ + +# Build boxkit image locally (without chunkah - chunkah requires buildah) +build-boxkit: + podman build --target=builder -f Containerfiles/boxkit -t boxkit:local . + +# Build fedora-example image locally (without chunkah - chunkah requires buildah) +build-fedora-example: + podman build --target=builder -f Containerfiles/fedora-example -t fedora-example:local . + +# Build all images +build-all: build-boxkit build-fedora-example + +# Run boxkit in distrobox +run-boxkit: + @distrobox create -i boxkit:local -n boxkit 2>/dev/null || true + distrobox enter boxkit + +# Run fedora-example in distrobox +run-fedora-example: + @distrobox create -i fedora-example:local -n fedora-example 2>/dev/null || true + distrobox enter fedora-example + +# Clean built images +clean: + podman rmi -f boxkit:local fedora-example:local 2>/dev/null || true + @echo "✓ Local images cleaned" + +# Clean distrobox containers +clean-distrobox: + @distrobox rm -f boxkit fedora-example 2>/dev/null || true + @echo "✓ Distrobox containers removed" + +# Full cleanup +clean-all: clean clean-distrobox diff --git a/README.md b/README.md index 75958e73..75476c61 100644 --- a/README.md +++ b/README.md @@ -24,18 +24,19 @@ Tag your image with `boxkit` to share with others! ### How everything is organized -- The ContainerFiles for the custom images are stored in the `ContainerFiles/` folder. +- The Containerfiles for the custom images are stored in the `Containerfiles/` folder. - The setup scripts for the custom images (if needed) are stored in the `scripts/` folder. - The package lists for the setup scripts (if needed) are stored in the `packages/` folder. +- The Justfile provides local build commands (`just build-boxkit`, `just build-all`). - The Github workflow that generates the images is `.github/workflows/build-boxkit.yml` ### How to make your own images 1. Fork this repo. -2. Add the ContainerFiles for your custom images to the `ContainerFiles/` folder. +2. Add the Containerfiles for your custom images to the `Containerfiles/` folder. 3. Add the setup scripts you want to use for your custom images (if needed) to the `scripts/` folder. 4. Add the package list you want to use for your custom images (if needed) to the `packages/` folder. -5. Add the name of the ContainerFiles of your custom images to the following section in `build-boxkit.yml`: +5. Add the name of the Containerfiles of your custom images to the following section in `build-boxkit.yml`: ```yaml jobs: @@ -50,11 +51,34 @@ jobs: - You can choose to only generate a single custom image if you want. - You can remove the boxkit and fedora-example images provided in the boxkit repo and only generate your own custom images. - The `scripts/` and `packages/` folders are optional, you can generate your custom images without them, but they are highly recommended to use. -- The name of your custom image and ContainerFile **MUST** be the same.
+- The name of your custom image and Containerfile **MUST** be the same.
- e.g. If you want to create a custom image named *appbox-debian*, the corresponding ContainerFile must be named `appbox-debian` and must be stored inside the `ContainerFiles/` folder. + e.g. If you want to create a custom image named *appbox-debian*, the corresponding Containerfile must be named `appbox-debian` and must be stored inside the `Containerfiles/` folder. - The URL for the generated images will be `ghcr.io//` by default. +## zstd:chunked and Content-Based Layers + +boxkit images use [chunkah](https://github.com/coreos/chunkah) for content-based layer splitting and `zstd:chunked` compression. This significantly reduces bandwidth when pulling updated images by only fetching changed content. + +**Benefits:** +- **Faster image pulls** - Only changed layers are downloaded +- **Reduced bandwidth usage** - Efficient delta transfers +- **Better layer reuse** - Package-based layers maximize caching across updates + +**How it works:** +- The `scripts/chunkah-tag.sh` script tags all installed files with their package names using extended file attributes +- The `chunkah` tool uses these tags to create intelligent package-based layers (up to 64 layers max) +- Images are compressed with `zstd:chunked` for efficient partial pulls +- Works with any OCI-compliant registry and requires no special client support + +**Supported distributions:** +- Alpine Linux (apk) +- Fedora/RHEL/CentOS (rpm/dnf/yum) +- Debian/Ubuntu (dpkg/apt) +- Arch Linux (pacman) + +The script auto-detects your package manager and tags files accordingly. If using an unsupported distribution, chunkah will still work but layer splitting will be less optimal. + ### Signing your images Although optional, it is **Highly recommended** you use container signing for your images. To sign your images, follow the steps below: diff --git a/scripts/chunkah-tag.sh b/scripts/chunkah-tag.sh new file mode 100644 index 00000000..3e55ada5 --- /dev/null +++ b/scripts/chunkah-tag.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# Tag files with package component names for chunkah content-based layering +# Supports: apk (Alpine), rpm (Fedora/RHEL), dpkg (Debian/Ubuntu), pacman (Arch) + +set -euo pipefail + +tag_files() { + pkgname="$1" + shift + for filepath in "$@"; do + if [ -f "$filepath" ]; then + setfattr -n user.component -v "$pkgname" "$filepath" 2>/dev/null || true + fi + done +} + +if command -v apk >/dev/null 2>&1; then + echo "==> Detected Alpine (apk) - tagging files for chunkah" + apk add --no-cache attr + apk info -q | while read pkgname; do + tag_files "$pkgname" $(apk info -L "$pkgname" 2>/dev/null | sed 's|^|/|') + done + apk del attr + echo "✓ Alpine file tagging complete" + +elif command -v rpm >/dev/null 2>&1; then + echo "==> Detected RPM-based (dnf/yum) - tagging files for chunkah" + if command -v dnf5 >/dev/null 2>&1; then + dnf5 install -y attr + elif command -v dnf >/dev/null 2>&1; then + dnf install -y attr + else + yum install -y attr + fi + + rpm -qa --qf '%{NAME}\n' | while read pkgname; do + tag_files "$pkgname" $(rpm -ql "$pkgname" 2>/dev/null) + done + + if command -v dnf5 >/dev/null 2>&1; then + dnf5 remove -y attr + elif command -v dnf >/dev/null 2>&1; then + dnf remove -y attr + else + yum remove -y attr + fi + echo "✓ RPM file tagging complete" + +elif command -v dpkg >/dev/null 2>&1; then + echo "==> Detected Debian-based (dpkg/apt) - tagging files for chunkah" + apt-get update && apt-get install -y attr + dpkg-query -W -f='${Package}\n' | while read pkgname; do + tag_files "$pkgname" $(dpkg -L "$pkgname" 2>/dev/null) + done + apt-get remove -y attr + echo "✓ Debian file tagging complete" + +elif command -v pacman >/dev/null 2>&1; then + echo "==> Detected Arch (pacman) - tagging files for chunkah" + pacman -Sy --noconfirm attr + pacman -Qq | while read pkgname; do + tag_files "$pkgname" $(pacman -Ql "$pkgname" 2>/dev/null | awk '{print $2}') + done + pacman -R --noconfirm attr + echo "✓ Arch file tagging complete" + +else + echo "WARNING: Unknown package manager detected" + echo "Chunkah will still work but layer splitting may be less optimal" + exit 0 +fi