From ed424986d01c3e11b7a9ac501696302c539ae3ab Mon Sep 17 00:00:00 2001 From: dx4homelab <145386996+dx4homelab@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:38:43 -0600 Subject: [PATCH 1/3] feat(disk): LVM layout for the 4 TB boot drive, and fix bib raw output dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes, the second discovered while validating the first. LVM layout (iso/disk.toml) -------------------------- Replaces the single [[customizations.filesystem]] entry — the simple and advanced (customizations.disk) forms are alternatives, not additive. z170asrock is swapping its boot NVMe from 931 GB to 4 TB. Measured live 2026-07-29, that host has NO LVM today: one 20 GiB xfs partition carries /sysroot + /var + /etc, and ~911 GB of the drive is unallocated because the dd install was never grown. (docs/hardware-inventory.md described a vg4base + vg4home layout that has not existed since the uCore migration.) vg4base now holds root (16 GiB) and var (10 GiB). Splitting /var out of root is the one structural change and it earns its keep on an unattended server: if container images, logs or Postgres fill a shared filesystem you cannot stage an ostree upgrade or cleanly log in to fix it. FCOS supports a separate /var as a first-class configuration. Nothing is needed in the image for this — lvm2, device-mapper, the dracut 70lvm module and coreos-populate-lvmdevices.service are already present, and lsinitrd confirms LVM is already in the initramfs. bootc-image-builder minsize semantics ------------------------------------- Documented in the file because it is not in the upstream docs and cost two builds to pin down. `minsize` on the lvm partition is REQUIRED (omitting it fails with "minsize is required"), and it is NOT the size of the volume group: it is applied to the ROOT lv, root's own minsize is ignored, other LVs get their own minsize, and the VG becomes the sum. VG 26 GiB + root 16 GiB + var 10 GiB -> root=26G var=10G VG=36G raw 37.2 GiB VG 16 GiB + root 16 GiB + var 10 GiB -> root=16G var=10G VG=26G raw 27.2 GiB Not just image size: XFS cannot shrink, so an oversized root LV is space that can never be handed back to var. Justfile: bib raw output lands in output/image, not output/raw -------------------------------------------------------------- The post-build step cleared output/${type} before `mv -f`. bib's subdir name does not track --type: iso -> bootiso (already special-cased), raw -> image, qcow2 -> qcow2. So for raw builds output/image was never cleared, mv failed with "cannot overwrite 'output/image': Directory not empty", and the recipe exited 1 while leaving the PREVIOUS build's disk.raw in place. A green-looking build that silently ships a stale artifact. Replaced with an explicit type -> dir mapping. Verified by building three times and loop-mounting each result: partition table 1M bios_grub / 200M EFI / 1G xfs /boot / 26G LVM2_member /boot and EFI outside LVM, as required by the bootloader vg4base root 16.00g xfs label=root (holds boot, ostree, var) var 10.00g xfs label=var APST karg present in /loader.1/entries/ostree-1.conf, so a dd'd system is protected from its first boot raw 29208084480 bytes, staged and sha256-verified to /mnt/pny2xxG/server4homelab/july-29-2026/ Co-Authored-By: Claude Opus 5 --- Justfile | 18 +++++++++---- iso/disk.toml | 74 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/Justfile b/Justfile index 3c3475d..752a4c9 100644 --- a/Justfile +++ b/Justfile @@ -299,11 +299,19 @@ _build-bib $target_image $tag $type $config: (_rootful_load_image target_image t # BIB writes its output into per-type subdirs (e.g. output/qcow2/disk.qcow2). # `mv -f` does not replace non-empty directories, so clear the type-specific # output dir first if a prior build of the same type left one behind. - if [[ "${type}" == "iso" ]]; then - sudo rm -rf output/bootiso - else - sudo rm -rf "output/${type}" - fi + # BIB's output subdir name does not always match the --type it was given: + # --type iso -> output/bootiso + # --type raw -> output/image <-- not output/raw + # --type qcow2 -> output/qcow2 + # Clearing the wrong one leaves the real dir non-empty and `mv -f` fails with + # "cannot overwrite 'output/image': Directory not empty", silently leaving the + # previous build's artifact in place. + case "${type}" in + iso) bib_outdir=bootiso ;; + raw) bib_outdir=image ;; + *) bib_outdir="${type}" ;; + esac + sudo rm -rf "output/${bib_outdir}" sudo mv -f $BUILDTMP/* output/ sudo rmdir $BUILDTMP sudo chown -R $USER:$USER output/ diff --git a/iso/disk.toml b/iso/disk.toml index 1e818f3..ba01450 100644 --- a/iso/disk.toml +++ b/iso/disk.toml @@ -14,9 +14,79 @@ [customizations.kernel] append = "nvme_core.default_ps_max_latency_us=0" -[[customizations.filesystem]] +# --------------------------------------------------------------------------- +# LVM disk layout +# +# Replaces the previous simple [[customizations.filesystem]] entry — the simple +# and advanced (customizations.disk) forms are alternatives, not additive. +# +# Why LVM: the boot NVMe is moving from 931 GB to 4 TB and will carry root, +# /var (containers, appdata, Postgres) and the ZFS pool keyfile. Sizes for +# appdata are not knowable up front, so being able to reallocate later without +# repartitioning is worth the small extra layer. Nothing needs adding to the +# image for this: lvm2, device-mapper, the dracut 70lvm module and +# coreos-populate-lvmdevices.service are already present, and LVM is already +# in the initramfs (verified with lsinitrd). +# +# /boot and /boot/efi are NOT listed here on purpose — bootc-image-builder +# creates them automatically as plain partitions when root is on an LV, which +# is required since the bootloader cannot read LVM. +# +# bootc-image-builder supports exactly ONE volume group, so everything lives in +# vg4base. This is a new layout, not a reproduction of an old one: as of +# 2026-07-29 z170asrock has NO LVM — a single 20 GiB xfs partition carries +# /sysroot, /var and /etc, with ~911 GB of the 931 GB drive unallocated because +# the previous dd install was never grown. +# +# Splitting /var out of root is the one structural change, and it is worth it on +# an unattended server: if container images, logs or Postgres fill a shared +# filesystem, you cannot stage an ostree upgrade or cleanly log in to fix it. +# A separate /var confines that blast radius to /var. FCOS supports a separate +# /var as a first-class configuration. +# +# Sizes are MINIMUMS, kept deliberately small so disk.raw stays quick to build +# and to stream over HTTP during the swap (the current image is ~22 GB; this is +# ~26 GB). The volume group is grown into the 4 TB free space after install — +# and note LVM lets you leave VG space UNALLOCATED and hand it out later, which +# is the whole reason for doing this. +# --------------------------------------------------------------------------- + +# IMPORTANT, learned by building this twice: +# +# `minsize` here is REQUIRED (omitting it fails the build with "minsize is +# required"), and bib does NOT treat it as "size of the whole VG". Measured +# behaviour: the ROOT lv is given this value — its own minsize below is ignored +# — every other LV gets its own minsize, and the VG ends up as the sum. +# +# VG minsize 26 GiB + root 16 GiB + var 10 GiB -> root=26G var=10G VG=36G +# VG minsize 16 GiB + root 16 GiB + var 10 GiB -> root=16G var=10G VG=26G +# +# So set this to the size you want ROOT to be. Getting it wrong is not merely a +# bigger disk.raw: XFS cannot shrink, so an oversized root LV is space that can +# never be handed back to var. +[[customizations.disk.partitions]] +type = "lvm" +name = "vg4base" +minsize = "16 GiB" + +# Root: ostree deployments only, once /var is split out. The whole system +# currently uses 6.1 GiB including /var, so 16 GiB holds several deployments +# plus rollback headroom comfortably. +[[customizations.disk.partitions.logical_volumes]] +name = "root" mountpoint = "/" -minsize = "20 GiB" +label = "root" +fs_type = "xfs" +minsize = "16 GiB" + +# /var: containers, quadlet state, appdata, Postgres, and the ZFS pool keyfile. +# Small at install time on purpose — this is the LV to extend into the 4 TB. +[[customizations.disk.partitions.logical_volumes]] +name = "var" +mountpoint = "/var" +label = "var" +fs_type = "xfs" +minsize = "10 GiB" [[customizations.user]] name = "developer" From d8d5dfb5201f3087c4df70b521445078b6305ddc Mon Sep 17 00:00:00 2001 From: dx4homelab <145386996+dx4homelab@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:58:56 -0600 Subject: [PATCH 2/3] =?UTF-8?q?fix(disk):=20keep=20LVM=20out=20of=20the=20?= =?UTF-8?q?boot=20path=20=E2=80=94=20all-plain=20layout,=20VM-verified?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverses course on LVM-on-root/-var. Three layouts were built, loop-mounted and confirmed byte-correct, and all three failed to boot. FCOS/uCore cannot carry a boot-critical filesystem on LVM: 1. root on an LV — the initramfs has NO LVM support at all: dracut modules present : crypt, dm (no lvm) lvm files in initramfs : none FCOS finds root by LABEL, nothing can activate the VG, so: dev-disk-by-label-root.device: Job ... failed with result 'timeout' Started emergency.service - Emergency Shell rd.lvm.lv= does not help; there is no LVM tooling to act on it. 2. root plain, /boot omitted — bib only auto-creates /boot when root is on an LV, and customizations.disk gives exactly what you declare: error: grub_search_label:527: no such device: boot. error: grub_loader_boot:196: you need to load the kernel first. 3. root plain + /boot + /var on an LV — boots into the real root, then: Timed out waiting for device dev-...- Dependency failed for var.mount - /var The image ships lvm2-monitor, lvm2-lvmpolld, lvm-devices-import and coreos-populate-lvmdevices but NO activation unit (no lvm2-activation*, no lvm2-pvscan@). And coreos-populate-lvmdevices declares RequiresMountsFor=/var/lib — it needs /var mounted, which is the very thing blocked. Circular, so it can never help here. This layout is all-plain and mirrors what already runs on z170asrock: 1M bios_grub | 200M ESP | 1G xfs /boot (label boot) | 20G xfs / (label root) Root is last, so it grows with plain `growpart && xfs_growfs /`. /boot is declared explicitly because the baked grub.cfg does `search --label boot` and bib will not create it for a plain root. VM-verified (qemu, OVMF, emulated NVMe, COW overlay grown to 4T so the GPT alt-header condition matches a real dd): GRUB loaded the kernel from /boot initramfs found /dev/disk/by-label/root; XFS (nvme0n1p4) mounted ostree-prepare-root: composefs mounted successfully reached "localhost login:" — no emergency mode nvme_core.default_ps_max_latency_us=0 present in the running cmdline LVM is not abandoned, just moved out of the boot path: per-app LVs created AFTER install and mounted under /var/lib/ with nofail and a device timeout. That is all the snapshot-to-ZFS workflow needs. Documented in the file. A loop-mount proves the bytes are right, not that it boots — all three failures passed loop-mount inspection. Always VM-boot a change to this file. Co-Authored-By: Claude Opus 5 --- iso/disk.toml | 137 ++++++++++++++++++++++++++++---------------------- 1 file changed, 78 insertions(+), 59 deletions(-) diff --git a/iso/disk.toml b/iso/disk.toml index ba01450..589c1c5 100644 --- a/iso/disk.toml +++ b/iso/disk.toml @@ -14,79 +14,98 @@ [customizations.kernel] append = "nvme_core.default_ps_max_latency_us=0" -# --------------------------------------------------------------------------- -# LVM disk layout +# =========================================================================== +# Disk layout: ALL PLAIN PARTITIONS. No LVM anywhere in the boot path. +# +# This mirrors the layout already running on z170asrock (and the working +# 21-july-2026 image), with /var living inside root. Root is the LAST partition, +# so after dd'ing to the target drive it grows with a plain: # -# Replaces the previous simple [[customizations.filesystem]] entry — the simple -# and advanced (customizations.disk) forms are alternatives, not additive. +# growpart && xfs_growfs / +# +# --------------------------------------------------------------------------- +# WHY THERE IS NO LVM HERE — established empirically 2026-07-29 +# --------------------------------------------------------------------------- +# Three separate attempts put a boot-critical filesystem on LVM. Each built +# cleanly, produced a byte-correct on-disk layout, and failed to boot: # -# Why LVM: the boot NVMe is moving from 931 GB to 4 TB and will carry root, -# /var (containers, appdata, Postgres) and the ZFS pool keyfile. Sizes for -# appdata are not knowable up front, so being able to reallocate later without -# repartitioning is worth the small extra layer. Nothing needs adding to the -# image for this: lvm2, device-mapper, the dracut 70lvm module and -# coreos-populate-lvmdevices.service are already present, and LVM is already -# in the initramfs (verified with lsinitrd). +# 1. root on an LV +# initramfs has NO LVM support at all: +# dracut modules present : crypt, dm <-- no lvm +# lvm files in initramfs : none +# FCOS finds root by LABEL; with no way to activate the VG the label never +# appears: +# dev-disk-by-label-root.device: Job ... failed with result 'timeout' +# Started emergency.service - Emergency Shell +# rd.lvm.lv= does NOT help — there is no LVM tooling in the initramfs to +# act on it. # -# /boot and /boot/efi are NOT listed here on purpose — bootc-image-builder -# creates them automatically as plain partitions when root is on an LV, which -# is required since the bootloader cannot read LVM. +# 2. root plain, /boot omitted (bib only auto-creates /boot when root is on an +# LV, and customizations.disk gives you exactly what you declare) +# error: grub_search_label:527: no such device: boot. +# error: grub_loader_boot:196: you need to load the kernel first. # -# bootc-image-builder supports exactly ONE volume group, so everything lives in -# vg4base. This is a new layout, not a reproduction of an old one: as of -# 2026-07-29 z170asrock has NO LVM — a single 20 GiB xfs partition carries -# /sysroot, /var and /etc, with ~911 GB of the 931 GB drive unallocated because -# the previous dd install was never grown. +# 3. root plain + /boot declared + /var on an LV +# Boots into the real root — GRUB, initramfs and ostree-prepare-root all +# fine — then dies because nothing activates the VG early enough: +# Timed out waiting for device dev-...- +# Dependency failed for var.mount - /var +# The image ships lvm2-monitor, lvm2-lvmpolld, lvm-devices-import and +# coreos-populate-lvmdevices, but NO activation unit (no lvm2-activation*, +# no lvm2-pvscan@). And coreos-populate-lvmdevices cannot possibly help — +# it declares RequiresMountsFor=/var/lib, i.e. it needs /var mounted, which +# is the very thing blocked. Circular. # -# Splitting /var out of root is the one structural change, and it is worth it on -# an unattended server: if container images, logs or Postgres fill a shared -# filesystem, you cannot stage an ostree upgrade or cleanly log in to fix it. -# A separate /var confines that blast radius to /var. FCOS supports a separate -# /var as a first-class configuration. +# Conclusion: in FCOS/uCore, LVM is only safe for storage mounted LATE, after +# basic.target. It cannot carry /, /boot or /var. # -# Sizes are MINIMUMS, kept deliberately small so disk.raw stays quick to build -# and to stream over HTTP during the swap (the current image is ~22 GB; this is -# ~26 GB). The volume group is grown into the 4 TB free space after install — -# and note LVM lets you leave VG space UNALLOCATED and hand it out later, which -# is the whole reason for doing this. # --------------------------------------------------------------------------- - -# IMPORTANT, learned by building this twice: +# WHERE LVM STILL BELONGS: app data, created AFTER install +# --------------------------------------------------------------------------- +# The goal LVM was wanted for — snapshot app data and copy the snapshots into +# ZFS — is fully served without touching the boot path. After install, carve a +# partition out of the free space and put per-app LVs on it, mounted under +# /var/lib/, with mount options that degrade instead of hanging a headless +# box: # -# `minsize` here is REQUIRED (omitting it fails the build with "minsize is -# required"), and bib does NOT treat it as "size of the whole VG". Measured -# behaviour: the ROOT lv is given this value — its own minsize below is ignored -# — every other LV gets its own minsize, and the VG ends up as the sum. +# nofail,x-systemd.device-timeout=30 # -# VG minsize 26 GiB + root 16 GiB + var 10 GiB -> root=26G var=10G VG=36G -# VG minsize 16 GiB + root 16 GiB + var 10 GiB -> root=16G var=10G VG=26G +# Then the backup flow is: +# xfs_freeze -f /var/lib/ +# lvcreate -s -L -n -snap vg4data/ +# xfs_freeze -u /var/lib/ +# mount -o ro,nouuid /dev/vg4data/-snap /mnt/snap +# rsync -aHAX --delete /mnt/snap/ /var/mnt//backup// +# zfs snapshot /backup/@ +# umount /mnt/snap && lvremove -f vg4data/-snap # -# So set this to the size you want ROOT to be. Getting it wrong is not merely a -# bigger disk.raw: XFS cannot shrink, so an oversized root LV is space that can -# never be handed back to var. -[[customizations.disk.partitions]] -type = "lvm" -name = "vg4base" -minsize = "16 GiB" +# See docs/storage-build-resume.md for the full post-install sequence. +# +# --------------------------------------------------------------------------- +# ALWAYS VM-BOOT A CHANGE TO THIS FILE BEFORE WRITING IT TO HARDWARE. +# A loop-mount proves the bytes are right, not that the thing boots. All three +# failures above passed loop-mount inspection. +# =========================================================================== -# Root: ostree deployments only, once /var is split out. The whole system -# currently uses 6.1 GiB including /var, so 16 GiB holds several deployments -# plus rollback headroom comfortably. -[[customizations.disk.partitions.logical_volumes]] -name = "root" -mountpoint = "/" -label = "root" +# /boot — MUST be declared explicitly and MUST be labelled "boot", because the +# baked grub.cfg does `search --label boot`. bib only auto-creates this when +# root is on an LV. 1 GiB matches the FCOS convention and the working install. +[[customizations.disk.partitions]] +type = "plain" +label = "boot" +mountpoint = "/boot" fs_type = "xfs" -minsize = "16 GiB" +minsize = "1 GiB" -# /var: containers, quadlet state, appdata, Postgres, and the ZFS pool keyfile. -# Small at install time on purpose — this is the LV to extend into the 4 TB. -[[customizations.disk.partitions.logical_volumes]] -name = "var" -mountpoint = "/var" -label = "var" +# Root — plain, labelled "root", which is what the initramfs looks for. Carries +# /var as well. 20 GiB matches the known-good image; it is the last partition, +# so grow it into the target drive after install. +[[customizations.disk.partitions]] +type = "plain" +label = "root" +mountpoint = "/" fs_type = "xfs" -minsize = "10 GiB" +minsize = "20 GiB" [[customizations.user]] name = "developer" From 972b1df26688cd1ab23455444a623fb35144c010 Mon Sep 17 00:00:00 2001 From: dx4homelab <145386996+dx4homelab@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:46:19 -0600 Subject: [PATCH 3/3] feat(install): self-sizing first boot, signed registry origin, auto-updates on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the raw disk image a complete starting point for a new server instead of something that needs a documented list of manual post-install steps. Three changes, all aimed at "dd it and walk away". 1. server4home-firstboot-grow.service — grows root into the real drive on first boot and hands the remainder to an LVM PV + empty VG (vg4data) for app data. Root deliberately stays 20 GiB in the image. The raw is sparse (~5 GB actual for a 22 GiB apparent file) but dd writes the *apparent* size, so baking a 250 GiB root would push a quarter-terabyte of mostly zeros onto every target drive. Growing on the hardware is also the only way one image can fit any boot drive size, which matters now that this image is meant to seed similar servers rather than one specific host. Target size comes from /etc/server4home/root-size (default 250G; the literal "max" takes the whole disk and creates no VG). If the drive is too small for the target, or the leftover would be under 50 GiB, root simply takes everything and no VG is made. The unit does the step that is easy to miss and impossible to skip: sgdisk -e. A small image written to a big drive leaves the GPT describing the *image's* size, so until the backup header is relocated there is literally no free space to grow into and growpart is a no-op. This is exactly the state z170asrock was found in — a 4 TB drive whose GPT reported "last usable sector 44455902, total free space 0 sectors". Root's own partition entry is never rewritten by hand: the unit creates the app-data partition at the point where root should stop, which bounds the gap, then lets growpart fill it exactly. Verified against z170asrock's real geometry — 250G yields a byte-exact 250 GiB root at an aligned boundary with 3.4 TiB left for the VG; 1T, 512G, 8T (too big) and max all behave correctly. Guarded for unattended use on a server: resolves root by filesystem LABEL rather than a hardcoded /dev/nvme0n1 (NVMe enumeration is not stable), refuses anything that is not XFS, grows the real XFS mountpoint (/sysroot — / is a composefs overlay and is not growable), treats growpart's exit 2 as "nothing to do", and keeps an attempt marker so a partial failure surfaces instead of retrying partition edits in a loop. 2. build-raw-ghcr — builds the raw from the published signed registry image. bib bakes whatever image reference it was given as the installed system's ostree origin. Building from localhost/server4home produces a host whose origin is ostree-image-signed:docker://localhost/server4home:stable, which no amount of enabling a timer can ever update — there is no such registry to pull from. This is the state z170asrock is in today. Building from the ghcr reference makes the machine self-updating from first boot and removes the post-install rpm-ostree rebase step entirely. 3. rpm-ostreed-automatic.timer enabled via timers.target.wants. It was inactive on every host, which is why one box sat 8 days behind its own healthy image pipeline. Paired with the existing AutomaticUpdates=stage policy this downloads and stages but leaves the operator to choose when it activates — deliberately not bootc-fetch-apply-updates.timer, which fetches and reboots, and which a storage server should not do unattended. Beszel is intentionally NOT pre-enabled. beszel.container gates on /etc/beszel/hub.enabled specifically so a single host runs the hub; baking that marker would make every server built from this image start its own. The agent gates on /etc/beszel/agent.env, which is a secret and must not ship in an image. Both quadlets and their tmpfiles are already baked, so the gates are the only thing left and they are correctly manual. Co-Authored-By: Claude Opus 5 (1M context) --- Justfile | 21 ++ build/files/etc/server4home/root-size | 17 ++ .../server4home-firstboot-grow.service | 1 + .../system/server4home-firstboot-grow.service | 25 +++ .../rpm-ostreed-automatic.timer | 1 + .../usr/libexec/server4home/firstboot-grow | 192 ++++++++++++++++++ iso/disk.toml | 17 +- 7 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 build/files/etc/server4home/root-size create mode 120000 build/files/usr/lib/systemd/system/multi-user.target.wants/server4home-firstboot-grow.service create mode 100644 build/files/usr/lib/systemd/system/server4home-firstboot-grow.service create mode 120000 build/files/usr/lib/systemd/system/timers.target.wants/rpm-ostreed-automatic.timer create mode 100755 build/files/usr/libexec/server4home/firstboot-grow diff --git a/Justfile b/Justfile index 752a4c9..f67d9a6 100644 --- a/Justfile +++ b/Justfile @@ -1,5 +1,8 @@ export image_name := env("IMAGE_NAME", "server4home") export default_tag := env("DEFAULT_TAG", "stable") +# Registry the published, signed images live in. Used by build-raw-ghcr so a disk +# image written to real hardware gets an origin that can actually be updated. +export registry := env("REGISTRY", "ghcr.io/dx4homelab") export bib_image := env("BIB_IMAGE", "quay.io/centos-bootc/bootc-image-builder:latest@sha256:903c01d110b8533f8891f07c69c0ba2377f8d4bc7e963311082b7028c04d529d") alias build-vm := build-qcow2 @@ -334,6 +337,24 @@ build-qcow2 $target_image=("localhost/" + image_name) $tag=default_tag: && (_bui [group('Build Virtal Machine Image')] build-raw $target_image=("localhost/" + image_name) $tag=default_tag: && (_build-bib target_image tag "raw" "iso/disk.toml") +# Build a RAW disk image FOR REAL HARDWARE, from the published signed registry image. +# +# Use this — not `build-raw` — for anything you intend to dd onto a machine. +# bib bakes the image reference it was given as the installed system's ostree +# origin. Building from `localhost/server4home` therefore produces a host whose +# origin is `ostree-image-signed:docker://localhost/server4home:stable`, which no +# amount of enabling rpm-ostreed-automatic.timer can ever update: there is no +# such registry to pull from. Building from the ghcr reference bakes +# `ostree-image-signed:docker://ghcr.io/dx4homelab/server4home:stable` instead, +# so the machine is self-updating from first boot and needs no post-install rebase. +# +# _rootful_load_image pulls the reference when it is not present locally, so this +# works from a clean checkout. + +# Build a RAW disk image for real hardware (signed registry origin, self-updating) +[group('Build Virtal Machine Image')] +build-raw-ghcr $tag=default_tag: && (_build-bib (registry + "/" + image_name) tag "raw" "iso/disk.toml") + # Build the base ("plain" / storage) installer ISO — non-LVM, no K3s rebase [group('Build Virtal Machine Image')] build-iso-plain $target_image=("localhost/" + image_name) $tag=default_tag: && (_build-bib target_image tag "iso" "iso/iso-plain.toml") diff --git a/build/files/etc/server4home/root-size b/build/files/etc/server4home/root-size new file mode 100644 index 0000000..008f729 --- /dev/null +++ b/build/files/etc/server4home/root-size @@ -0,0 +1,17 @@ +# Target size for the root partition, applied once on first boot by +# server4home-firstboot-grow.service. +# +# The disk image ships a small (20 GiB) root so `dd` stays fast; this is the size +# root is grown to on the real hardware. Whatever is left over becomes an LVM PV +# and an empty VG named vg4data, for app data (see docs/storage-build-resume.md). +# +# Accepts IEC sizes: 250G, 512G, 1T, ... +# The literal value "max" grows root over the entire disk and creates no VG. +# +# To change it for a given host, edit this file BEFORE the first boot (loop-mount +# the raw, or write it during provisioning). After the grow has run the stamp at +# /var/lib/server4home/firstboot-grow.done makes this file inert — root cannot be +# shrunk afterwards, since XFS only grows. +# +# Comments and blank lines are ignored; the first real line is used. +250G diff --git a/build/files/usr/lib/systemd/system/multi-user.target.wants/server4home-firstboot-grow.service b/build/files/usr/lib/systemd/system/multi-user.target.wants/server4home-firstboot-grow.service new file mode 120000 index 0000000..2f80cd6 --- /dev/null +++ b/build/files/usr/lib/systemd/system/multi-user.target.wants/server4home-firstboot-grow.service @@ -0,0 +1 @@ +../server4home-firstboot-grow.service \ No newline at end of file diff --git a/build/files/usr/lib/systemd/system/server4home-firstboot-grow.service b/build/files/usr/lib/systemd/system/server4home-firstboot-grow.service new file mode 100644 index 0000000..62fae2e --- /dev/null +++ b/build/files/usr/lib/systemd/system/server4home-firstboot-grow.service @@ -0,0 +1,25 @@ +[Unit] +Description=Grow root into the boot drive and create the app-data VG (first boot) +Documentation=https://github.com/dx4homelab/server4home +# Needs udev populated, the root filesystem mounted rw, and LVM tooling available. +# basic.target covers all three; nothing here is boot-critical, so running late +# is correct — a failure must degrade, not hang a headless server. +After=basic.target +Wants=basic.target +ConditionPathExists=/usr/libexec/server4home/firstboot-grow +# The stamp makes this a genuine one-shot across reboots. +ConditionPathExists=!/var/lib/server4home/firstboot-grow.done +# Nothing to grow in a container build. +ConditionVirtualization=!container + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/libexec/server4home/firstboot-grow +# Partition edits must not be interrupted part-way. +TimeoutStartSec=600 +# Deliberately no Restart=: the script keeps its own attempt marker and refuses +# to retry partition edits unattended. A failure should be looked at, not looped. + +[Install] +WantedBy=multi-user.target diff --git a/build/files/usr/lib/systemd/system/timers.target.wants/rpm-ostreed-automatic.timer b/build/files/usr/lib/systemd/system/timers.target.wants/rpm-ostreed-automatic.timer new file mode 120000 index 0000000..1dcb732 --- /dev/null +++ b/build/files/usr/lib/systemd/system/timers.target.wants/rpm-ostreed-automatic.timer @@ -0,0 +1 @@ +../rpm-ostreed-automatic.timer \ No newline at end of file diff --git a/build/files/usr/libexec/server4home/firstboot-grow b/build/files/usr/libexec/server4home/firstboot-grow new file mode 100755 index 0000000..effdafd --- /dev/null +++ b/build/files/usr/libexec/server4home/firstboot-grow @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# Grow root into the target drive on first boot, and hand the remainder to an +# LVM PV for app data. +# +# Why this exists +# --------------- +# The disk image ships a deliberately small root (20 GiB) so the raw stays quick +# to write — a 250 GiB root would make `dd` push a quarter-terabyte of zeros on +# every install. The real sizing therefore has to happen on the target hardware, +# which is also the only place that knows how big the drive actually is. Doing it +# here rather than by hand means a freshly dd'd server reaches its final layout +# unattended, and the same image works on any boot drive size. +# +# What it does +# ------------ +# 1. sgdisk -e — relocate the backup GPT header to the true end of the disk. +# A small image written to a big drive always leaves the GPT +# describing the *image's* size, so until this runs there is +# literally no free space to grow into. +# 2. create the app-data partition at the point where root should stop, which +# bounds root's growth to the target size. growpart then fills exactly the +# gap — root is never rewritten by hand. +# 3. growpart + xfs_growfs to expand root online. +# 4. pvcreate/vgcreate the app-data partition (empty VG; LVs are created later +# per app — see docs/storage-build-resume.md). +# +# Sizing +# ------ +# Target root size is read from /etc/server4home/root-size, e.g. "250G", "1T". +# The literal value "max" grows root over the whole disk and creates no VG. +# If the drive is too small for the target, or the leftover would be under +# MIN_VG, root simply takes the whole disk and no VG is created. +# +# Why no LVM for root itself: FCOS/uCore cannot boot a root/boot/var on LVM — +# the initramfs has no LVM support at all. See iso/disk.toml for the three +# documented failure modes. LVM is only safe for storage mounted late, which is +# exactly what the app-data VG is. + +set -euo pipefail + +CONF="/etc/server4home/root-size" +DEFAULT_TARGET="250G" +VG_NAME="vg4data" +# Not worth carving a VG out of scraps; below this root just takes everything. +MIN_VG_BYTES=$((50 * 1024 * 1024 * 1024)) +STAMP="/var/lib/server4home/firstboot-grow.done" +ATTEMPT="/var/lib/server4home/firstboot-grow.attempted" + +log() { echo "firstboot-grow: $*"; } +die() { log "ERROR: $*"; exit 1; } + +if [ -e "${STAMP}" ]; then + log "already completed on an earlier boot; nothing to do" + exit 0 +fi + +# Loop guard. If a previous boot got part-way and failed, do not keep retrying +# partition edits unattended on a server — surface it and stop. +if [ -e "${ATTEMPT}" ]; then + die "a previous attempt did not complete. Refusing to retry partition edits + automatically. Inspect the disk, then remove ${ATTEMPT} to allow a retry." +fi + +# --------------------------------------------------------------------------- +# Locate root by filesystem LABEL, never by a hardcoded /dev/nvme0n1 — kernel +# NVMe enumeration is not stable across reboots. +# --------------------------------------------------------------------------- +ROOT="$(lsblk -pnro NAME,LABEL | awk '$2=="root"{print $1; exit}')" +[ -n "${ROOT}" ] || die "no partition with filesystem label 'root' found" +[ -b "${ROOT}" ] || die "${ROOT} is not a block device" + +DISK="$(lsblk -pnro PKNAME "${ROOT}" | head -1)" +[ -n "${DISK}" ] && [ -b "${DISK}" ] || die "could not derive parent disk of ${ROOT}" + +NUM="${ROOT##*[!0-9]}" +[ -n "${NUM}" ] || die "could not derive partition number from ${ROOT}" + +# Only XFS can be grown by this script, and only forward. +FSTYPE="$(lsblk -pnro FSTYPE "${ROOT}")" +[ "${FSTYPE}" = "xfs" ] || die "root filesystem is '${FSTYPE}', expected xfs" + +# The XFS is mounted at /sysroot on an ostree system; / is a composefs overlay +# and is NOT growable, so resolve the real mountpoint rather than assuming "/". +MP="" +for cand in /sysroot /; do + if [ "$(findmnt -nro SOURCE --target "${cand}" 2>/dev/null)" = "${ROOT}" ]; then + MP="${cand}" + break + fi +done +[ -n "${MP}" ] || die "could not find the mountpoint backed by ${ROOT}" + +log "root=${ROOT} disk=${DISK} partnum=${NUM} mountpoint=${MP}" + +# --------------------------------------------------------------------------- +# Step 1 — make the whole drive visible to the GPT. +# --------------------------------------------------------------------------- +mkdir -p "$(dirname "${ATTEMPT}")" +: >"${ATTEMPT}" + +log "relocating backup GPT header to the end of ${DISK}" +sgdisk -e "${DISK}" +partprobe "${DISK}" 2>/dev/null || true + +SS="$(blockdev --getss "${DISK}")" +ROOT_START="$(sgdisk -i "${NUM}" "${DISK}" | awk '/^First sector/{print $3}')" +LAST_USABLE="$(sgdisk -p "${DISK}" | awk '/last usable sector/{print $NF}')" +[ -n "${SS}" ] && [ -n "${ROOT_START}" ] && [ -n "${LAST_USABLE}" ] || + die "could not read disk geometry (ss=${SS} start=${ROOT_START} last=${LAST_USABLE})" + +log "sector size=${SS} root start=${ROOT_START} last usable=${LAST_USABLE}" + +# --------------------------------------------------------------------------- +# Step 2 — decide where root should stop, and bound it with the VG partition. +# --------------------------------------------------------------------------- +TARGET="${DEFAULT_TARGET}" +if [ -r "${CONF}" ]; then + # First non-comment, non-blank line wins, so the shipped file can document itself. + CONF_VAL="$(sed -e 's/#.*//' -e 's/[[:space:]]//g' "${CONF}" | grep -m1 . || true)" + [ -n "${CONF_VAL}" ] && TARGET="${CONF_VAL}" +fi +log "target root size: ${TARGET} (from ${CONF} if present, else default)" + +make_vg=0 +if [ "${TARGET}" = "max" ]; then + log "target is 'max' — root takes the whole disk, no ${VG_NAME}" +else + TARGET_BYTES="$(numfmt --from=iec "${TARGET}" 2>/dev/null)" || + die "could not parse target size '${TARGET}' (expected e.g. 250G, 1T, or max)" + TARGET_SECTORS=$((TARGET_BYTES / SS)) + # Align the next partition start to 2048 sectors (1 MiB) — root ends just before it. + NEXT_START=$(((ROOT_START + TARGET_SECTORS + 2047) / 2048 * 2048)) + + if [ "${NEXT_START}" -ge "${LAST_USABLE}" ]; then + log "drive is too small for a ${TARGET} root plus a VG — root takes the whole disk" + else + REMAIN_BYTES=$(((LAST_USABLE - NEXT_START + 1) * SS)) + if [ "${REMAIN_BYTES}" -lt "${MIN_VG_BYTES}" ]; then + log "only $((REMAIN_BYTES / 1024 / 1024 / 1024))G would remain, under the" + log "$((MIN_VG_BYTES / 1024 / 1024 / 1024))G minimum — root takes the whole disk" + elif [ -n "$(sgdisk -p "${DISK}" | awk -v n="${NUM}" '$1 ~ /^[0-9]+$/ && $1+0 > n+0 {print $1; exit}')" ]; then + log "a partition already exists after root — leaving it alone, not creating ${VG_NAME}" + else + log "creating ${VG_NAME} partition at sector ${NEXT_START} (bounds root to ~${TARGET})" + sgdisk -n "0:${NEXT_START}:0" -t "0:8e00" -c "0:${VG_NAME}" "${DISK}" + partprobe "${DISK}" 2>/dev/null || true + make_vg=1 + fi + fi +fi + +# --------------------------------------------------------------------------- +# Step 3 — grow root into whatever space is now available before the next partition. +# --------------------------------------------------------------------------- +log "growing partition ${NUM} on ${DISK}" +rc=0 +growpart "${DISK}" "${NUM}" || rc=$? +# growpart: 0 = resized, 2 = nothing to do. Anything else is a real failure. +if [ "${rc}" -ne 0 ] && [ "${rc}" -ne 2 ]; then + die "growpart failed with exit code ${rc}" +fi +[ "${rc}" -eq 2 ] && log "partition already at its maximum; nothing to grow" +partprobe "${DISK}" 2>/dev/null || true + +log "growing the xfs filesystem at ${MP}" +xfs_growfs "${MP}" + +# --------------------------------------------------------------------------- +# Step 4 — initialise the app-data VG (empty; LVs come later, per app). +# --------------------------------------------------------------------------- +if [ "${make_vg}" -eq 1 ]; then + VGPART="$(lsblk -pnro NAME,PARTLABEL "${DISK}" | awk -v l="${VG_NAME}" '$2==l{print $1; exit}')" + if [ -n "${VGPART}" ] && [ -b "${VGPART}" ]; then + if pvs "${VGPART}" >/dev/null 2>&1; then + log "${VGPART} is already a PV; leaving it alone" + else + log "initialising ${VGPART} as PV and creating VG ${VG_NAME}" + pvcreate "${VGPART}" + vgcreate "${VG_NAME}" "${VGPART}" + fi + else + log "WARNING: could not locate the ${VG_NAME} partition after partprobe;" + log "WARNING: create the PV/VG by hand (see docs/storage-build-resume.md)" + fi +fi + +log "final layout:" +lsblk -o NAME,SIZE,FSTYPE,PARTLABEL,MOUNTPOINT "${DISK}" || true + +: >"${STAMP}" +rm -f "${ATTEMPT}" +log "complete" diff --git a/iso/disk.toml b/iso/disk.toml index 589c1c5..04426d0 100644 --- a/iso/disk.toml +++ b/iso/disk.toml @@ -98,8 +98,21 @@ fs_type = "xfs" minsize = "1 GiB" # Root — plain, labelled "root", which is what the initramfs looks for. Carries -# /var as well. 20 GiB matches the known-good image; it is the last partition, -# so grow it into the target drive after install. +# /var as well. +# +# Deliberately left at 20 GiB rather than the final size. The raw is sparse +# (~5 GB actual for a 22 GiB apparent file), but `dd` writes the *apparent* size, +# so baking a 250 GiB root would push a quarter-terabyte of mostly zeros onto +# every target drive. Instead root is grown on the real hardware by +# server4home-firstboot-grow.service, which also knows how big the drive actually +# is — so one image fits any boot drive size. +# +# Root must stay the LAST declared partition: the first-boot unit bounds its +# growth by creating the app-data partition after it, then lets growpart fill the +# gap, so root's own entry is never rewritten by hand. +# +# Target size comes from /etc/server4home/root-size (default 250G, "max" = whole +# disk). See build/files/usr/libexec/server4home/firstboot-grow. [[customizations.disk.partitions]] type = "plain" label = "root"