test(kairos): run component canaries

Author: robarnold

.github/workflows/image-kairos-fedora.yml

@@ -6,24 +6,33 @@ on: # yamllint disable-line rule:truthy
     branches:
       - main
     paths:
-      - .devcontainer/Dockerfile
+      - .devcontainer/**
       - .github/workflows/image-kairos-fedora.yml
+      - ansible/**
       - images/kairos-fedora/**
       - justfile
+      - kustomization/**
     tags:
       - kairos-fedora-*
   pull_request:
     branches:
       - main
     paths:
-      - .devcontainer/Dockerfile
+      - .devcontainer/**
       - .github/workflows/image-kairos-fedora.yml
+      - ansible/**
       - images/kairos-fedora/**
       - justfile
+      - kustomization/**
 
 jobs:
   build-container:
+    name: E2E (${{ matrix.component }})
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ github.ref_type == 'tag' && fromJSON('["priorityclass"]') || fromJSON('["kube-vip", "priorityclass"]') }}
     timeout-minutes: 90
     permissions:
       contents: read
@@ -63,7 +72,19 @@ jobs:
       - name: Install E2E Dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes curl openssh-client ovmf qemu-system-x86 qemu-utils
+          sudo apt-get install --no-install-recommends --yes curl openssh-client ovmf python3-venv qemu-system-x86 qemu-utils
+
+          python3 -m venv "${RUNNER_TEMP}/kairos-e2e-venv"
+          "${RUNNER_TEMP}/kairos-e2e-venv/bin/pip" install --requirement ansible/requirements.txt
+          "${RUNNER_TEMP}/kairos-e2e-venv/bin/ansible-galaxy" collection install --force ansible
+          echo "${RUNNER_TEMP}/kairos-e2e-venv/bin" >> "$GITHUB_PATH"
+
+          mkdir -p "${RUNNER_TEMP}/kustomize-bin"
+          kustomize_container="$(docker create registry.k8s.io/kustomize/kustomize:v5.8.1@sha256:899fcd3bc898160e62bcaf82932b0cb29ba38d16272353db2e7acbba82129429)"
+          trap 'docker rm --force "$kustomize_container" >/dev/null' EXIT
+          docker cp "${kustomize_container}:/app/kustomize" "${RUNNER_TEMP}/kustomize-bin/kustomize"
+          chmod 0755 "${RUNNER_TEMP}/kustomize-bin/kustomize"
+          echo "${RUNNER_TEMP}/kustomize-bin" >> "$GITHUB_PATH"
       - name: Enable KVM Access
         run: |
           test -e /dev/kvm
@@ -73,13 +94,14 @@ jobs:
           ls -l /dev/kvm
       - name: Run Kairos Fedora E2E
         env:
+          E2E_KUSTOMIZATION_TEST: ${{ matrix.component }}
           KAIROS_IMAGE: ghcr.io/${{ github.repository_owner }}/kairos-fedora:${{ steps.version_tag.outputs.tag }}
         run: bash images/kairos-fedora/e2e.sh
       - name: Upload E2E Logs
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: kairos-fedora-e2e
+          name: kairos-fedora-e2e-${{ matrix.component }}
           path: |
             build/e2e/kairos-fedora/install-cloud-config.yaml
             build/e2e/kairos-fedora/id_ed25519.pub

ansible/roles/minikube_test/defaults/main.yml

@@ -5,6 +5,10 @@
 # https://minikube.sigs.k8s.io/docs/tutorials/local_path_provisioner/
 minikube_test__enable_local_path_storage: false
 
+# If this role should provision a minikube cluster. Disable this to apply the
+# kustomization to an existing Kubernetes context.
+minikube_test__provision_cluster: true
+
 # renovate: datasource=github-releases depName=kubernetes/kubernetes
 minikube_test__kubernetes_version: v1.36.2
 

ansible/roles/minikube_test/tasks/main.yml

@@ -1,6 +1,7 @@
 ---
 # tasks file for minikube_test
 - name: Start minikube
+  when: minikube_test__provision_cluster | bool
   ansible.builtin.command:
     argv:
       - minikube
@@ -13,7 +14,9 @@
   changed_when: true
   notify: stop minikube
 - name: Setup local path provisioner
-  when: minikube_test__enable_local_path_storage | bool
+  when:
+    - minikube_test__provision_cluster | bool
+    - minikube_test__enable_local_path_storage | bool
   block:
     - name: Enable local path provisioner
       ansible.builtin.command:

images/kairos-fedora/README.md

@@ -19,18 +19,32 @@ The harness validates the same path a bare-metal install uses:
 3. Boot the installer ISO in QEMU.
 4. Submit the install through the Kairos WebUI on port 8080.
 5. Power off, detach the ISO, boot from the installed disk, and verify the OS over SSH.
+6. Reboot into the immutable system and verify SSH and K3s again.
+
+Pass a component name to apply its existing kustomization test to K3s after the immutable reboot:
+
+```sh
+just e2e-kairos-fedora priorityclass
+```
 
 By default, `just e2e-kairos-fedora` builds `images/kairos-fedora/Containerfile` locally as `kairos-fedora:0.0.0-e2e`. Set `KAIROS_IMAGE` to test a specific image ref that already exists in Docker or can be pulled by AuroraBoot:
 
 ```sh
 KAIROS_IMAGE=ghcr.io/marinatedconcrete/kairos-fedora:0.8.1 just e2e-kairos-fedora
 ```
 
+When iterating on a component, reuse an existing installer ISO to skip both the image and ISO builds:
+
+```sh
+KAIROS_ISO=/path/to/kairos-fedora.iso just e2e-kairos-fedora priorityclass
+```
+
 Useful overrides:
 
 - `KAIROS_IMAGE`: OCI image ref to convert into an installer ISO.
 - `KAIROS_ISO`: existing ISO path for debugging the boot/install path without regenerating media.
 - `E2E_WORKDIR`: artifact and log directory, default `build/e2e/kairos-fedora`.
+- `E2E_K3S_PORT`: host port forwarded to the guest K3s API, default `16443`.
 - `E2E_INSTALL_TIMEOUT`, `E2E_BOOT_TIMEOUT`, `E2E_K3S_TIMEOUT`: timeout overrides.
 
 The CI workflow runs the same script after building the image into the local Docker daemon. On release tags, the workflow pushes the image only after the e2e test passes.

images/kairos-fedora/e2e.sh

@@ -42,6 +42,8 @@ E2E_BOOT_TIMEOUT="${E2E_BOOT_TIMEOUT:-15m}"
 E2E_K3S_TIMEOUT="${E2E_K3S_TIMEOUT:-10m}"
 E2E_WEBUI_PORT="${E2E_WEBUI_PORT:-18080}"
 E2E_SSH_PORT="${E2E_SSH_PORT:-10022}"
+E2E_K3S_PORT="${E2E_K3S_PORT:-16443}"
+E2E_KUSTOMIZATION_TEST="${E2E_KUSTOMIZATION_TEST:-}"
 KAIROS_E2E_VERSION="${KAIROS_E2E_VERSION:-0.0.0-e2e}"
 # renovate: datasource=docker depName=quay.io/kairos/auroraboot
 AURORABOOT_IMAGE="${AURORABOOT_IMAGE:-quay.io/kairos/auroraboot:v0.20.1}"
@@ -50,6 +52,7 @@ logs_dir="${E2E_WORKDIR}/logs"
 disk_path="${E2E_WORKDIR}/kairos-fedora.qcow2"
 ssh_key="${E2E_WORKDIR}/id_ed25519"
 cloud_config="${E2E_WORKDIR}/install-cloud-config.yaml"
+kubeconfig="${E2E_WORKDIR}/kubeconfig"
 qemu_pid=""
 ssh_ready=0
 phase="host"
@@ -192,7 +195,7 @@ start_qemu() {
     else
         qemu_args+=(
             -boot "order=c,menu=off"
-            -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${E2E_SSH_PORT}-:22"
+            -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${E2E_SSH_PORT}-:22,hostfwd=tcp:127.0.0.1:${E2E_K3S_PORT}-:6443"
             -device "virtio-net-pci,netdev=net0"
         )
     fi
@@ -358,6 +361,29 @@ reboot_and_wait_for_ssh() {
     die "SSH did not become reachable after reboot before ${E2E_BOOT_TIMEOUT}"
 }
 
+write_kubeconfig() {
+    log "Writing host kubeconfig for K3s on localhost:${E2E_K3S_PORT}"
+    ssh_cmd 'sudo -n cat /etc/rancher/k3s/k3s.yaml' >"$kubeconfig"
+    sed -i "s#server: https://127.0.0.1:6443#server: https://127.0.0.1:${E2E_K3S_PORT}#" "$kubeconfig"
+    chmod 0600 "$kubeconfig"
+    grep -q "server: https://127.0.0.1:${E2E_K3S_PORT}" "$kubeconfig" || die "failed to rewrite K3s server in ${kubeconfig}"
+}
+
+run_kustomization_test() {
+    local component="$1"
+    local playbook="${repo_root}/kustomization/tests/${component}/test.yml"
+    local test_log="${logs_dir}/kustomization-${component}.log"
+
+    [[ "$component" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "invalid kustomization test name: ${component}"
+    [[ -f "$playbook" ]] || die "kustomization test does not exist: ${component}"
+
+    log "Running ${component} kustomization test against Kairos K3s"
+    KUBECONFIG="$kubeconfig" ansible-playbook "$playbook" \
+        --extra-vars minikube_test__provision_cluster=false \
+        --extra-vars kubernetes_core_k8s__context=default \
+        2>&1 | tee "$test_log"
+}
+
 collect_ssh_diagnostics() {
     [[ "$ssh_ready" == 1 ]] || return 0
 
@@ -386,6 +412,7 @@ on_exit() {
     fi
 
     cleanup_qemu
+    rm -f -- "$kubeconfig"
     exit "$status"
 }
 
@@ -429,11 +456,16 @@ main() {
     require_cmd ssh
     require_cmd ssh-keygen
 
+    if [[ -n "$E2E_KUSTOMIZATION_TEST" ]]; then
+        require_cmd ansible-playbook
+        require_cmd kustomize
+    fi
+
     [[ -e /dev/kvm ]] || die "/dev/kvm is unavailable; enable KVM before running this test"
     [[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm exists but is not readable and writable by this user"
 
     mkdir -p "$logs_dir"
-    if [[ -z "${KAIROS_IMAGE:-}" ]]; then
+    if [[ -z "${KAIROS_IMAGE:-}" && -z "${KAIROS_ISO:-}" ]]; then
         build_default_image "$kairos_image"
     fi
 
@@ -466,6 +498,12 @@ main() {
     reboot_and_wait_for_ssh
     run_guest_checks
 
+    if [[ -n "$E2E_KUSTOMIZATION_TEST" ]]; then
+        phase="kustomization-${E2E_KUSTOMIZATION_TEST}"
+        write_kubeconfig
+        run_kustomization_test "$E2E_KUSTOMIZATION_TEST"
+    fi
+
     phase="complete"
     log "Kairos Fedora e2e completed successfully"
 }

justfile

@@ -221,10 +221,10 @@ lint: ansible-lint hado-lint kustomize-lint renovate-lint shellcheck-lint
 
 # Run the Kairos Fedora boot/install end-to-end test
 [group('test')]
-e2e-kairos-fedora:
+e2e-kairos-fedora component="":
     #!/usr/bin/env bash
     set -euo pipefail
-    bash images/kairos-fedora/e2e.sh
+    E2E_KUSTOMIZATION_TEST="{{ component }}" bash images/kairos-fedora/e2e.sh
 
 # Run a single test
 [group('test')]