WIP: Add LXD infra provider

.github/workflows/e2e.yaml

@@ -18,8 +18,7 @@ jobs:
     name: Build & Run E2E Images
     runs-on: [self-hosted, linux, X64, jammy, xlarge]
     steps:
-      -
-        name: Login to GitHub Container Registry
+      - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
           # We run into rate limiting issues if we don't authenticate
@@ -40,28 +39,16 @@ jobs:
           sudo snap install kubectl --classic --channel=1.32/stable
       - name: Build provider images
         run: sudo env "PATH=$PATH" make docker-build-e2e
-      - name: Build k8s-snap images
-        working-directory: hack/
-        run: |
-          ./build-e2e-images.sh
       - name: Save provider image
         run: |
           sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev
           sudo chmod 775 provider-images.tar
-      - name: Save k8s-snap image
-        run: |
-          sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old
-          sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new
-          sudo chmod 775 k8s-snap-image-old.tar
-          sudo chmod 775 k8s-snap-image-new.tar
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: e2e-images
           path: |
             provider-images.tar
-            k8s-snap-image-old.tar
-            k8s-snap-image-new.tar
 
   run-e2e-tests:
     name: Run E2E Tests
@@ -80,8 +67,7 @@ jobs:
       # TODO(ben): Remove once all tests are running stable.
       fail-fast: false
     steps:
-      -
-        name: Login to GitHub Container Registry
+      - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
           # We run into rate limiting issues if we don't authenticate
@@ -94,6 +80,13 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
+      - name: Setup LXD
+        uses: canonical/[email protected]
+        with:
+          bridges: "lxdbr0"
+      - name: Configure LXD
+        run: |
+          sudo ./hack/setup-lxd.sh
       - name: Install requirements
         run: |
           sudo apt update
@@ -107,13 +100,6 @@ jobs:
           path: .
       - name: Load provider image
         run: sudo docker load -i provider-images.tar
-      - name: Load k8s-snap old image
-        run: |
-          sudo docker load -i k8s-snap-image-old.tar
-      - name: Load k8s-snap new image
-        if: matrix.ginkgo_focus == 'Workload cluster upgrade'
-        run: |
-          sudo docker load -i k8s-snap-image-new.tar
       - name: Create docker network
         run: |
           sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true

Makefile

@@ -99,7 +99,7 @@ GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run
 GINKGO_TIMEOUT ?= 2h
 GINKGO_POLL_PROGRESS_AFTER ?= 60m
 GINKGO_POLL_PROGRESS_INTERVAL ?= 5m
-E2E_INFRA ?= docker
+E2E_INFRA ?= incus
 E2E_CONF_FILE ?= $(TEST_DIR)/e2e/config/ck8s-$(E2E_INFRA).yaml
 SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false

hack/setup-lxd.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+# https://capn.linuxcontainers.org/tutorial/quick-start.html
+ip_address="$(ip -o route get to 1.1.1.1 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')"
+sudo lxc config set core.https_address "$ip_address:8443"
+
+token="$(sudo lxc config trust add --name client | tail -1)"
+sudo lxc remote add local-https --token "$token" "https://$(sudo lxc config get core.https_address)"

pkg/cloudinit/controlplane_init.go

@@ -54,6 +54,7 @@ func NewInitControlPlane(input InitControlPlaneInput) (CloudConfig, error) {
 	// run commands
 	config.RunCommands = append(config.RunCommands, input.PreRunCommands...)
 	if !input.AirGapped {
+		// config.RunCommands = append(config.RunCommands, "/capi/scripts/ensure-snapd.sh", "/capi/scripts/install.sh")
 		config.RunCommands = append(config.RunCommands, "/capi/scripts/install.sh")
 	}
 	config.RunCommands = append(config.RunCommands,

pkg/cloudinit/controlplane_init_test.go

@@ -66,6 +66,7 @@ func TestNewInitControlPlane(t *testing.T) {
 		"/capi/scripts/configure-snapstore-proxy.sh",
 		"prerun1",
 		"prerun2",
+		// "/capi/scripts/ensure-snapd.sh",
 		"/capi/scripts/install.sh",
 		"/capi/scripts/disable-host-services.sh",
 		"/capi/scripts/bootstrap.sh",
@@ -82,6 +83,7 @@ func TestNewInitControlPlane(t *testing.T) {
 	// NOTE (mateoflorido): Keep this test in sync with the expected paths in the controlplane_init.go file.
 	g.Expect(config.WriteFiles).To(ConsistOf(
 		HaveField("Path", "/capi/scripts/disable-host-services.sh"),
+		// HaveField("Path", "/capi/scripts/ensure-snapd.sh"),
 		HaveField("Path", "/capi/scripts/install.sh"),
 		HaveField("Path", "/capi/scripts/bootstrap.sh"),
 		HaveField("Path", "/capi/scripts/load-images.sh"),
@@ -228,6 +230,7 @@ func TestNewInitControlPlaneAirGapped(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	// Verify the run commands is missing install.sh script.
+	// g.Expect(config.RunCommands).NotTo(ContainElement("/capi/scripts/ensure-snapd.sh"))
 	g.Expect(config.RunCommands).NotTo(ContainElement("/capi/scripts/install.sh"))
 }
 

pkg/cloudinit/controlplane_join.go

@@ -43,6 +43,7 @@ func NewJoinControlPlane(input JoinControlPlaneInput) (CloudConfig, error) {
 	// run commands
 	config.RunCommands = append(config.RunCommands, input.PreRunCommands...)
 	if !input.AirGapped {
+		// config.RunCommands = append(config.RunCommands, "/capi/scripts/ensure-snapd.sh", "/capi/scripts/install.sh")
 		config.RunCommands = append(config.RunCommands, "/capi/scripts/install.sh")
 	}
 	config.RunCommands = append(config.RunCommands,

pkg/cloudinit/controlplane_join_test.go

@@ -46,6 +46,7 @@ func TestNewJoinControlPlane(t *testing.T) {
 		"/capi/scripts/configure-snapstore-proxy.sh",
 		"prerun1",
 		"prerun2",
+		// "/capi/scripts/ensure-snapd.sh",
 		"/capi/scripts/install.sh",
 		"/capi/scripts/disable-host-services.sh",
 		"/capi/scripts/load-images.sh",
@@ -59,6 +60,7 @@ func TestNewJoinControlPlane(t *testing.T) {
 
 	// NOTE (mateoflorido): Keep this test in sync with the expected paths in the controlplane_join.go file.
 	g.Expect(config.WriteFiles).To(ConsistOf(
+		// HaveField("Path", "/capi/scripts/ensure-snapd.sh"),
 		HaveField("Path", "/capi/scripts/install.sh"),
 		HaveField("Path", "/capi/scripts/disable-host-services.sh"),
 		HaveField("Path", "/capi/scripts/bootstrap.sh"),
@@ -184,6 +186,7 @@ func TestNewJoinControlPlaneAirGapped(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	// Verify the run commands is missing install.sh script.
+	// g.Expect(config.RunCommands).NotTo(ContainElement("/capi/scripts/ensure-snapd.sh"))
 	g.Expect(config.RunCommands).NotTo(ContainElement("/capi/scripts/install.sh"))
 }
 

pkg/cloudinit/embed.go

@@ -15,6 +15,7 @@
 // NOTE(eac): If you want to use a script from pkg/cloudinit/scripts in your code (for example, you want to include a script in the user-data.txt),
 // you need to add it to the scripts map below.
 var (
+	// scriptEnsureSnapd             script = "ensure-snapd.sh"
 	scriptInstall                 script = "install.sh"
 	scriptDisableHostServices     script = "disable-host-services.sh"
 	scriptBootstrap               script = "bootstrap.sh"
@@ -40,6 +41,7 @@
 var (
 	// scripts is a map of all embedded bash scripts used in the cloud-init.
 	scripts = map[script]string{
+		// scriptEnsureSnapd:             mustEmbed(scriptEnsureSnapd),
 		scriptInstall:                 mustEmbed(scriptInstall),
 		scriptDisableHostServices:     mustEmbed(scriptDisableHostServices),
 		scriptBootstrap:               mustEmbed(scriptBootstrap),

pkg/cloudinit/scripts/bootstrap.sh

@@ -10,5 +10,6 @@ name="$(cat /capi/etc/node-name)"
 config_file="/capi/etc/config.yaml"
 
 if [ ! -f /etc/kubernetes/pki/ca.crt ]; then
+  # /snap/bin/k8s bootstrap --name "${name}" --address "${address}" --file "${config_file}"
   k8s bootstrap --name "${name}" --address "${address}" --file "${config_file}"
 fi

pkg/cloudinit/scripts/configure-auth-token.sh

@@ -4,4 +4,5 @@
 ## - k8s is installed and cluster is bootstrapped
 ## - /capi/etc/token contains the token CAPI providers can use to authenticate with k8sd
 
+# /snap/bin/k8s x-capi set-auth-token "$(cat /capi/etc/token)"
 k8s x-capi set-auth-token "$(cat /capi/etc/token)"

pkg/cloudinit/scripts/configure-node-token.sh

@@ -4,4 +4,5 @@
 ## - k8s is installed and cluster is bootstrapped
 ## - /capi/etc/node-token contains the token CAPI providers can use to authenticate with k8sd for per-node operations
 
+# /snap/bin/k8s x-capi set-node-token "$(cat /capi/etc/node-token)"
 k8s x-capi set-node-token "$(cat /capi/etc/node-token)"

pkg/cloudinit/scripts/deploy-manifests.sh

@@ -5,5 +5,6 @@
 ## - /capi/manifests/ is a directory with YAML manifests to deploy once on the cluster.
 
 for file in $(find /capi/manifests/ -name '*.yaml' || true | sort); do
+  # /snap/bin/k8s kubectl apply -f "$file"
   k8s kubectl apply -f "$file"
 done

pkg/cloudinit/scripts/ensure-snapd.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -xe
+
+apt-get update
+apt-get install -y snapd
+snap wait core seed.loaded

pkg/cloudinit/scripts/install.sh

@@ -6,12 +6,38 @@
 ## - /capi/etc/snap-local-path contains the path to the local snap file to be installed (e.g. /path/to/k8s.snap),
 ##   or the path to a folder containing the local snap files to be installed (e.g. /path/to)
 
+# Function to retry snap installation with a maximum number of attempts
+# and a delay between attempts. This is useful in case of transient errors
+retry_snap_install() {
+  local max_attempts=5
+  local delay=3
+  local attempt=1
+
+  while [ $attempt -le $max_attempts ]; do
+    echo "Attempt $attempt of $max_attempts to install snap..."
+    if "$@"; then
+      echo "Snap installation succeeded"
+      return 0
+    else
+      echo "Snap installation failed"
+      if [ $attempt -lt $max_attempts ]; then
+        echo "Retrying in $delay seconds..."
+        sleep $delay
+      fi
+    fi
+    attempt=$((attempt + 1))
+  done
+
+  echo "Failed to install snap after $max_attempts attempts"
+  return 1
+}
+
 if [ -f "/capi/etc/snap-channel" ]; then
   snap_channel="$(cat /capi/etc/snap-channel)"
-  snap install k8s --classic --channel "${snap_channel}"
+  retry_snap_install snap install k8s --classic --channel "${snap_channel}"
 elif [ -f "/capi/etc/snap-revision" ]; then
   snap_revision="$(cat /capi/etc/snap-revision)"
-  snap install k8s --classic --revision "${snap_revision}"
+  retry_snap_install snap install k8s --classic --revision "${snap_revision}"
 elif [ -f "/capi/etc/snap-local-path" ]; then
   snap_local_path="$(cat /capi/etc/snap-local-path)"
   snap_local_paths=( "${snap_local_path}" )
@@ -20,7 +46,7 @@ elif [ -f "/capi/etc/snap-local-path" ]; then
   if [[ -d "${snap_local_path}" ]]; then
     snap_local_paths=($(ls ${snap_local_path}/*.snap))
   fi
-  snap install --classic --dangerous "${snap_local_paths[@]}"
+  retry_snap_install snap install --classic --dangerous "${snap_local_paths[@]}"
 else
   echo "No snap installation option found"
   exit 1

pkg/cloudinit/scripts/join-cluster.sh

@@ -10,4 +10,5 @@ name="$(cat /capi/etc/node-name)"
 config_file="/capi/etc/config.yaml"
 token="$(cat /capi/etc/join-token)"
 
+# /snap/bin/k8s join-cluster "${token}" --name "${name}" --address "${address}" --file "${config_file}"
 k8s join-cluster "${token}" --name "${name}" --address "${address}" --file "${config_file}"

pkg/cloudinit/scripts/wait-apiserver-ready.sh

@@ -3,6 +3,7 @@
 ## Assumptions:
 ## - k8s is installed and bootstrapped
 
+# while ! /snap/bin/k8s kubectl get --raw /readyz; do
 while ! k8s kubectl get --raw /readyz; do
   echo "kube-apiserver not yet ready"
   sleep 1

pkg/cloudinit/worker_join.go

@@ -43,6 +43,7 @@ func NewJoinWorker(input JoinWorkerInput) (CloudConfig, error) {
 	// run commands
 	config.RunCommands = append(config.RunCommands, input.PreRunCommands...)
 	if !input.AirGapped {
+		// config.RunCommands = append(config.RunCommands, "/capi/scripts/ensure-snapd.sh", "/capi/scripts/install.sh")
 		config.RunCommands = append(config.RunCommands, "/capi/scripts/install.sh")
 	}
 	config.RunCommands = append(config.RunCommands,

pkg/cloudinit/worker_join_test.go

@@ -47,6 +47,7 @@ func TestNewJoinWorker(t *testing.T) {
 		"/capi/scripts/configure-snapstore-proxy.sh",
 		"prerun1",
 		"prerun2",
+		// "/capi/scripts/ensure-snapd.sh",
 		"/capi/scripts/install.sh",
 		"/capi/scripts/disable-host-services.sh",
 		"/capi/scripts/load-images.sh",
@@ -59,6 +60,7 @@ func TestNewJoinWorker(t *testing.T) {
 
 	// NOTE (mateoflorido): Keep this test in sync with the expected paths in the worker_join.go file.
 	g.Expect(config.WriteFiles).To(ConsistOf(
+		// HaveField("Path", "/capi/scripts/ensure-snapd.sh"),
 		HaveField("Path", "/capi/scripts/install.sh"),
 		HaveField("Path", "/capi/scripts/disable-host-services.sh"),
 		HaveField("Path", "/capi/scripts/bootstrap.sh"),
@@ -185,6 +187,7 @@ func TestNewJoinWorkerAirGapped(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	// Verify the run commands is missing install.sh script.
+	// g.Expect(config.RunCommands).NotTo(ContainElement("/capi/scripts/ensure-snapd.sh"))
 	g.Expect(config.RunCommands).NotTo(ContainElement("/capi/scripts/install.sh"))
 }