Enhance SandboxFleet to support Kata runtime

- Updated README to include instructions for building and deploying with Kata runtime.
- Added e2e workflow for testing Kata runtime, including setup for KVM device permissions.
- Modified CRI runtime configuration to support host devices, allowing for nested virtualization.
- Updated API definitions and validation rules to include host devices in SandboxPool.
- Enhanced StatefulSetBuilder to mount host devices in Worker Pods.
- Added tests to validate the integration of Kata runtime and host device mounting functionality.

Author: FAUST-BENCHOU

.github/workflows/e2e.yaml

@@ -43,3 +43,70 @@ jobs:
 
       - name: Run e2e tests
         run: ./hack/verify-e2e.sh
+
+  e2e-kata:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          df -h /
+
+      - name: Enable KVM device permissions
+        id: kvm
+        run: |
+          if [[ ! -e /dev/kvm ]]; then
+            echo "available=false" >> "$GITHUB_OUTPUT"
+            echo "/dev/kvm missing; skipping nested-virt e2e"
+            exit 0
+          fi
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          if ! docker run --rm --device /dev/kvm busybox true; then
+            echo "available=false" >> "$GITHUB_OUTPUT"
+            echo "/dev/kvm not usable from Docker; skipping"
+            exit 0
+          fi
+          echo "available=true" >> "$GITHUB_OUTPUT"
+
+      - name: Install kind
+        if: steps.kvm.outputs.available == 'true'
+        run: |
+          curl -fsSL -o ./kind "https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64"
+          chmod +x ./kind
+          sudo mv ./kind /usr/local/bin/kind
+          kind version
+
+      - name: Install kubectl
+        if: steps.kvm.outputs.available == 'true'
+        run: |
+          curl -fsSL -o ./kubectl "https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x ./kubectl
+          sudo mv ./kubectl /usr/local/bin/kubectl
+          kubectl version --client
+
+      - name: Deploy SandboxFleet (kata)
+        if: steps.kvm.outputs.available == 'true'
+        env:
+          WORKER_RUNTIME: kata
+          APPLY_SAMPLES: "0"
+          CLUSTER_NAME: sandboxfleet-kata
+          ENSURE_NESTED_VIRT: "1"
+        run: ./hack/deploy-kind.sh
+
+      - name: Run e2e tests (kata)
+        if: steps.kvm.outputs.available == 'true'
+        env:
+          CLUSTER_NAME: sandboxfleet-kata
+        run: ./hack/verify-e2e.sh

README.md

@@ -42,10 +42,12 @@ Optional cleanup:
 - `deploy-kind.sh` writes kubeconfig to `bin/KUBECONFIG` and runtime selection to
   `bin/runtime.env` (used by `verify-e2e.sh`).
 - `WORKER_RUNTIME` selects which Worker image to build and load (`gvisor` default,
-  or `runc`). Example: `WORKER_RUNTIME=runc ./hack/deploy-kind.sh` builds only the
-  base image and sets `runtimeHandler=runc`.
+  `runc`, or `kata`). It only picks image + `runtimeHandler` (+ optional sample
+  `hostDevices`). Nested virt is handled by `hack/ensure-kind-cluster.sh` via
+  `ENSURE_NESTED_VIRT=auto|1|0` (independent of runtime name).
+  Kata pools declare `spec.runtime.cri.hostDevices: ["/dev/kvm"]`.
 - `APPLY_SAMPLES=1` (default) also applies demo Pool/Sandbox manifests; e2e uses
   its own namespace and does not depend on those samples.
 - `APPLY_SAMPLES=0 ./hack/deploy-kind.sh` installs only the control plane.
 - Re-run tests later without rebuilding: `./hack/verify-e2e.sh`
-- Build Worker images alone: `./hack/build-worker-images.sh runc|gvisor|all`
+- Build Worker images alone: `./hack/build-worker-images.sh runc|gvisor|kata|all`

api/v1alpha1/sandboxpool_types.go

@@ -11,11 +11,21 @@ const RuntimeBackendCRI RuntimeBackend = "cri"
 
 type CRIRuntimeConfig struct {
 	// RuntimeHandler is the CRI runtime handler name configured in the Worker's
-	// containerd. It is an opaque string (for example "runsc" or "runc");
+	// containerd. It is an opaque string (for example "runsc", "runc", or "kata");
 	// SandboxFleet does not interpret runtime-specific values.
 	// +kubebuilder:validation:MinLength=1
 	// +kubebuilder:validation:MaxLength=253
 	RuntimeHandler string `json:"runtimeHandler"`
+
+	// HostDevices lists host paths to mount into every Worker Pod for this Pool
+	// (for example "/dev/kvm"). Runtime-agnostic: the controller mounts whatever
+	// is declared here and does not special-case handler names.
+	// +optional
+	// +listType=set
+	// +kubebuilder:validation:MaxItems=16
+	// +kubebuilder:validation:items:MinLength=1
+	// +kubebuilder:validation:items:MaxLength=256
+	HostDevices []string `json:"hostDevices,omitempty"`
 }
 
 // +kubebuilder:validation:XValidation:rule="self.backend != 'cri' || has(self.cri)",message="cri configuration is required for the cri backend"

api/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1.7
+
+# Kata Worker image: generic CRI base + kata-containers (Cloud Hypervisor).
+# Control-plane code stays runtime-agnostic; only this image knows about Kata.
+
+ARG BASE_IMAGE=sandboxfleet-worker-base:latest
+ARG KATA_VERSION=4.0.0
+
+FROM ${BASE_IMAGE}
+ARG TARGETARCH
+ARG KATA_VERSION
+
+RUN apt-get update \
+	&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+		zstd \
+	&& rm -rf /var/lib/apt/lists/*
+
+RUN case "${TARGETARCH}" in \
+		amd64) ARCH=amd64 ;; \
+		arm64) ARCH=arm64 ;; \
+		*) echo "unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \
+	esac \
+	&& curl -fsSL -o /tmp/kata-static.tar.zst \
+		"https://github.com/kata-containers/kata-containers/releases/download/${KATA_VERSION}/kata-static-${KATA_VERSION}-${ARCH}.tar.zst" \
+	&& mkdir -p /tmp/kata-extract \
+	&& tar --zstd -xf /tmp/kata-static.tar.zst -C /tmp/kata-extract \
+	&& cp -a /tmp/kata-extract/opt/kata /opt/kata \
+	&& ln -sf /opt/kata/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2 \
+	&& ln -sf /opt/kata/bin/kata-runtime /usr/local/bin/kata-runtime \
+	&& test -x /opt/kata/bin/cloud-hypervisor \
+	&& test -f /opt/kata/share/defaults/kata-containers/configuration-clh.toml \
+	&& rm -rf /tmp/kata-static.tar.zst /tmp/kata-extract
+
+COPY build/runtimes/kata/containerd.toml /etc/containerd/config.toml

build/runtimes/kata/Dockerfile

@@ -0,0 +1,30 @@
+# Local containerd config for the Kata Worker image (Cloud Hypervisor VMM).
+# Registers opaque CRI handler name "kata" used by SandboxPool.spec.runtime.cri.runtimeHandler.
+
+version = 2
+root = "/var/lib/containerd"
+state = "/run/containerd"
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+
+[plugins."io.containerd.grpc.v1.cri"]
+  sandbox_image = "registry.k8s.io/pause:3.10"
+
+  [plugins."io.containerd.grpc.v1.cri".cni]
+    bin_dir = "/opt/cni/bin"
+    conf_dir = "/etc/cni/net.d"
+
+  [plugins."io.containerd.grpc.v1.cri".containerd]
+    default_runtime_name = "runc"
+
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+      runtime_type = "io.containerd.runc.v2"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+        SystemdCgroup = false
+
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
+      runtime_type = "io.containerd.kata.v2"
+      privileged_without_host_devices = true
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
+        ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration-clh.toml"

build/runtimes/kata/containerd.toml

@@ -64,6 +64,15 @@ spec:
                           type: string
                           minLength: 1
                           maxLength: 253
+                        hostDevices:
+                          type: array
+                          maxItems: 16
+                          uniqueItems: true
+                          items:
+                            type: string
+                            minLength: 1
+                            maxLength: 256
+                          description: Host paths mounted into every Worker Pod (for example /dev/kvm).
                   x-kubernetes-validations:
                     - rule: "self.backend != 'cri' || has(self.cri)"
                       message: cri configuration is required for the cri backend

config/crd/sandboxpools.yaml

@@ -0,0 +1,29 @@
+# Sample Pool for Kata Worker image (WORKER_RUNTIME=kata).
+# Requires /dev/kvm on the node (see hack/ensure-kind-cluster.sh).
+apiVersion: sandboxfleet.io/v1alpha1
+kind: SandboxPool
+metadata:
+  name: demo
+  namespace: default
+spec:
+  runtime:
+    backend: cri
+    cri:
+      runtimeHandler: kata
+      hostDevices:
+        - /dev/kvm
+  slotProfiles:
+    - name: default
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 100m
+          memory: 128Mi
+  workerTemplates:
+    - name: mixed
+      replicas: 1
+      slots:
+        - profile: default
+          count: 2

config/samples/sandboxpool-kata.yaml

@@ -0,0 +1,26 @@
+# Sample Pool for runc Worker image (WORKER_RUNTIME=runc).
+apiVersion: sandboxfleet.io/v1alpha1
+kind: SandboxPool
+metadata:
+  name: demo
+  namespace: default
+spec:
+  runtime:
+    backend: cri
+    cri:
+      runtimeHandler: runc
+  slotProfiles:
+    - name: default
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 100m
+          memory: 128Mi
+  workerTemplates:
+    - name: mixed
+      replicas: 1
+      slots:
+        - profile: default
+          count: 2

config/samples/sandboxpool-runc.yaml

@@ -1,3 +1,5 @@
+# Default sample Pool (gVisor / runsc).
+# Other runtimes: sandboxpool-runc.yaml, sandboxpool-kata.yaml
 apiVersion: sandboxfleet.io/v1alpha1
 kind: SandboxPool
 metadata: