Initial phase 1-3 of a per-node Kata config delivery

Closes: #77

Signed-off-by: Erick Bourgeois <erick@jeb.ca>

Author: ebourgeois

.claude/CHANGELOG.md

@@ -41,6 +41,12 @@ AVD-KSV-0048
 # which is impractical for a cluster-wide controller.
 AVD-KSV-0041
 
+# NOTE: KSV-0049 (create/update/patch/delete on 'configmaps') is intentionally
+# NOT suppressed. The controller ClusterRole grants `configmaps: ["get"]` only
+# (read-only kata source-object probe, ADR 0002); the reclaim-agent ConfigMap
+# projection writes ride the resolved child-cluster kubeconfig identity, not the
+# controller token. There is no configmap-write finding to suppress.
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Deployment — deploy/deployment/deployment.yaml
 # ─────────────────────────────────────────────────────────────────────────────
@@ -190,6 +196,61 @@ AVD-KSV-0116
 # armed.
 AVD-KSV-0022
 
+# ─────────────────────────────────────────────────────────────────────────────
+# DaemonSet — deploy/kata-config-agent/daemonset.yaml
+# ─────────────────────────────────────────────────────────────────────────────
+#
+# The kata-config-agent shares most of its hardened-but-rooted posture with the
+# reclaim agent, so the hostPath (KSV-0023/0121), hostPID (KSV-0010), and
+# root/runAsNonRoot/runAsGroup findings (KSV-0012/0118/0105/0020/0021/0116) are
+# already suppressed above by the same global rule IDs. The two additional
+# findings are privileged (KSV-0017) and its capability corollary (KSV-0106):
+
+# KSV-0017 — Container 'agent' should not run as privileged.
+# Architecturally required (ADR 0003). The agent's job is to write a Kata
+# containerd drop-in to the HOST filesystem and bounce the host k0s service via
+# `nsenter -t 1 -m -u -i -n -p -- systemctl restart <restartService>` so
+# containerd reloads it. setns() into the host mount/PID namespaces requires
+# `privileged: true` (hostPID alone is insufficient for the mount-namespace
+# entry). The escalation is bounded by: opt-in nodeSelector
+# (5spot.finos.org/kata-config=enabled — lands only on nodes the controller
+# labelled, which only happens when an SM with spec.kata resolves a source on
+# this cluster), readOnlyRootFilesystem, seccomp RuntimeDefault, a single-purpose
+# binary with no shell, and a narrow node-scoped ServiceAccount (nodes get/patch
+# on its own Node + read-only configmaps/secrets in its target namespace).
+AVD-KSV-0017
+
+# KSV-0106 — Container 'agent' should drop all capabilities (and only add
+# NET_BIND_SERVICE). Moot under privileged: true (KSV-0017 above, ADR 0003): a
+# privileged container holds the full Linux capability set regardless of any
+# securityContext.capabilities.drop list, so `capabilities.drop: [ALL]` would be
+# a no-op rather than a real hardening — adding it would misrepresent the
+# container's effective privileges in an audit. The agent needs no
+# NET_BIND_SERVICE (it binds no privileged ports; it writes the host drop-in and
+# bounces the k0s service via nsenter). The escalation is bounded by the same
+# controls justified for KSV-0017: opt-in nodeSelector, readOnlyRootFilesystem,
+# seccomp RuntimeDefault, a shell-less single-purpose binary, and a narrow
+# node-scoped ServiceAccount.
+AVD-KSV-0106
+
+# ─────────────────────────────────────────────────────────────────────────────
+# RBAC — deploy/kata-config-agent/rbac.yaml
+# ─────────────────────────────────────────────────────────────────────────────
+
+# KSV-0113 — Role '5spot-kata-config-agent' shouldn't view/manage secrets in
+# namespace '5spot-system'.
+# Access is READ-ONLY and narrowed to `get` (no list/watch, no create/update/
+# patch/delete) — the "manage" wording is misleading; the rule fires on any
+# namespace-scoped secret read. The grant is architecturally required: a Kata
+# drop-in source object may be a Secret (spec.kata referencing a Secret, ADR
+# 0002), and the agent resolves it with a direct GET by the exact name carried
+# in its Node's kata-config-ref annotation — it never enumerates. Same posture
+# as the controller's KSV-0041 read-only secrets suppression. Tightening
+# further (e.g. dropping `get`) would remove the Secret-backed drop-in source
+# capability entirely; scope is already minimal (single namespace, get-only,
+# opt-in nodeSelector).
+AVD-KSV-0113
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Dockerfile — Dockerfile, Dockerfile.chainguard
 # ─────────────────────────────────────────────────────────────────────────────

.trivyignore

@@ -68,6 +68,9 @@ lazy_static = "1.5"
 # enabled explicitly.
 warp = { version = "0.4", default-features = false, features = ["server"] }
 toml = "1.1"
+# SHA-256 for the kata-config agent's content/drift hashing and (Phase 4)
+# the applied-hash node annotation. RustCrypto, pure-Rust, no system deps.
+sha2 = "0.10"
 
 # Linux-only — used by the reclaim-agent's rung 2 netlink proc connector
 # subscriber (`src/netlink_proc.rs`). The portable byte parsers in that
@@ -81,6 +84,10 @@ nix = { version = "0.31", default-features = false, features = ["socket", "net",
 name = "5spot-reclaim-agent"
 path = "src/bin/reclaim_agent.rs"
 
+[[bin]]
+name = "5spot-kata-config-agent"
+path = "src/bin/kata_config_agent.rs"
+
 [dev-dependencies]
 tokio-test = "0.4"
 mockall = "0.14"

Cargo.lock

@@ -134,6 +134,89 @@ spec:
                 - kind
                 - spec
                 type: object
+              kata:
+                description: |-
+                  Optional reference to a `Secret` or `ConfigMap` on the **workload
+                  cluster** holding a Kata containerd drop-in to deliver to the node(s) this
+                  resource owns.
+
+                  When set, the controller resolves the referenced object on the workload
+                  cluster (via the `kubeconfig-<clusterName>` Secret) in `kata.namespace`
+                  (default `5spot-system`). If it is present, the controller stamps the
+                  `5spot.finos.org/kata-config=enabled` opt-in label **and** a reference
+                  annotation on the backing Node; the `5spot-kata-config-agent` DaemonSet —
+                  scheduled onto labelled nodes — reads the object from the workload API,
+                  writes the drop-in to `destPath`, and restarts `restartService` so
+                  containerd reloads it. If the object (or its namespace) is absent, the
+                  controller does **not** label the Node and reports a fail-fast status
+                  condition — 5-Spot never creates the object (it must pre-exist,
+                  Flux-delivered).
+
+                  This is config *delivery*, not a Kata install: the `/opt/kata` binaries
+                  remain `kata-deploy`'s responsibility, and the existing
+                  `katacontainers.io/kata-runtime` opt-in label is unaffected. See ADR 0002
+                  and ADR 0003.
+                nullable: true
+                properties:
+                  destPath:
+                    default: /etc/k0s/container.d/kata-containers.toml
+                    description: |-
+                      Absolute host path the agent writes the drop-in to. Defaults to
+                      `/etc/k0s/container.d/kata-containers.toml` (the k0s containerd drop-in
+                      directory). Configurable because layouts vary (k0s-in-docker, kairos,
+                      vanilla containerd).
+                    maxLength: 4096
+                    minLength: 1
+                    pattern: ^/[^\u0000]*$
+                    type: string
+                  key:
+                    default: kata-containers.toml
+                    description: |-
+                      Key within the source's `data` map whose value is the drop-in content.
+                      Defaults to `kata-containers.toml`.
+                    maxLength: 253
+                    minLength: 1
+                    pattern: ^[A-Za-z0-9._-]+$
+                    type: string
+                  kind:
+                    description: Whether the source is a `ConfigMap` or a `Secret`. Required (no default).
+                    enum:
+                    - ConfigMap
+                    - Secret
+                    type: string
+                  name:
+                    description: |-
+                      Name of the source object on the workload cluster. RFC-1123 DNS subdomain
+                      (max 253 chars).
+                    maxLength: 253
+                    minLength: 1
+                    pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                    type: string
+                  namespace:
+                    default: 5spot-system
+                    description: |-
+                      Workload-cluster namespace the agent reads the object from. Defaults to
+                      `5spot-system` (the agent's own namespace, so no cross-namespace agent
+                      RBAC is needed). Override to place config in a per-tenant namespace.
+                    maxLength: 63
+                    minLength: 1
+                    pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                    type: string
+                  restartService:
+                    default: k0sworker.service
+                    description: |-
+                      systemd unit the node agent restarts (via `nsenter`) so containerd
+                      reloads the drop-in. Defaults to `k0sworker.service`; override with
+                      `k0scontroller.service` on single-node / controller-runs-workloads
+                      layouts.
+                    maxLength: 255
+                    minLength: 1
+                    pattern: ^[A-Za-z0-9@._-]+\.service$
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
               killIfCommands:
                 description: |-
                   Optional list of process patterns that trigger an emergency node

Cargo.toml

@@ -61,11 +61,43 @@ rules:
     resources: ["events"]
     verbs: ["create", "patch"]
 
-  # Secrets for SSH keys and bootstrap data
+  # Secrets for SSH keys, bootstrap data, and child-cluster kubeconfigs.
+  #
+  # Read-only. Also covers reading a `kata` source Secret on the workload
+  # cluster in the degenerate co-located case (ADR 0002); in true multi-cluster
+  # that read rides the kubeconfig identity. The controller never writes
+  # Secrets, so no create/patch/delete here.
   - apiGroups: [""]
     resources: ["secrets"]
     verbs: ["get", "list", "watch"]
 
+  # ConfigMaps — read-only.
+  #
+  # `get` is the kata source-object existence probe (ADR 0002): the controller
+  # checks that the referenced source ConfigMap exists, then stamps a Node label
+  # + reference annotation. The node-side agent — not the controller — reads the
+  # source and writes the host drop-in, so the controller never writes a kata
+  # ConfigMap.
+  #
+  # The reclaim-agent per-node ConfigMap projection (`reclaim-agent-<node>`) does
+  # write ConfigMaps, but on the workload cluster via the resolved
+  # `kubeconfig-<clusterName>` child-cluster identity — not this controller
+  # token. This matches the posture that shipped with reclaim: before the kata
+  # branch this ClusterRole had no `configmaps` rule at all. Keeping write verbs
+  # off the controller ServiceAccount preserves least privilege and avoids the
+  # Trivy KSV-0049 finding entirely (no suppression needed).
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+
+  # Namespaces — read-only. The `kata` fail-fast probe distinguishes a missing
+  # source object from a missing target namespace (ADR 0002). In the degenerate
+  # co-located case this uses the controller SA; in multi-cluster it rides the
+  # kubeconfig identity.
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get"]
+
   # Nodes for drain operations and reclaim-agent label / taint management.
   #
   # `update` was removed in Phase 5 of the 2026-04-25 security audit

deploy/crds/scheduledmachine.yaml

@@ -0,0 +1,113 @@
+# 5-Spot Kata Config Agent DaemonSet (workload cluster)
+#
+# Opt-in node-side delivery of a Kata containerd drop-in. Lands only on nodes
+# the controller has labelled `5spot.finos.org/kata-config=enabled` (which only
+# happens when an SM with `spec.kata` resolves its source object on this cluster).
+#
+# The agent reads its Node's `5spot.finos.org/kata-config-ref` annotation, GETs
+# the referenced ConfigMap/Secret from the workload API (NOT a mounted volume — a
+# DaemonSet cannot template a configMap.name per replica, ADR 0002 §3), and
+# atomically writes the drop-in to the configured destPath under the hostPath
+# /host mount.
+#
+# The pod is `privileged: true` + `hostPID: true` (ADR 0003): writing the host
+# drop-in and bouncing the host k0s service via `nsenter -t 1 -m -u -i -n -p --
+# systemctl restart <restartService>` require setns into the host namespaces and
+# resolving host PID 1. `privileged` is the architecturally-required escalation,
+# bounded by opt-in label-gating (lands only on nodes the controller labelled),
+# a read-only root filesystem, and seccomp RuntimeDefault. Justification is in
+# `.trivyignore` (kata-config-agent banner) and docs/src/security/threat-model.md.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: 5spot-kata-config-agent
+  namespace: 5spot-system
+  labels:
+    app: 5spot-kata-config-agent
+    app.kubernetes.io/name: 5spot
+    app.kubernetes.io/component: kata-config-agent
+spec:
+  selector:
+    matchLabels:
+      app: 5spot-kata-config-agent
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 10%
+  template:
+    metadata:
+      labels:
+        app: 5spot-kata-config-agent
+        app.kubernetes.io/name: 5spot
+        app.kubernetes.io/component: kata-config-agent
+    spec:
+      serviceAccountName: 5spot-kata-config-agent
+      # Opt-in: only land on nodes the controller has labelled.
+      nodeSelector:
+        5spot.finos.org/kata-config: enabled
+      # Reach cordoned / NotReady nodes too — delivery is infra-level.
+      tolerations:
+        - operator: Exists
+      priorityClassName: system-node-critical
+      terminationGracePeriodSeconds: 5
+      # hostPID so `nsenter -t 1` resolves host PID 1 (systemd) for the k0s
+      # service restart (ADR 0003).
+      hostPID: true
+      securityContext:
+        # Root is required to write the drop-in under /etc/k0s on the host.
+        # nosemgrep: yaml.kubernetes.security.run-as-non-root-unsafe-value.run-as-non-root-unsafe-value
+        runAsNonRoot: false
+        runAsUser: 0
+        runAsGroup: 0
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: agent
+          # Pin a released tag — never :latest in production (KSV-0013).
+          image: ghcr.io/finos/5-spot-kata-config-agent:v0.1.0
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: RUST_LOG
+              value: "info"
+            - name: RUST_LOG_FORMAT
+              value: "json"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOST_ROOT
+              value: "/host"
+          securityContext:
+            # privileged is required for setns into host mount/PID namespaces
+            # (the `nsenter` k0s-service restart, ADR 0003). With privileged,
+            # allowPrivilegeEscalation must be true; the root filesystem stays
+            # read-only and seccomp stays RuntimeDefault to keep the escalation
+            # as narrow as the feature allows.
+            privileged: true
+            runAsUser: 0
+            runAsGroup: 0
+            # nosemgrep: yaml.kubernetes.security.run-as-non-root-unsafe-value.run-as-non-root-unsafe-value
+            runAsNonRoot: false
+            allowPrivilegeEscalation: true
+            readOnlyRootFilesystem: true
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            limits:
+              cpu: 50m
+              memory: 64Mi
+            requests:
+              cpu: 10m
+              memory: 16Mi
+          volumeMounts:
+            # Host root mounted writable so the agent can write the configurable
+            # destPath (default /etc/k0s/container.d/). The dest path is
+            # operator-configurable per SM, so a narrower subPath cannot be
+            # pinned in a shared DaemonSet spec (ADR 0003).
+            - name: host-root
+              mountPath: /host
+      volumes:
+        - name: host-root
+          hostPath:
+            path: /
+            type: Directory