From 8d5caaacb3fdee0f2233a9966634ebf7499b03b7 Mon Sep 17 00:00:00 2001 From: Erick Bourgeois Date: Tue, 9 Jun 2026 07:55:29 -0400 Subject: [PATCH] Initial phase 1-3 of a per-node Kata config delivery Closes: #77 Signed-off-by: Erick Bourgeois --- .claude/CHANGELOG.md | 366 ++++++++++++++++++ .trivyignore | 61 +++ Cargo.lock | 1 + Cargo.toml | 7 + deploy/crds/scheduledmachine.yaml | 83 ++++ deploy/deployment/rbac/clusterrole.yaml | 34 +- deploy/kata-config-agent/daemonset.yaml | 113 ++++++ deploy/kata-config-agent/rbac.yaml | 93 +++++ ...0002-kata-config-delivery-via-spec-kata.md | 199 ++++++++++ ...in-pod-host-service-restart-via-nsenter.md | 162 ++++++++ docs/adr/README.md | 2 + docs/architecture/calm/architecture.json | 191 ++++++++- docs/src/reference/api.md | 33 ++ src/bin/crddoc.rs | 33 ++ src/bin/kata_config_agent.rs | 270 +++++++++++++ src/constants.rs | 43 ++ src/crd.rs | 178 +++++++++ src/crd_tests.rs | 251 ++++++++++++ src/kata_config_agent.rs | 259 +++++++++++++ src/kata_config_agent_tests.rs | 301 ++++++++++++++ src/lib.rs | 1 + src/reconcilers/child_client_tests.rs | 1 + src/reconcilers/helpers.rs | 240 ++++++++++++ src/reconcilers/helpers_tests.rs | 243 ++++++++++++ src/reconcilers/scheduled_machine.rs | 74 ++++ src/reconcilers/scheduled_machine_tests.rs | 1 + tests/integration_child_kubeconfig.rs | 1 + 27 files changed, 3237 insertions(+), 4 deletions(-) create mode 100644 deploy/kata-config-agent/daemonset.yaml create mode 100644 deploy/kata-config-agent/rbac.yaml create mode 100644 docs/adr/0002-kata-config-delivery-via-spec-kata.md create mode 100644 docs/adr/0003-in-pod-host-service-restart-via-nsenter.md create mode 100644 src/bin/kata_config_agent.rs create mode 100644 src/kata_config_agent.rs create mode 100644 src/kata_config_agent_tests.rs diff --git a/.claude/CHANGELOG.md b/.claude/CHANGELOG.md index ba4151c..820fa61 100644 --- a/.claude/CHANGELOG.md +++ b/.claude/CHANGELOG.md @@ -9,6 +9,372 @@ The format is based on the regulated environment requirements: --- +## [2026-06-09 18:45] - Tighten kata-config-agent secret RBAC to get-only (KSV-0113) + +**Author:** Erick Bourgeois + +### Changed +- `deploy/kata-config-agent/rbac.yaml`: narrow the namespaced Role's + `configmaps`/`secrets` verbs from `["get", "list", "watch"]` to `["get"]`. + The agent resolves a single source object by exact name (the ref carried in + its Node's `kata-config-ref` annotation) via a direct GET — it never + enumerates and runs no Kubernetes informer (the drift-watch is a node-local + file poll), so `list`/`watch` were dead grants. Updated the file's RBAC + banner accordingly. +- `.trivyignore`: add `AVD-KSV-0113` for the now get-only secrets read. The + finding still fires on any namespace-scoped secret read; the access is + architecturally required because a Kata drop-in source may be a Secret + (`spec.kata` → Secret, ADR 0002). Justification mirrors the controller's + read-only `KSV-0041` suppression. + +### Why +GitHub Advanced Security / Trivy flagged KSV-0113 (MEDIUM) on the agent Role. +The grant is read-only — never "manage" — but was broader than needed. +Tightening to `get`-only caps a stolen token's blast radius to objects the +attacker can already name, rather than the whole `5spot-system` namespace, then +the irreducible read-only finding is suppressed with justification. + +### Impact +- [x] Config change only +- [ ] Breaking change +- [ ] Requires cluster rollout + +## [2026-06-09 18:30] - Suppress KSV-0106 for the privileged kata-config-agent + +**Author:** Erick Bourgeois + +### Changed +- `.trivyignore`: add `AVD-KSV-0106` to the kata-config-agent block. The rule + ("drop all capabilities, only add NET_BIND_SERVICE") is moot under + `privileged: true` (KSV-0017, ADR 0003) — a privileged container holds the + full capability set regardless of any `capabilities.drop` list, so the drop + would be a no-op rather than real hardening. The agent needs no + NET_BIND_SERVICE. Justification mirrors the existing KSV-0017 entry. + +### Why +Trivy KSV-0106 (LOW) fired on `deploy/kata-config-agent/daemonset.yaml`. Adding +`capabilities.drop: [ALL]` would satisfy the scanner but misrepresent the +container's effective privileges in an audit, so the finding is suppressed with +a written justification per the `.trivyignore` policy. + +### Impact +- [x] Config change only +- [x] Documentation only + +## [2026-06-09 18:00] - Kata tear-down handshake + privileged agent DaemonSet + +**Author:** Erick Bourgeois + +### Changed +- `src/reconcilers/helpers.rs`: `reconcile_kata_config_delivery` tear-down now + clears **only** the reference annotation and leaves the opt-in label in place + (so the agent is not descheduled before it can clean up the host file). +- `src/bin/kata_config_agent.rs`: the agent records the applied `destPath` in a + `5spot.finos.org/kata-config-applied` Node annotation on write; on tear-down + (ref annotation gone) it unlinks the recorded host file, then removes its own + opt-in label to deschedule (`read_node_state` / `record_applied` / `clear_optin`). +- `src/constants.rs`: add `KATA_CONFIG_APPLIED_ANNOTATION`. +- `src/reconcilers/helpers_tests.rs`: tear-down tests assert the label is left + untouched and only the annotation is cleared. +- `deploy/kata-config-agent/daemonset.yaml`: `privileged: true` + `hostPID: true` + (ADR 0003 — required for the `nsenter` k0s-service restart). +- `deploy/kata-config-agent/rbac.yaml`: agent ClusterRole `nodes: get` → `get, patch` + (records applied annotation + removes its own label). +- `.trivyignore`: kata-config-agent banner; add `AVD-KSV-0017` (privileged) + with architectural justification (other findings share the reclaim agent's + already-suppressed global rule IDs). + +### Why +Closes two gaps: (1) the tear-down race where label-removal descheduled the agent +before it could unlink the host drop-in — now the agent owns its own +descheduling after cleanup; (2) the DaemonSet must be privileged to `nsenter` +into host PID 1 and bounce k0s so containerd reloads the drop-in (ADR 0003). + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [ ] Config change only +- [x] Documentation only (agent/manifests not yet released) + +## [2026-06-09 17:15] - Kata agent: API-read by Node annotation + manifests + +**Author:** Erick Bourgeois + +### Changed +- `src/kata_config_agent.rs`: add `KataRef` + `parse_kata_ref` (parse the + `5spot.finos.org/kata-config-ref` Node annotation) and `sync_content` + (reconcile the host file from in-memory content); `sync_once` now delegates to + `sync_content`. +- `src/bin/kata_config_agent.rs`: rewrite from mounted-file (`SOURCE_PATH`) to + **API-read** — each tick reads the Node annotation, `get`s the referenced + workload `ConfigMap`/`Secret`, extracts the key, and syncs to `HOST_ROOT`+`destPath`. + No ConfigMap volume. +- `src/kata_config_agent_tests.rs`: tests for `parse_kata_ref` (full/missing/non-JSON), + `sync_content` (write/delete/noop), and `host_path`. 561 lib tests + bin tests pass. +- `deploy/kata-config-agent/rbac.yaml`: ServiceAccount + namespaced Role + (`configmaps`/`secrets` get/list/watch in `5spot-system`) + ClusterRole + (`nodes: get` for the own-Node annotation read). No API writes. +- `deploy/kata-config-agent/daemonset.yaml`: opt-in DaemonSet (nodeSelector + `5spot.finos.org/kata-config=enabled`), `NODE_NAME` downward API, hostPath `/` + at `/host`, no ConfigMap volume. Phase-3 posture (root, caps dropped, RO rootfs, + no privileged/hostPID); Phase 4 adds privileged+hostPID+`nsenter` per ADR 0003. + +### Why +Completes ADR 0002 §3: a cluster-wide DaemonSet can't template a `configMap.name` +volume per replica, so the agent reads its source object from the workload API +keyed by the controller-stamped Node annotation. + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [ ] Config change only +- [x] Documentation only — new node-agent manifests + agent binary (not yet released) + +## [2026-06-09 16:30] - Implement spec.kata workload-read delivery (ADR 0002) + +**Author:** Erick Bourgeois + +### Changed +- `src/crd.rs`: rename `spec.kataConfigRef`/`KataConfigRef` → `spec.kata`/`KataConfig`; + add `namespace` field (workload-cluster target, default `5spot-system`) with + schema + default; docs reflect workload-cluster resolution. +- `src/reconcilers/helpers.rs`: replace the per-node ConfigMap projection + (`reconcile_kata_config_provision`, `build_kata_config_configmap`, + `per_node_kata_configmap_name`, `extract_*`, `resolve_kata_config_content`) + with `reconcile_kata_config_delivery` (read-only workload `get_opt` of the + source object + namespace probe → Node opt-in label/annotation, or fail-fast + `SourceNotFound`/`TargetNamespaceMissing`), `build_kata_config_ref_annotation_patch`, + and the `KataDeliveryOutcome` enum. No controller ConfigMap/Secret writes. +- `src/reconcilers/scheduled_machine.rs`: call site reads on the workload client, + logs the fail-fast outcome. +- `src/constants.rs`: add `KATA_CONFIG_REF_ANNOTATION`; drop the + projection-only `KATA_CONFIG_CONFIGMAP_PREFIX` / `KATA_CONFIG_DATA_KEY` / + `KATA_CONFIG_FIELD_MANAGER`. +- `src/crd_tests.rs`, `src/reconcilers/helpers_tests.rs`: migrate tests to the + new contract (namespace default/override, annotation builders, mock-API + delivery tests for present / source-missing / namespace-missing / disabled); + remove obsolete projection tests. 555 lib tests pass. +- `src/bin/crddoc.rs`: regenerate API docs text for `spec.kata`. +- `deploy/crds/scheduledmachine.yaml`, `docs/src/reference/api.md`: regenerated. +- `deploy/deployment/rbac/clusterrole.yaml`: kata is read-only (`configmaps: get` + + new `namespaces: get`); the `configmaps` write verbs are re-attributed to the + reclaim-agent projection (the honest KSV-0049 owner). `.trivyignore`: re-add + `AVD-KSV-0049` attributed to reclaim. +- `docs/architecture/calm/architecture.json`: kata flow updated to read-not-project; + `make calm-validate` clean; diagrams regenerated. +- `docs/adr/0002-kata-config-delivery-via-spec-kata.md`: §5/Consequences corrected + — kata adds no write privilege but does NOT eliminate KSV-0049 (reclaim still + holds it). + +### Why +Implements the approved ADR 0002 redesign: 5-Spot resolves the Flux-delivered +kata object on the workload cluster and opts the Node in (or fails fast with a +status reason), instead of projecting per-node ConfigMaps. Removes duplication, +the controller-owned sync burden, and kata's contribution to the ConfigMap write +grant. + +### Impact +- [x] Breaking change (CRD field `spec.kataConfigRef` → `spec.kata`; unreleased) +- [ ] Requires cluster rollout +- [ ] Config change only +- [ ] Documentation only + +## [2026-06-09 15:00] - Redesign kata config delivery (ADR 0002): workload-read, no writes + +**Author:** Erick Bourgeois + +### Changed +- `docs/adr/0002-kata-config-delivery-via-spec-kata.md`: rewritten (and renamed + from `0002-per-node-kata-config-delivery.md`) to the accepted design — replace + `spec.kataConfigRef` with `spec.kata`; the controller **resolves** the kata + object on the workload cluster (via `kubeconfig-`) and either opts + the Node in (label + reference annotation) or **fails fast** with a status + reason (`SourceNotFound` / `TargetNamespaceMissing`). 5-Spot performs **no** + ConfigMap/Secret writes and **no** namespace creation for kata delivery; the + config must pre-exist (Flux-delivered). Target namespace defaults to + `5spot-system`, overridable. KSV-0049 eliminated by construction. +- `docs/adr/0003-in-pod-host-service-restart-via-nsenter.md`: aligned with the + above — controller reads (not projects); agent consumes the object via the + workload kube API keyed by Node annotation (no ConfigMap volume); `spec.kata`. +- `docs/adr/README.md`: index reflects the consolidated set (0001–0003). +- Removed the two intermediate, never-committed-to-main ADRs (the per-node + `spec.kataConfigRef` projection and its KSV-0049 ownership/RBAC defense) and + reverted this session's per-node documentation (concept page, `.trivyignore` + `AVD-KSV-0049`, threat-model rows, `clusterrole.yaml` / `helpers.rs` comments). + +### Why +Review against the platform model (tenants create nothing; Flux delivers config; +a tenant namespace backs 1+ k0smotron control planes; the agent runs on the +workload cluster the controller already has kubeconfig access to) rejected the +per-node ConfigMap projection: wrong granularity, controller-owned duplication, a +sync burden 5-Spot shouldn't own, and an unnecessary ConfigMap write grant. The +accepted design makes 5-Spot a read-only resolver + opt-in labeler that fails +fast when config is absent. + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [ ] Config change only +- [x] Documentation only (ADR/design; implementation tracked separately) + +## [2026-06-09 01:00] - Kata config delivery Phase 3 — node agent host-sync engine + +**Author:** Erick Bourgeois + +### Changed +- `src/kata_config_agent.rs`: new host-filesystem sync engine — `sha256_hex`, + `file_sha256`, `SyncAction` + `decide_action` (pure decision), `atomic_write` + (temp + `rename`, mode 0644, parent-dir create, crash-safe), `remove_if_present`, + and `sync_once` → `SyncOutcome` (Wrote/Deleted/Unchanged). Drift self-heals. +- `src/kata_config_agent_tests.rs`: 22 unit tests (SHA-256 vectors, decision + matrix, atomic write incl. mode/parent-dirs/no-temp-leak/crash-safety, + idempotent unlink, end-to-end sync incl. drift correction + GitOps delete). +- `src/bin/kata_config_agent.rs`: new `5spot-kata-config-agent` binary — clap + CLI (`SOURCE_PATH`/`DEST_PATH`/`NODE_NAME`/`POLL_INTERVAL_SECS`/`--oneshot`) + driving a drift-watch poll loop over `sync_once`. +- `src/lib.rs`: export `kata_config_agent` module. +- `Cargo.toml`: add `sha2` dependency (already in the graph transitively) and + the `5spot-kata-config-agent` bin target. + +### Why +Phase 3 of ADR 0003 — land the drop-in on the host and self-heal drift. The +applied-hash node annotation and the `nsenter` host k0s-service restart are +Phase 4; the DaemonSet manifest, agent RBAC, `.trivyignore`, and metrics land +in Phase 4 with the final (privileged) security posture so they are decided as +one unit. The drift-watch is a deliberate node-local poll — out-of-band host +edits raise no Kubernetes event to watch on. + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [ ] Config change only +- [ ] Documentation only + +> New binary only; not yet deployed (no DaemonSet until Phase 4). No controller +> or CRD behavior change. Verified end-to-end via `--oneshot`: write → no-op → +> delete, parent dirs created, no temp-file leakage. + +--- + +## [2026-06-09 00:00] - Kata config delivery Phase 2 — controller projection + RBAC + +**Author:** Erick Bourgeois + +### Changed +- `src/constants.rs`: new kata-config constants — `KATA_CONFIG_LABEL` + (`5spot.finos.org/kata-config`), `KATA_CONFIG_LABEL_ENABLED`, + `KATA_CONFIG_NAMESPACE`, `KATA_CONFIG_CONFIGMAP_PREFIX`, + `KATA_CONFIG_DATA_KEY`, `KATA_CONFIG_FIELD_MANAGER` + (`5spot-controller-kata-config`). +- `src/reconcilers/helpers.rs`: kata projection — pure builders + `per_node_kata_configmap_name`, `build_kata_config_configmap`, + `build_kata_config_label_patch`; content extractors + `extract_kata_content_from_configmap` / `_from_secret`; async + `resolve_kata_config_content` and `reconcile_kata_config_provision` + (resolve source → stamp label → SSA per-node ConfigMap; tear down on + `None`, 404-on-delete benign). Mirrors the reclaim-agent projection. +- `src/reconcilers/helpers_tests.rs`: 18 new tests — pure builders/extractors + (happy/missing-key/non-UTF-8) plus five mock-client orchestrator tests + (ConfigMap source, Secret source w/ custom key, tear-down, 404-benign, + source-404 short-circuit). +- `src/reconcilers/scheduled_machine.rs`: `provision_kata_config_best_effort` + wired into `handle_active_phase` beside reclaim/taints — child-cluster-scoped + writes, management-cluster source read, non-fatal. +- `deploy/deployment/rbac/clusterrole.yaml`: new `configmaps` + get/create/patch/delete rule (first such rule); secrets rule comment notes + kataConfigRef source reads. + +### Why +Phase 2 of ADR 0002 — project the referenced kata drop-in into a per-node +ConfigMap on the child cluster and stamp the opt-in label, so the Phase 3 node +agent can consume it. Implements the `rel-controller-kata-config-projection` +relationship already modeled in CALM. + +### Impact +- [ ] Breaking change +- [x] Requires cluster rollout +- [ ] Config change only +- [ ] Documentation only + +> Apply the updated controller ClusterRole (new `configmaps` rule) before +> rolling out the controller, or kata-config projection will fail with RBAC +> Forbidden. Behavior is inert for ScheduledMachines without `kataConfigRef`. + +--- + +## [2026-06-08 01:00] - Kata config delivery Phase 1 — CRD `spec.kataConfigRef` + +**Author:** Erick Bourgeois + +### Changed +- `src/crd.rs`: new `kata_config_ref: Option` field on + `ScheduledMachineSpec`; new `KataConfigRef` struct (kind/name/key/destPath/ + restartService, `deny_unknown_fields`, in-namespace only) and + `KataConfigSourceKind` enum (`ConfigMap`/`Secret`); four field schema + functions and three default functions. +- `src/crd_tests.rs`: 12 new TDD tests (serde defaults, round-trip, PascalCase + kind, rejects invalid kind / unknown fields, required kind+name, optional in + schema, name/destPath/restartService schema bounds). Existing spec literals + updated with `kata_config_ref: None`. +- `src/reconcilers/{scheduled_machine_tests,helpers_tests,child_client_tests}.rs`, + `tests/integration_child_kubeconfig.rs`: spec literals updated with + `kata_config_ref: None`. +- `src/bin/crddoc.rs`: documents the new `kataConfigRef` field + example. +- `deploy/crds/scheduledmachine.yaml`, `docs/src/reference/api.md`: regenerated + (`make crds` → `make crddoc`). + +### Why +Phase 1 of ADR 0002 — establish the `spec.kataConfigRef` CRD contract (types, +schema bounds, serde defaults) test-first before the controller projection +(Phase 2) and node agent (Phases 3–4). No behavioral/controller code yet. + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [x] Config change only +- [ ] Documentation only + +> CRD schema change (additive, optional field). Apply the regenerated +> `deploy/crds/scheduledmachine.yaml` to pick up `kataConfigRef`; existing +> ScheduledMachines are unaffected (field is optional, omitted when unset). + +--- + +## [2026-06-08 00:00] - ADR + CALM: per-node Kata config delivery (design only) + +**Author:** Erick Bourgeois + +### Changed +- `docs/adr/0002-per-node-kata-config-delivery.md`: new ADR (Proposed) — the + `spec.kataConfigRef` contract and controller-side per-node ConfigMap + projection, mirroring the reclaim-agent projection; in-namespace source only + (like `KubeconfigSecretRef`). +- `docs/adr/0003-in-pod-host-service-restart-via-nsenter.md`: new ADR (Proposed) + — the privileged `5spot-kata-config-agent` DaemonSet that writes the host + drop-in and restarts the host k0s service via `nsenter -t 1 … systemctl + restart`, with the applied-hash annotation as the restart-loop guard. +- `docs/adr/README.md`: index updated with ADR-0002 and ADR-0003. +- `docs/architecture/calm/architecture.json`: modeled the feature — new nodes + `service-kata-config-agent` and `data-asset-kata-config-configmap`; new + relationships `rel-controller-kata-config-projection`, + `rel-kata-agent-workload-kube-api`, `rel-kata-agent-writes-host`, + `rel-kata-configmap-stored-in-workload-api`; new flow + `flow-kata-config-delivery`; `adrs` linked. `make calm-validate` passes; + `make calm-diagrams` renders. + +### Why +ADD requires the decision to be recorded and modeled before code. This entry +covers the ADR + CALM deliverables only; no Rust, CRD, or manifest changes yet — +implementation (TDD) is gated on review of these two ADRs. + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [ ] Config change only +- [x] Documentation only + +--- + ## [2026-05-31 00:00] - Adopt Architecture Driven Development (ADD) methodology **Author:** Erick Bourgeois diff --git a/.trivyignore b/.trivyignore index 271a745..81054ff 100644 --- a/.trivyignore +++ b/.trivyignore @@ -41,6 +41,12 @@ AVD-KSV-0048 # which is impractical for a cluster-wide controller. AVD-KSV-0041 +# NOTE: KSV-0049 (create/update/patch/delete on 'configmaps') is intentionally +# NOT suppressed. The controller ClusterRole grants `configmaps: ["get"]` only +# (read-only kata source-object probe, ADR 0002); the reclaim-agent ConfigMap +# projection writes ride the resolved child-cluster kubeconfig identity, not the +# controller token. There is no configmap-write finding to suppress. + # ───────────────────────────────────────────────────────────────────────────── # Deployment — deploy/deployment/deployment.yaml # ───────────────────────────────────────────────────────────────────────────── @@ -190,6 +196,61 @@ AVD-KSV-0116 # armed. AVD-KSV-0022 +# ───────────────────────────────────────────────────────────────────────────── +# DaemonSet — deploy/kata-config-agent/daemonset.yaml +# ───────────────────────────────────────────────────────────────────────────── +# +# The kata-config-agent shares most of its hardened-but-rooted posture with the +# reclaim agent, so the hostPath (KSV-0023/0121), hostPID (KSV-0010), and +# root/runAsNonRoot/runAsGroup findings (KSV-0012/0118/0105/0020/0021/0116) are +# already suppressed above by the same global rule IDs. The two additional +# findings are privileged (KSV-0017) and its capability corollary (KSV-0106): + +# KSV-0017 — Container 'agent' should not run as privileged. +# Architecturally required (ADR 0003). The agent's job is to write a Kata +# containerd drop-in to the HOST filesystem and bounce the host k0s service via +# `nsenter -t 1 -m -u -i -n -p -- systemctl restart ` so +# containerd reloads it. setns() into the host mount/PID namespaces requires +# `privileged: true` (hostPID alone is insufficient for the mount-namespace +# entry). The escalation is bounded by: opt-in nodeSelector +# (5spot.finos.org/kata-config=enabled — lands only on nodes the controller +# labelled, which only happens when an SM with spec.kata resolves a source on +# this cluster), readOnlyRootFilesystem, seccomp RuntimeDefault, a single-purpose +# binary with no shell, and a narrow node-scoped ServiceAccount (nodes get/patch +# on its own Node + read-only configmaps/secrets in its target namespace). +AVD-KSV-0017 + +# KSV-0106 — Container 'agent' should drop all capabilities (and only add +# NET_BIND_SERVICE). Moot under privileged: true (KSV-0017 above, ADR 0003): a +# privileged container holds the full Linux capability set regardless of any +# securityContext.capabilities.drop list, so `capabilities.drop: [ALL]` would be +# a no-op rather than a real hardening — adding it would misrepresent the +# container's effective privileges in an audit. The agent needs no +# NET_BIND_SERVICE (it binds no privileged ports; it writes the host drop-in and +# bounces the k0s service via nsenter). The escalation is bounded by the same +# controls justified for KSV-0017: opt-in nodeSelector, readOnlyRootFilesystem, +# seccomp RuntimeDefault, a shell-less single-purpose binary, and a narrow +# node-scoped ServiceAccount. +AVD-KSV-0106 + +# ───────────────────────────────────────────────────────────────────────────── +# RBAC — deploy/kata-config-agent/rbac.yaml +# ───────────────────────────────────────────────────────────────────────────── + +# KSV-0113 — Role '5spot-kata-config-agent' shouldn't view/manage secrets in +# namespace '5spot-system'. +# Access is READ-ONLY and narrowed to `get` (no list/watch, no create/update/ +# patch/delete) — the "manage" wording is misleading; the rule fires on any +# namespace-scoped secret read. The grant is architecturally required: a Kata +# drop-in source object may be a Secret (spec.kata referencing a Secret, ADR +# 0002), and the agent resolves it with a direct GET by the exact name carried +# in its Node's kata-config-ref annotation — it never enumerates. Same posture +# as the controller's KSV-0041 read-only secrets suppression. Tightening +# further (e.g. dropping `get`) would remove the Secret-backed drop-in source +# capability entirely; scope is already minimal (single namespace, get-only, +# opt-in nodeSelector). +AVD-KSV-0113 + # ───────────────────────────────────────────────────────────────────────────── # Dockerfile — Dockerfile, Dockerfile.chainguard # ───────────────────────────────────────────────────────────────────────────── diff --git a/Cargo.lock b/Cargo.lock index d85e5c6..bd3370b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -566,6 +566,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", "tempfile", "test-log", "thiserror 2.0.18", diff --git a/Cargo.toml b/Cargo.toml index 6cc0c1e..6258c68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,9 @@ lazy_static = "1.5" # enabled explicitly. warp = { version = "0.4", default-features = false, features = ["server"] } toml = "1.1" +# SHA-256 for the kata-config agent's content/drift hashing and (Phase 4) +# the applied-hash node annotation. RustCrypto, pure-Rust, no system deps. +sha2 = "0.10" # Linux-only — used by the reclaim-agent's rung 2 netlink proc connector # subscriber (`src/netlink_proc.rs`). The portable byte parsers in that @@ -81,6 +84,10 @@ nix = { version = "0.31", default-features = false, features = ["socket", "net", name = "5spot-reclaim-agent" path = "src/bin/reclaim_agent.rs" +[[bin]] +name = "5spot-kata-config-agent" +path = "src/bin/kata_config_agent.rs" + [dev-dependencies] tokio-test = "0.4" mockall = "0.14" diff --git a/deploy/crds/scheduledmachine.yaml b/deploy/crds/scheduledmachine.yaml index 9698d5e..6a52418 100644 --- a/deploy/crds/scheduledmachine.yaml +++ b/deploy/crds/scheduledmachine.yaml @@ -134,6 +134,89 @@ spec: - kind - spec type: object + kata: + description: |- + Optional reference to a `Secret` or `ConfigMap` on the **workload + cluster** holding a Kata containerd drop-in to deliver to the node(s) this + resource owns. + + When set, the controller resolves the referenced object on the workload + cluster (via the `kubeconfig-` Secret) in `kata.namespace` + (default `5spot-system`). If it is present, the controller stamps the + `5spot.finos.org/kata-config=enabled` opt-in label **and** a reference + annotation on the backing Node; the `5spot-kata-config-agent` DaemonSet — + scheduled onto labelled nodes — reads the object from the workload API, + writes the drop-in to `destPath`, and restarts `restartService` so + containerd reloads it. If the object (or its namespace) is absent, the + controller does **not** label the Node and reports a fail-fast status + condition — 5-Spot never creates the object (it must pre-exist, + Flux-delivered). + + This is config *delivery*, not a Kata install: the `/opt/kata` binaries + remain `kata-deploy`'s responsibility, and the existing + `katacontainers.io/kata-runtime` opt-in label is unaffected. See ADR 0002 + and ADR 0003. + nullable: true + properties: + destPath: + default: /etc/k0s/container.d/kata-containers.toml + description: |- + Absolute host path the agent writes the drop-in to. Defaults to + `/etc/k0s/container.d/kata-containers.toml` (the k0s containerd drop-in + directory). Configurable because layouts vary (k0s-in-docker, kairos, + vanilla containerd). + maxLength: 4096 + minLength: 1 + pattern: ^/[^\u0000]*$ + type: string + key: + default: kata-containers.toml + description: |- + Key within the source's `data` map whose value is the drop-in content. + Defaults to `kata-containers.toml`. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z0-9._-]+$ + type: string + kind: + description: Whether the source is a `ConfigMap` or a `Secret`. Required (no default). + enum: + - ConfigMap + - Secret + type: string + name: + description: |- + Name of the source object on the workload cluster. RFC-1123 DNS subdomain + (max 253 chars). + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + namespace: + default: 5spot-system + description: |- + Workload-cluster namespace the agent reads the object from. Defaults to + `5spot-system` (the agent's own namespace, so no cross-namespace agent + RBAC is needed). Override to place config in a per-tenant namespace. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + restartService: + default: k0sworker.service + description: |- + systemd unit the node agent restarts (via `nsenter`) so containerd + reloads the drop-in. Defaults to `k0sworker.service`; override with + `k0scontroller.service` on single-node / controller-runs-workloads + layouts. + maxLength: 255 + minLength: 1 + pattern: ^[A-Za-z0-9@._-]+\.service$ + type: string + required: + - kind + - name + type: object killIfCommands: description: |- Optional list of process patterns that trigger an emergency node diff --git a/deploy/deployment/rbac/clusterrole.yaml b/deploy/deployment/rbac/clusterrole.yaml index 2c6419c..ab42490 100644 --- a/deploy/deployment/rbac/clusterrole.yaml +++ b/deploy/deployment/rbac/clusterrole.yaml @@ -61,11 +61,43 @@ rules: resources: ["events"] verbs: ["create", "patch"] - # Secrets for SSH keys and bootstrap data + # Secrets for SSH keys, bootstrap data, and child-cluster kubeconfigs. + # + # Read-only. Also covers reading a `kata` source Secret on the workload + # cluster in the degenerate co-located case (ADR 0002); in true multi-cluster + # that read rides the kubeconfig identity. The controller never writes + # Secrets, so no create/patch/delete here. - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch"] + # ConfigMaps — read-only. + # + # `get` is the kata source-object existence probe (ADR 0002): the controller + # checks that the referenced source ConfigMap exists, then stamps a Node label + # + reference annotation. The node-side agent — not the controller — reads the + # source and writes the host drop-in, so the controller never writes a kata + # ConfigMap. + # + # The reclaim-agent per-node ConfigMap projection (`reclaim-agent-`) does + # write ConfigMaps, but on the workload cluster via the resolved + # `kubeconfig-` child-cluster identity — not this controller + # token. This matches the posture that shipped with reclaim: before the kata + # branch this ClusterRole had no `configmaps` rule at all. Keeping write verbs + # off the controller ServiceAccount preserves least privilege and avoids the + # Trivy KSV-0049 finding entirely (no suppression needed). + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + + # Namespaces — read-only. The `kata` fail-fast probe distinguishes a missing + # source object from a missing target namespace (ADR 0002). In the degenerate + # co-located case this uses the controller SA; in multi-cluster it rides the + # kubeconfig identity. + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get"] + # Nodes for drain operations and reclaim-agent label / taint management. # # `update` was removed in Phase 5 of the 2026-04-25 security audit diff --git a/deploy/kata-config-agent/daemonset.yaml b/deploy/kata-config-agent/daemonset.yaml new file mode 100644 index 0000000..6f7652c --- /dev/null +++ b/deploy/kata-config-agent/daemonset.yaml @@ -0,0 +1,113 @@ +# 5-Spot Kata Config Agent DaemonSet (workload cluster) +# +# Opt-in node-side delivery of a Kata containerd drop-in. Lands only on nodes +# the controller has labelled `5spot.finos.org/kata-config=enabled` (which only +# happens when an SM with `spec.kata` resolves its source object on this cluster). +# +# The agent reads its Node's `5spot.finos.org/kata-config-ref` annotation, GETs +# the referenced ConfigMap/Secret from the workload API (NOT a mounted volume — a +# DaemonSet cannot template a configMap.name per replica, ADR 0002 §3), and +# atomically writes the drop-in to the configured destPath under the hostPath +# /host mount. +# +# The pod is `privileged: true` + `hostPID: true` (ADR 0003): writing the host +# drop-in and bouncing the host k0s service via `nsenter -t 1 -m -u -i -n -p -- +# systemctl restart ` require setns into the host namespaces and +# resolving host PID 1. `privileged` is the architecturally-required escalation, +# bounded by opt-in label-gating (lands only on nodes the controller labelled), +# a read-only root filesystem, and seccomp RuntimeDefault. Justification is in +# `.trivyignore` (kata-config-agent banner) and docs/src/security/threat-model.md. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: 5spot-kata-config-agent + namespace: 5spot-system + labels: + app: 5spot-kata-config-agent + app.kubernetes.io/name: 5spot + app.kubernetes.io/component: kata-config-agent +spec: + selector: + matchLabels: + app: 5spot-kata-config-agent + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 10% + template: + metadata: + labels: + app: 5spot-kata-config-agent + app.kubernetes.io/name: 5spot + app.kubernetes.io/component: kata-config-agent + spec: + serviceAccountName: 5spot-kata-config-agent + # Opt-in: only land on nodes the controller has labelled. + nodeSelector: + 5spot.finos.org/kata-config: enabled + # Reach cordoned / NotReady nodes too — delivery is infra-level. + tolerations: + - operator: Exists + priorityClassName: system-node-critical + terminationGracePeriodSeconds: 5 + # hostPID so `nsenter -t 1` resolves host PID 1 (systemd) for the k0s + # service restart (ADR 0003). + hostPID: true + securityContext: + # Root is required to write the drop-in under /etc/k0s on the host. + # nosemgrep: yaml.kubernetes.security.run-as-non-root-unsafe-value.run-as-non-root-unsafe-value + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + containers: + - name: agent + # Pin a released tag — never :latest in production (KSV-0013). + image: ghcr.io/finos/5-spot-kata-config-agent:v0.1.0 + imagePullPolicy: IfNotPresent + env: + - name: RUST_LOG + value: "info" + - name: RUST_LOG_FORMAT + value: "json" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HOST_ROOT + value: "/host" + securityContext: + # privileged is required for setns into host mount/PID namespaces + # (the `nsenter` k0s-service restart, ADR 0003). With privileged, + # allowPrivilegeEscalation must be true; the root filesystem stays + # read-only and seccomp stays RuntimeDefault to keep the escalation + # as narrow as the feature allows. + privileged: true + runAsUser: 0 + runAsGroup: 0 + # nosemgrep: yaml.kubernetes.security.run-as-non-root-unsafe-value.run-as-non-root-unsafe-value + runAsNonRoot: false + allowPrivilegeEscalation: true + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + resources: + limits: + cpu: 50m + memory: 64Mi + requests: + cpu: 10m + memory: 16Mi + volumeMounts: + # Host root mounted writable so the agent can write the configurable + # destPath (default /etc/k0s/container.d/). The dest path is + # operator-configurable per SM, so a narrower subPath cannot be + # pinned in a shared DaemonSet spec (ADR 0003). + - name: host-root + mountPath: /host + volumes: + - name: host-root + hostPath: + path: / + type: Directory diff --git a/deploy/kata-config-agent/rbac.yaml b/deploy/kata-config-agent/rbac.yaml new file mode 100644 index 0000000..0ec3734 --- /dev/null +++ b/deploy/kata-config-agent/rbac.yaml @@ -0,0 +1,93 @@ +# 5-Spot Kata Config Agent — ServiceAccount + RBAC (workload cluster) +# +# The agent runs on the workload cluster and needs only: +# - get + patch its own Node object: get reads the +# 5spot.finos.org/kata-config-ref annotation the controller stamps; patch +# records the 5spot.finos.org/kata-config-applied annotation (the dest it +# manages) and, on tear-down, removes the opt-in label so the DaemonSet +# deschedules (the cleanup handshake, ADR 0002). Cluster-scoped (Node is +# cluster-scoped); no list/watch — the agent only ever touches the single +# Node named by its downward-API NODE_NAME. +# - get ConfigMaps and Secrets in its target namespace (the +# spec.kata.namespace, default 5spot-system) to resolve the drop-in content. +# `get` only — it resolves one object by exact name (no list/watch). +# +# It writes no ConfigMaps/Secrets; its only API writes are the merge-patch on its +# own Node's kata annotations + label. +# +# NOTE: if an SM sets spec.kata.namespace to something other than 5spot-system, +# the agent needs read access there too — add a Role/RoleBinding per such +# namespace, or widen to a read-only ClusterRole on configmaps/secrets. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: 5spot-kata-config-agent + namespace: 5spot-system + labels: + app.kubernetes.io/name: 5spot + app.kubernetes.io/component: kata-config-agent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: 5spot-kata-config-agent + labels: + app.kubernetes.io/name: 5spot + app.kubernetes.io/component: kata-config-agent +rules: + # get the own Node (read kata-config-ref); patch it (record kata-config-applied, + # remove the opt-in label on tear-down). No list/watch. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: 5spot-kata-config-agent + labels: + app.kubernetes.io/name: 5spot + app.kubernetes.io/component: kata-config-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 5spot-kata-config-agent +subjects: + - kind: ServiceAccount + name: 5spot-kata-config-agent + namespace: 5spot-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: 5spot-kata-config-agent + namespace: 5spot-system + labels: + app.kubernetes.io/name: 5spot + app.kubernetes.io/component: kata-config-agent +rules: + # Read-only resolution of the source object in the default target namespace. + # `get` only — the agent resolves a single object by exact name (the ref from + # its Node annotation) via a direct GET; it never enumerates, so it needs + # neither `list` nor `watch`. This caps the blast radius of a stolen token to + # objects the attacker can already name, not the whole namespace. + - apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: 5spot-kata-config-agent + namespace: 5spot-system + labels: + app.kubernetes.io/name: 5spot + app.kubernetes.io/component: kata-config-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: 5spot-kata-config-agent +subjects: + - kind: ServiceAccount + name: 5spot-kata-config-agent + namespace: 5spot-system diff --git a/docs/adr/0002-kata-config-delivery-via-spec-kata.md b/docs/adr/0002-kata-config-delivery-via-spec-kata.md new file mode 100644 index 0000000..f84143e --- /dev/null +++ b/docs/adr/0002-kata-config-delivery-via-spec-kata.md @@ -0,0 +1,199 @@ + +# 0002 — Kata config delivery via `spec.kata`: workload-cluster resolution, fail-fast on absence + +- **Status:** Accepted +- **Date:** 2026-06-09 +- **Deciders:** Erick Bourgeois +- **Supersedes:** — +- **Related:** ADR [0003](./0003-in-pod-host-service-restart-via-nsenter.md) (the + node-side host-write + `nsenter` restart half of this feature); + `child_clients.resolve` (`src/reconcilers/child_client.rs`); + `KubeconfigSecretRef` (`src/crd.rs`); the reclaim-agent projection + (`reconcile_reclaim_agent_provision`) as the opt-in-label shape to mirror; + Trivy **KSV-0049**. + +## Context + +5-Spot's target nodes are k0s workers whose containerd runtime consumes drop-in +config from `/etc/k0s/container.d/`. To run Kata-isolated workloads, a specific +kata containerd drop-in (e.g. `/etc/k0s/container.d/kata-containers.toml`) must +land on the worker node(s) a given `ScheduledMachine` owns, reconciled the GitOps +way (present in source ⇒ present on host; removed ⇒ removed). + +**An initial design was rejected.** The first cut had the controller *project a +per-node `ConfigMap`* (`kata-config-`) onto the workload cluster, forking +the source content per node. Reviewed against the platform model, this was wrong: + +- **Per-node fan-out is the wrong granularity** — two SMs pointing at the same + source produce byte-identical copies; cardinality is O(opted-in nodes) with + full duplication. +- **It creates a sync burden 5-Spot shouldn't own** — every source edit forces + re-reconciling all N copies. +- **It mis-assigns ownership** — the config is platform-owned and GitOps-managed; + the controller forking and deleting copies fights that ownership. + +The platform model that drives the accepted design: + +- **Tenants create nothing.** The platform owner manages kata config in Git; + **Flux** reconciles it onto the workload clusters. 5-Spot treats it as a + read-only input, never the author. +- **A tenant namespace backs one *or more* clusters / control planes.** In the + k0smotron topology a namespace (e.g. `foo`) on the management cluster hosts one + or more child control planes, each addressed by a `kubeconfig-` + Secret (CAPI convention). The `ScheduledMachine`'s `clusterName` + that Secret + tell the controller which workload cluster a bound node lives on. +- **The agent runs on the workload cluster**, and the controller **already has + workload-cluster access** via `child_clients.resolve`. The cross-cluster + boundary that an initial design used to justify controller-side projection is a + *non-issue* — the controller can read the workload cluster with an existing + credential. + +**Design principle adopted: 5-Spot writes nothing for kata delivery.** The +delivered artifact is whatever Flux has already placed on the workload cluster. +5-Spot's only job is to *resolve* it, *opt the node in*, and — if it is not there +— **fail fast with a clear status reason**. 5-Spot does **not** copy objects, +does **not** create namespaces, and does **not** manufacture per-node ConfigMaps. +Things must exist for it to work; absence is a reported failure, not something +5-Spot papers over by creating resources. + +## Decision + +### 1. `spec.kata` on `ScheduledMachine` + +`src/crd.rs` is the source of truth; the CRD YAML and API docs are regenerated +(`regen-crds` → `regen-api-docs`), never hand-edited. + +```rust +pub kata: Option, // on ScheduledMachineSpec + +pub struct KataConfig { + pub kind: KataConfigSourceKind, // ConfigMap | Secret (both supported) + pub name: String, // object name on the WORKLOAD cluster + pub key: Option, // default: kata-containers.toml + pub dest_path: Option, // default: /etc/k0s/container.d/kata-containers.toml + pub restart_service: Option, // default: k0sworker.service + pub namespace: Option, // target ns on the WORKLOAD cluster; + // default: 5spot-system (the agent's namespace) +} + +pub enum KataConfigSourceKind { ConfigMap, Secret } +``` + +`KataConfig` keeps the `KubeconfigSecretRef` derive/serde conventions +(`deny_unknown_fields`, camelCase, schemars bounds). There is **no** +`createNamespace` and **no** `namespaceMetadata` — 5-Spot never provisions a +namespace. Both `ConfigMap` and `Secret` sources are supported symmetrically. + +### 2. Resolution: workload-cluster only, fail-fast on absence + +For an SM with `clusterName: cluster-1` and target namespace `T` +(`spec.kata.namespace`, default `5spot-system`): + +1. Resolve the workload client via `kubeconfig-cluster-1` + (`child_clients.resolve`). +2. **Read `T/name`** of the requested `kind` from the **workload** cluster (a + read-only `get`). + - **Present** ⇒ stamp the bound Node (workload cluster) with the opt-in label + **and** a reference annotation carrying `namespace`/`name`/`key`/`destPath`/ + `restartService`; set status `Ready`. + - **Absent** (object missing, or its namespace missing) ⇒ **do not label the + Node**; set a status condition with a precise reason (§4). 5-Spot creates + nothing and waits for Flux to deliver it; the next reconcile re-checks. + +There is exactly one namespace lookup, on the workload cluster. The default +`T = 5spot-system` means the agent reads from **its own namespace**, so no +cross-namespace agent RBAC is needed; an override exists for tenants who place +config elsewhere. + +### 3. Node opt-in + agent consumption + +The controller's only mutation is a Node patch: the opt-in label +(`5spot.finos.org/kata-config=enabled`, gating the agent's `nodeSelector`) plus a +reference annotation pointing at `T/name`. The node-side agent (**ADR 0003**) +reads that annotation, `get`s the object from the workload API, watches it for +live updates, and writes/heals the host file. This replaces any per-node +ConfigMap volume — a cluster-wide DaemonSet cannot template a `configMap.name` +volume per replica, which is exactly why the rejected design needed per-node +names. + +**Tear-down handshake.** Removing the host drop-in cannot be a simple +"controller removes the label," because that would *deschedule the agent before +it could unlink the file*. Instead: + +- The controller, on tear-down, clears **only the reference annotation** and + leaves the opt-in label in place. +- On each successful write the agent records the absolute `destPath` it manages + in a `5spot.finos.org/kata-config-applied` annotation on its own Node. +- When the agent sees the reference annotation gone, it unlinks the recorded host + file (and, Phase 4, bounces k0s), then **removes the opt-in label from its own + Node** — descheduling itself only *after* cleanup completes. + +The agent therefore needs `nodes: get` **and** `patch` (its own Node only). This +is the one place the agent writes to the kube API; it still writes no +ConfigMaps/Secrets. + +### 4. Status conditions (fail-fast, non-fatal to scheduling) + +The `Active`-phase kata step sets a `KataConfigReady`-style condition. Reasons +cover at least: + +- `Ready` — object resolved on the workload cluster; Node opted in. +- `SourceNotFound` — `T/name` absent on the workload cluster. +- `TargetNamespaceMissing` — namespace `T` absent on the workload cluster. + +Failures are **best-effort / non-fatal** to scheduling (a missing kata config +degrades kata delivery but does not break day-to-day machine scheduling), exactly +as the reclaim-agent projection is — but they are **surfaced loudly** in status +so the operator sees *why* nothing landed. + +### 5. RBAC — kata adds no write privilege + +- 5-Spot performs **no writes** for kata delivery beyond the Node label/annotation + patch it already has (`nodes: patch`). No ConfigMap/Secret writes, no namespace + creation. +- Kata's reads — the source-object existence check and the namespace fail-fast + probe — ride the `kubeconfig-` identity in true multi-cluster, and + the controller's own ServiceAccount only in the degenerate co-located case. + Those reads are covered by **read-only** ClusterRole rules: `configmaps: get`, + `secrets: get/list/watch` (already present), and a new `namespaces: get`. +- The **agent's workload-cluster ServiceAccount** gets `get/watch` on + `configmaps` (and `secrets`, for `Secret` sources) in its target namespace + (default `5spot-system`) — a separate manifest, not this ClusterRole. +- **KSV-0049:** the controller `ClusterRole` holds **`configmaps: ["get"]` only** + — no `create/patch/delete`. Kata is read-only, and the **reclaim-agent** + per-node ConfigMap projection writes on the *workload* cluster via the resolved + `kubeconfig-` identity, not the controller token (the posture + reclaim shipped with — this ClusterRole had no `configmaps` rule before the kata + branch). KSV-0049 therefore **does not fire** and no suppression is needed. + *Caveat:* in the degenerate co-located dev/test posture (no kubeconfig Secret, + `child_clients.resolve` falls back to the controller SA), the reclaim ConfigMap + projection 403s — but it is best-effort / non-fatal, so this is an accepted + degradation, not a failure. + +## Consequences + +- **Easier:** no per-node fan-out, no controller-owned duplication, no sync + burden; Flux stays the single source of truth; the controller gains **no** new + write ability anywhere; the controller `ClusterRole` ConfigMap rule is + `get`-only, so KSV-0049 does not fire (see §5). +- **Operator contract:** kata config (and its namespace) **must pre-exist** on the + workload cluster, delivered by Flux, before an SM references it. A missing + object is a fail-fast, clearly-reasoned status condition — never silently + created. This is the intended behaviour, not a limitation. +- **Edge case:** if a single workload cluster is shared by multiple tenants + placing same-named objects in the default `5spot-system`, names can collide; + the `spec.kata.namespace` override is the escape hatch (documented in the + concept page). +- **Ruled out:** per-node ConfigMaps; controller-side copying/projection of + config; controller-created namespaces; controller-SA ConfigMap writes; any + 5-Spot ownership of the delivered object's lifecycle. +- **CALM impact:** **updated.** The kata flow is "controller reads the + workload-cluster object and opts the node in (or fails with status); agent reads + the workload object." Reflect this in `data-asset-kata-config-configmap`, + `rel-controller-kata-config-projection` (now a read + Node label/annotation, not + a write), and `flow-kata-config-delivery`; the agent's read edge points at the + workload-target namespace. `make calm-validate` + `make calm-diagrams` before + implementation. ADR 0003 models the host-write/restart edges. diff --git a/docs/adr/0003-in-pod-host-service-restart-via-nsenter.md b/docs/adr/0003-in-pod-host-service-restart-via-nsenter.md new file mode 100644 index 0000000..b6d59a9 --- /dev/null +++ b/docs/adr/0003-in-pod-host-service-restart-via-nsenter.md @@ -0,0 +1,162 @@ + +# 0003 — In-pod host k0s-service restart via `nsenter` (privileged Kata-config agent) + +- **Status:** Accepted +- **Date:** 2026-06-08 +- **Deciders:** Erick Bourgeois +- **Supersedes:** — +- **Related:** ADR [0002](./0002-kata-config-delivery-via-spec-kata.md) (the `spec.kata` CRD contract + workload-cluster resolution this builds on); the reclaim agent (`src/bin/reclaim_agent.rs`, `deploy/node-agent/daemonset.yaml`) as the DaemonSet shape to mirror; `.trivyignore` (reclaim-agent suppression block); `docs/src/security/threat-model.md`. + +## Context + +ADR 0002 resolves the kata containerd drop-in object on the workload cluster and +stamps the bound Node with an opt-in label plus a reference annotation, but stops +at the kube API. Two steps remain, and neither has any precedent in the codebase: + +1. **Write the drop-in to the host filesystem** at the configured `destPath`. +2. **Make containerd reload it.** On k0s, containerd is managed by the host + `k0sworker.service`; picking up a new `/etc/k0s/container.d/` drop-in requires + restarting that service. Restarting it bounces containerd and therefore every + pod on the node — including the agent pod itself. + +The reclaim agent — the only existing node-side component — is **read-only on the +host** and **write-only to the kube API**. There is **no host-filesystem write, +no host command execution, no `nsenter`, and no executor abstraction** anywhere +in the tree. So this is net-new architecture, not a mirror of reclaim. + +Options weighed for *triggering the host service restart*: + +- **(A) Controller-side restart.** Impossible — the controller is on the + management cluster with no host reach. +- **(B) A Kubernetes `Job` per node, `nodeName`-pinned, that nsenters.** Adds + Job RBAC, per-restart object lifecycle, garbage collection, and a second + moving part to reason about. The agent already runs on exactly the right node + with exactly the right privileges; a Job buys nothing but indirection. +- **(C) A host-side systemd path-unit / drop-in watcher.** Pushes + responsibility onto host provisioning *outside* 5-Spot, breaking the + self-contained "SM declares it, 5-Spot delivers it" model and coupling us to + bespoke node images. +- **(D) In-pod `nsenter -t 1` into host PID 1 (systemd), then `systemctl + restart`.** Chosen. This is precisely the mechanism upstream `kata-deploy` + uses, so it is a known-good pattern for this exact problem. It requires the + agent pod to run `privileged: true` with `hostPID: true`. + +The cost of (D) is real: `privileged: true` is a **strict escalation beyond the +reclaim agent**, which runs `runAsUser: 0` + `hostPID: true` + `NET_ADMIN` but is +*not* privileged. `privileged` is required because `setns()` into the host mount +and other namespaces needs it; `hostPID` is required so `nsenter -t 1` resolves +to the host's systemd rather than the container's PID 1. + +The other hard problem is the **restart loop**: write → restart → containerd +SIGKILLs the agent → kubelet restarts the agent → agent must *not* write+restart +again. The on-disk content hash already converges this (after the restart the +file matches the source, so no second write), but we add belt-and-braces: before +calling `nsenter`, the agent records the content hash it is about to apply on its +own Node as `5spot.finos.org/kata-config-applied=`. On every loop it +compares the source content hash to that annotation and only restarts when the +annotation is missing or stale. This also covers the case where an operator edits +the host file back to old content: the drift path rewrites the file but does +**not** re-restart if the applied hash already matches. + +## Decision + +We ship a dedicated **privileged node DaemonSet**, `5spot-kata-config-agent`, +that owns the host write and the host-service restart end-to-end. The controller +does **not** orchestrate the restart — its job stays narrow (resolve the object +on the workload cluster, stamp the opt-in label + reference annotation, per ADR +0002). + +**Binary + library.** New `src/bin/kata_config_agent.rs` and +`src/kata_config_agent.rs` (I/O-light, unit-testable core), mirroring the +reclaim-agent file layout. A new `5spot-kata-config-agent` `[[bin]]` target in +`Cargo.toml`. + +**Agent loop.** The agent reads its `spec.kata` reference off its own Node +annotation (namespace/name/key/destPath/restartService) and consumes the object +via the **workload kube API** (a `get` + `watch`, not a mounted file — a +cluster-wide DaemonSet cannot template a `configMap.name` volume per replica, per +ADR 0002 §3). + +1. Resolve the source object (`/`, kind ConfigMap or Secret) from + the workload API and extract ``; compute its SHA-256. +2. Compute SHA-256 of `/host` if present. +3. Hashes equal **and** applied-annotation matches → wait for the next watch + event / poll tick (no-op). +4. Source present, hash differs → **atomic write**: temp file in the dest + directory + `rename()`, mode `0644`, owner `root:root`. +5. Source absent (object or key gone) → `unlink` the dest file (404 benign). + GitOps: absent in source ⇒ absent on host. +6. Record `5spot.finos.org/kata-config-applied=` (or `=absent`) on the + agent's own Node via the kubelet node-scoped token, **before** the restart. +7. `nsenter -t 1 -m -u -i -n -p -- systemctl restart `. + Expect to be SIGKILL'd mid-call — systemd still processes the D-Bus job after + the client dies. On pod restart the hashes match → no-op. Single-cycle + convergence. + +The restart invocation is hidden behind a `RestartExecutor` trait so unit tests +assert the constructed `nsenter` command line without ever executing the real +binary; the concrete implementation is exercised only in integration tests. + +**DaemonSet (`deploy/kata-config-agent/`).** `namespace: 5spot-system`; +`nodeSelector: { 5spot.finos.org/kata-config: enabled }` (label-gated opt-in +from ADR 0002); `priorityClassName: system-node-critical`; `tolerations: [{ +operator: Exists }]`. Pod `securityContext`: `runAsUser/Group: 0`, +`runAsNonRoot: false`, `hostPID: true`, `seccompProfile: RuntimeDefault`. +Container `securityContext`: `privileged: true`, `readOnlyRootFilesystem: true`, +`seccompProfile: RuntimeDefault`. Volumes: hostPath `/` mounted at `/host` (the +dest path is user-configurable, so we cannot pin a `subPath`). There is **no** +ConfigMap volume — the source is read from the workload API, keyed by the Node +annotation. Env: `NODE_NAME` (downward API); `destPath` / `restartService` / +`key` are read from the annotation, with the binary defaults as fallback. + +**Applied-hash annotation** `5spot.finos.org/kata-config-applied` is the +restart-loop guard; the existing node `patch` grant already covers it (no new +controller RBAC, no Job RBAC). The agent gets its own workload-cluster +ServiceAccount + narrow Role mirroring the reclaim agent's: `configmaps` (and +`secrets`, for `Secret` sources) `get/watch` in its target namespace (default +`5spot-system`, per `spec.kata.namespace`), plus node-scoped self-patch. + +**Metrics.** `kata_config_writes_total`, `kata_config_deletes_total`, +`kata_config_drift_corrected_total`, `kata_config_last_sync_timestamp_seconds`. +(Note: unlike the reclaim agent, which emits no agent-side metrics, this agent +does — there is no existing agent-side metrics scaffold to copy.) + +**Security justification.** `privileged: true` + `hostPID: true` are documented +in `.trivyignore` under a new `kata-config-agent` banner block following the +reclaim-agent format (rule-ID + written architectural rationale), and called out +in `docs/src/security/threat-model.md`. Mitigations of record: the agent lands +**only** on nodes the controller has label-gated (which only happens when an SM +with `spec.kata` binds that node), `readOnlyRootFilesystem: true`, and a +narrow node-scoped RBAC footprint. + +## Consequences + +- **Easier:** single-cycle, self-healing delivery of kata config to the exact + node — write, restart, converge — owned entirely by one node-resident agent + with no controller-side restart orchestration to coordinate. +- **Harder / riskier:** this is the **first host-filesystem write and first host + command execution** in 5-Spot, and the **first `privileged: true` workload**. + That privilege is a genuine attack-surface increase, justified only by being + opt-in, label-gated, read-only-rootfs, and node-scoped. It must be reviewed as + such in the regulated context and re-validated whenever the agent changes. +- **Open questions to resolve in implementation/e2e (Phase 5 of the roadmap):** + (1) k0s service-name autodetection — `k0sworker.service` vs + `k0scontroller.service`; ship default + `restartService` override, autodetect + as follow-up. (2) First-provision always restarts (no prior config) — expected + but documented so operators aren't surprised by the initial containerd bounce. + (3) Double-restart interaction with `kata-deploy` if both write on the same + node — verify experimentally. (4) `/etc/k0s/container.d/` drop-in path + stability across k0s minor upgrades — `destPath` being configurable insulates + us; pin a validated k0s version range in the guide. +- **Ruled out:** controller-side restart (no host reach), per-node Job + indirection, and host-side systemd watchers that push setup outside 5-Spot. +- **CALM impact:** **updated.** Adds `service-kata-config-agent`'s security + controls (least-privilege, container-hardening noting the privileged + posture), the `rel-kata-agent-writes-host` relationship (hostPath write + + `nsenter` restart against `network-physical-node`) with its host-isolation + control, the `rel-kata-agent-workload-kube-api` relationship (applied-hash + self-annotation), and the host-write/restart transitions of + `flow-kata-config-delivery`. diff --git a/docs/adr/README.md b/docs/adr/README.md index 423db89..547151c 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -37,3 +37,5 @@ refactors) need neither. When unsure, **write the ADR.** | ADR | Title | Status | |----:|-------|--------| | [0001](./0001-adopt-architecture-driven-development.md) | Adopt Architecture Driven Development (ADD) | Accepted | +| [0002](./0002-kata-config-delivery-via-spec-kata.md) | Kata config delivery via `spec.kata` (workload-cluster resolution, fail-fast on absence) | Accepted | +| [0003](./0003-in-pod-host-service-restart-via-nsenter.md) | In-pod host k0s-service restart via `nsenter` (privileged Kata-config agent) | Accepted | diff --git a/docs/architecture/calm/architecture.json b/docs/architecture/calm/architecture.json index f5e8119..a7190ae 100644 --- a/docs/architecture/calm/architecture.json +++ b/docs/architecture/calm/architecture.json @@ -230,6 +230,46 @@ "node-type": "data-asset", "name": "Per-Node Reclaim Agent ConfigMap", "description": "Per-node ConfigMap named reclaim-agent- in namespace 5spot-system on the workload cluster. Controller-projected from ScheduledMachine.spec.killIfCommands via server-side apply with field manager 5spot-controller-reclaim-agent; consumed by the agent via a kube runtime watcher scoped by field selector. Empty killIfCommands deletes the ConfigMap; 404-on-delete is benign by contract." + }, + { + "unique-id": "service-kata-config-agent", + "node-type": "service", + "name": "5-Spot Kata Config Agent", + "description": "Privileged node-side DaemonSet on the workload cluster that delivers a per-node Kata containerd drop-in to the host filesystem and bounces the host k0s service so containerd reloads it. Reads its source ConfigMap/Secret from the workload API (keyed by the 5spot.finos.org/kata-config-ref Node annotation set by the controller per ADR 0002), atomically writes the configured destPath under a hostPath /host mount, then runs `nsenter -t 1 -m -u -i -n -p -- systemctl restart ` to restart the host k0s service. Runs privileged with hostPID=true. Records 5spot.finos.org/kata-config-applied= on its own Node before restarting to break the restart loop. Distinct field manager (5spot-kata-config-agent) attributes every write. Lands only on nodes carrying the 5spot.finos.org/kata-config=enabled opt-in label. See ADR 0003.", + "controls": { + "least-privilege": { + "description": "Agent holds a namespaced Role in 5spot-system granting get,list,watch on ConfigMaps and Secrets (read-only source resolution), plus a ClusterRole granting get,patch on Nodes for its OWN Node only (read the kata-config-ref annotation; record kata-config-applied; remove the opt-in label on tear-down). No ConfigMap/Secret writes; no Job RBAC.", + "requirements": [ + { + "requirement-url": "https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final", + "config": { + "framework": "NIST SP 800-53 Rev. 5", + "controls": ["AC-3", "AC-6"], + "evidence": "deploy/kata-config-agent/rbac.yaml" + } + } + ] + }, + "container-hardening": { + "description": "privileged=true and hostPID=true are unavoidable: setns() into host namespaces and nsenter -t 1 into host PID 1 (systemd) require them to restart the host k0s service. Compensating controls: opt-in only (lands solely on label-gated nodes), readOnlyRootFilesystem=true on the container, seccompProfile RuntimeDefault, and a node-scoped RBAC footprint. The privileged posture and its Trivy suppressions are documented with architectural rationale in .trivyignore and docs/src/security/threat-model.md.", + "requirements": [ + { + "requirement-url": "https://csrc.nist.gov/pubs/sp/800/190/final", + "config": { + "framework": "NIST SP 800-190", + "controls": ["CM-6", "CM-7", "SI-7"], + "evidence": "deploy/kata-config-agent/daemonset.yaml, .trivyignore" + } + } + ] + } + } + }, + { + "unique-id": "data-asset-kata-config-configmap", + "node-type": "data-asset", + "name": "Per-Node Kata Config ConfigMap", + "description": "Source ConfigMap or Secret on the workload cluster, in spec.kata.namespace (default 5spot-system), holding the Kata containerd drop-in. Platform-/Flux-managed — 5-Spot never creates, writes, or deletes it. The controller resolves it read-only (existence check) and the kata config agent reads it via a kube runtime watcher keyed by the 5spot.finos.org/kata-config-ref Node annotation. If absent, delivery fails fast with a status reason and the Node is not opted in (ADR 0002)." } ], @@ -252,14 +292,15 @@ }, { "unique-id": "rel-workload-cluster-contains-node", - "description": "When a schedule is active, the physical node becomes a worker in the CAPI-managed workload cluster. The workload cluster also hosts its own API server and, where ScheduledMachine.spec.killIfCommands is set, the 5-Spot reclaim agent as a DaemonSet.", + "description": "When a schedule is active, the physical node becomes a worker in the CAPI-managed workload cluster. The workload cluster also hosts its own API server and, where ScheduledMachine.spec.killIfCommands is set, the 5-Spot reclaim agent as a DaemonSet; where ScheduledMachine.spec.kata is set, the 5-Spot kata config agent as a DaemonSet.", "relationship-type": { "deployed-in": { "container": "ecosystem-workload-cluster", "nodes": [ "network-physical-node", "service-workload-kubernetes-api", - "service-reclaim-agent" + "service-reclaim-agent", + "service-kata-config-agent" ] } } @@ -475,6 +516,99 @@ "nodes": ["data-asset-reclaim-request-annotation"] } } + }, + { + "unique-id": "rel-controller-kata-config-projection", + "description": "Controller reads (get) the spec.kata source object on the workload cluster (read-only) and, if present, stamps the 5spot.finos.org/kata-config=enabled opt-in label plus the 5spot.finos.org/kata-config-ref annotation onto the backing Node. If the object or its namespace is absent it clears the opt-in and reports a fail-fast status condition. 5-Spot writes no ConfigMap/Secret and creates no namespace.", + "relationship-type": { + "connects": { + "source": { "node": "service-five-spot-controller" }, + "destination": { + "node": "service-workload-kubernetes-api", + "interfaces": ["workload-kube-api-https"] + } + } + }, + "protocol": "HTTPS", + "controls": { + "audit-attribution": { + "description": "Kata delivery performs no object writes: the controller reads the source object on the workload cluster (read-only) and patches only the Node (opt-in label + reference annotation). The agent's read uses its workload-cluster ServiceAccount scoped to spec.kata.namespace (default 5spot-system). See ADR 0002.", + "requirements": [ + { + "requirement-url": "https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final", + "config": { + "framework": "NIST SP 800-53 Rev. 5", + "controls": ["AU-2", "AU-10", "AC-6"], + "evidence": "src/reconcilers/helpers.rs::reconcile_kata_config_delivery" + } + } + ] + } + } + }, + { + "unique-id": "rel-kata-agent-workload-kube-api", + "description": "Kata config agent uses the workload cluster API for: a read-only get/watch of the source ConfigMap/Secret named by its Node's 5spot.finos.org/kata-config-ref annotation, and merge-patches on its OWN Node — recording the 5spot.finos.org/kata-config-applied dest annotation on write, and (on tear-down, when the controller has cleared the ref annotation) unlinking the recorded host file then removing the opt-in label to deschedule itself. Scoped to its own Node by name.", + "relationship-type": { + "connects": { + "source": { "node": "service-kata-config-agent" }, + "destination": { + "node": "service-workload-kubernetes-api", + "interfaces": ["workload-kube-api-https"] + } + } + }, + "protocol": "HTTPS", + "controls": { + "node-scoped-credentials": { + "description": "Agent writes to Node objects only via the kubelet node-scoped token (patch nodes/). ConfigMap reads are gated by a namespaced Role/RoleBinding in 5spot-system granting get,list,watch only. No cluster-wide API access.", + "requirements": [ + { + "requirement-url": "https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final", + "config": { + "framework": "NIST SP 800-53 Rev. 5", + "controls": ["AC-6"], + "evidence": "deploy/kata-config-agent/rbac.yaml" + } + } + ] + } + } + }, + { + "unique-id": "rel-kata-agent-writes-host", + "description": "Kata config agent writes the projected drop-in to the host filesystem at the configured destPath (atomic temp-file + rename under a hostPath /host mount) and restarts the host k0s service via `nsenter -t 1 -m -u -i -n -p -- systemctl restart `, which bounces containerd so it reloads the drop-in. This is the only host-filesystem write and host command execution in 5-Spot.", + "relationship-type": { + "connects": { + "source": { "node": "service-kata-config-agent" }, + "destination": { "node": "network-physical-node" } + } + }, + "controls": { + "host-isolation-boundary": { + "description": "Host write + restart require privileged=true and hostPID=true on the agent pod (setns into host namespaces; nsenter to host PID 1 systemd). The escalation is bounded by opt-in label-gating (the agent lands only on nodes the controller has labelled, which only happens when an SM with kata binds that node), readOnlyRootFilesystem on the container, and atomic, idempotent, hash-gated writes that restart at most once per distinct config change. Documented in .trivyignore and docs/src/security/threat-model.md.", + "requirements": [ + { + "requirement-url": "https://csrc.nist.gov/pubs/sp/800/190/final", + "config": { + "framework": "NIST SP 800-190", + "controls": ["AC-6", "CM-7", "SI-7"], + "evidence": "deploy/kata-config-agent/daemonset.yaml, src/kata_config_agent.rs, .trivyignore" + } + } + ] + } + } + }, + { + "unique-id": "rel-kata-configmap-stored-in-workload-api", + "description": "The per-node kata-config ConfigMap is persisted in the workload cluster API server's etcd, namespace 5spot-system.", + "relationship-type": { + "composed-of": { + "container": "service-workload-kubernetes-api", + "nodes": ["data-asset-kata-config-configmap"] + } + } } ], @@ -626,6 +760,54 @@ ] } } + }, + { + "unique-id": "flow-kata-config-delivery", + "name": "Kata Config Delivery (per-node drop-in + k0s restart)", + "description": "A ScheduledMachine with spec.kata set drives delivery of a Kata containerd drop-in onto its backing node and a one-shot host k0s restart so containerd reloads it. The applied-hash annotation (step 4, written BEFORE the restart in step 5) is the load-bearing restart-loop guard: it ensures the agent restarts at most once per distinct config change even though the restart SIGKILLs the agent. Decisions: ADR 0002 (contract + projection) and ADR 0003 (host write + nsenter restart).", + "transitions": [ + { + "sequence-number": 1, + "relationship-unique-id": "rel-controller-kata-config-projection", + "description": "Controller (Active phase, best-effort, workload-cluster-scoped) reads (get) the source Secret/ConfigMap named in spec.kata in spec.kata.namespace (default 5spot-system) on the workload cluster. If present, it stamps 5spot.finos.org/kata-config=enabled plus the 5spot.finos.org/kata-config-ref annotation on the backing Node; if absent, it clears the opt-in and reports a fail-fast status reason (SourceNotFound / TargetNamespaceMissing)." + }, + { + "sequence-number": 2, + "relationship-unique-id": "rel-kata-agent-workload-kube-api", + "description": "The kata config agent DaemonSet (scheduled onto the node by the opt-in label) watches its per-node ConfigMap and reads the projected drop-in content at /etc/5spot/kata-config/.", + "direction": "destination-to-source" + }, + { + "sequence-number": 3, + "relationship-unique-id": "rel-kata-agent-writes-host", + "description": "Agent compares the SHA-256 of the source content against /host and the kata-config-applied annotation. On mismatch it atomically writes the drop-in (temp-file + rename, mode 0644, root:root) to the host destPath. If the source key is absent it unlinks the host file (GitOps delete)." + }, + { + "sequence-number": 4, + "relationship-unique-id": "rel-kata-agent-workload-kube-api", + "description": "Agent records 5spot.finos.org/kata-config-applied= (or =absent) on its own Node via the kubelet node-scoped token. Load-bearing: written BEFORE the restart so a crash/SIGKILL during restart does not re-trigger on the next loop." + }, + { + "sequence-number": 5, + "relationship-unique-id": "rel-kata-agent-writes-host", + "description": "Agent runs `nsenter -t 1 -m -u -i -n -p -- systemctl restart ` to restart the host k0s service; containerd reloads the new drop-in. The restart SIGKILLs the agent pod; systemd still completes the D-Bus job. Kubelet restarts the agent; on the next loop hashes match the applied annotation, so it is a no-op. Single-cycle convergence." + } + ], + "controls": { + "restart-loop-idempotence": { + "description": "The applied-hash annotation written before the restart (step 4) guarantees the agent restarts the host service at most once per distinct config change, despite the restart killing the agent. Drift correction rewrites the file but does not re-restart when the applied hash already matches.", + "requirements": [ + { + "requirement-url": "https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final", + "config": { + "framework": "NIST SP 800-53 Rev. 5", + "controls": ["SI-7", "CM-6", "AU-2"], + "evidence": "src/kata_config_agent.rs, docs/src/concepts/kata-config-delivery.md" + } + } + ] + } + } } ], @@ -658,5 +840,8 @@ } }, - "adrs": [] + "adrs": [ + "../../adr/0002-kata-config-delivery-via-spec-kata.md", + "../../adr/0003-in-pod-host-service-restart-via-nsenter.md" + ] } diff --git a/docs/src/reference/api.md b/docs/src/reference/api.md index cf476e6..4ecd84e 100644 --- a/docs/src/reference/api.md +++ b/docs/src/reference/api.md @@ -59,6 +59,9 @@ spec: - key: workload value: batch effect: NoSchedule + kataConfigRef: + kind: ConfigMap + name: kata-drop-in ``` ### Spec Fields @@ -188,6 +191,36 @@ Duplicate `(key, effect)` pairs are rejected at admission. Admin-added taints colliding on `(key, effect)` are surfaced as a `TaintOwnershipConflict` condition rather than overwritten. +#### kata + +(optional, KataConfig) Reference to a `Secret` or `ConfigMap` **on the workload +cluster** holding a Kata containerd drop-in to deliver to the node(s) this resource +owns. When set, the controller resolves the object on the workload cluster (via the +`kubeconfig-` Secret) in `kata.namespace` (default `5spot-system`). If +present, it stamps the `5spot.finos.org/kata-config=enabled` opt-in label plus a +reference annotation on the Node; the `5spot-kata-config-agent` DaemonSet reads the +object from the workload API, writes the drop-in to `destPath`, and restarts +`restartService` so containerd reloads it. If the object (or its namespace) is +absent, the controller does NOT label the Node and reports a fail-fast status +condition — 5-Spot never creates the object (it must pre-exist, Flux-delivered). +This is config delivery, not a Kata install — `/opt/kata` binaries remain +`kata-deploy`'s job. See ADR 0002 and ADR 0003. + +`KataConfig` has the following fields: + +- **kind** (required, enum): One of `ConfigMap`, `Secret` — the source kind. +- **name** (required, string): Source object name on the workload cluster, + RFC-1123 DNS subdomain (≤ 253 chars). +- **namespace** (optional, string, default: `5spot-system`): workload-cluster + namespace the agent reads the object from. Override for per-tenant placement. +- **key** (optional, string, default: `kata-containers.toml`): `data` key whose + value is the drop-in content. +- **destPath** (optional, string, default: + `/etc/k0s/container.d/kata-containers.toml`): absolute host path written to. +- **restartService** (optional, string, default: `k0sworker.service`): systemd + unit restarted via `nsenter` so containerd reloads the drop-in. Override with + `k0scontroller.service` on single-node layouts. + ### Status Fields #### phase diff --git a/src/bin/crddoc.rs b/src/bin/crddoc.rs index 5141f1f..222f857 100644 --- a/src/bin/crddoc.rs +++ b/src/bin/crddoc.rs @@ -93,6 +93,9 @@ fn main() { println!(" - key: workload"); println!(" value: batch"); println!(" effect: NoSchedule"); + println!(" kataConfigRef:"); + println!(" kind: ConfigMap"); + println!(" name: kata-drop-in"); println!("```"); println!(); println!("### Spec Fields"); @@ -240,6 +243,36 @@ fn main() { println!("colliding on `(key, effect)` are surfaced as a `TaintOwnershipConflict` condition"); println!("rather than overwritten."); println!(); + println!("#### kata"); + println!(); + println!("(optional, KataConfig) Reference to a `Secret` or `ConfigMap` **on the workload"); + println!("cluster** holding a Kata containerd drop-in to deliver to the node(s) this resource"); + println!("owns. When set, the controller resolves the object on the workload cluster (via the"); + println!("`kubeconfig-` Secret) in `kata.namespace` (default `5spot-system`). If"); + println!("present, it stamps the `5spot.finos.org/kata-config=enabled` opt-in label plus a"); + println!("reference annotation on the Node; the `5spot-kata-config-agent` DaemonSet reads the"); + println!("object from the workload API, writes the drop-in to `destPath`, and restarts"); + println!("`restartService` so containerd reloads it. If the object (or its namespace) is"); + println!("absent, the controller does NOT label the Node and reports a fail-fast status"); + println!("condition — 5-Spot never creates the object (it must pre-exist, Flux-delivered)."); + println!("This is config delivery, not a Kata install — `/opt/kata` binaries remain"); + println!("`kata-deploy`'s job. See ADR 0002 and ADR 0003."); + println!(); + println!("`KataConfig` has the following fields:"); + println!(); + println!("- **kind** (required, enum): One of `ConfigMap`, `Secret` — the source kind."); + println!("- **name** (required, string): Source object name on the workload cluster,"); + println!(" RFC-1123 DNS subdomain (≤ 253 chars)."); + println!("- **namespace** (optional, string, default: `5spot-system`): workload-cluster"); + println!(" namespace the agent reads the object from. Override for per-tenant placement."); + println!("- **key** (optional, string, default: `kata-containers.toml`): `data` key whose"); + println!(" value is the drop-in content."); + println!("- **destPath** (optional, string, default:"); + println!(" `/etc/k0s/container.d/kata-containers.toml`): absolute host path written to."); + println!("- **restartService** (optional, string, default: `k0sworker.service`): systemd"); + println!(" unit restarted via `nsenter` so containerd reloads the drop-in. Override with"); + println!(" `k0scontroller.service` on single-node layouts."); + println!(); println!("### Status Fields"); println!(); println!("#### phase"); diff --git a/src/bin/kata_config_agent.rs b/src/bin/kata_config_agent.rs new file mode 100644 index 0000000..2d34ae1 --- /dev/null +++ b/src/bin/kata_config_agent.rs @@ -0,0 +1,270 @@ +// Copyright (c) 2026 Erick Bourgeois, 5-Spot +// SPDX-License-Identifier: Apache-2.0 +//! # `5spot-kata-config-agent` +//! +//! Node-side DaemonSet binary that delivers a Kata containerd drop-in to the +//! host filesystem (ADR 0002 / ADR 0003, roadmap Phase 3). On each tick it reads +//! its own Node's `5spot.finos.org/kata-config-ref` annotation (stamped by the +//! controller), resolves the named `ConfigMap`/`Secret` from the workload-cluster +//! API, and reconciles the host destination file to match — writing atomically +//! on change and self-healing drift. +//! +//! The agent reads via the kube API rather than a mounted ConfigMap volume +//! because a cluster-wide DaemonSet cannot template a `configMap.name` volume per +//! replica (ADR 0002 §3). The host k0s-service restart (`nsenter`) and the +//! applied-hash node annotation are Phase 4 and not yet wired here — this binary +//! lands and self-heals the file; containerd reloads it once Phase 4 triggers the +//! restart. The drift-watch is a deliberate node-local poll: out-of-band edits to +//! the host file generate no Kubernetes event, so there is nothing to watch on +//! the API for that signal. + +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use clap::Parser; +use k8s_openapi::api::core::v1::{ConfigMap, Node, Secret}; +use kube::api::{Patch, PatchParams}; +use kube::{Api, Client}; +use serde_json::json; +use tracing::{debug, info, warn}; + +use five_spot::constants::{ + KATA_CONFIG_APPLIED_ANNOTATION, KATA_CONFIG_LABEL, KATA_CONFIG_REF_ANNOTATION, +}; +use five_spot::kata_config_agent::{ + parse_kata_ref, sync_content, KataRef, SyncOutcome, DEFAULT_POLL_INTERVAL_SECS, +}; + +/// CLI / environment configuration for the kata-config agent. +#[derive(Debug, Parser)] +#[command( + name = "5spot-kata-config-agent", + about = "Delivers a Kata containerd drop-in to the host filesystem" +)] +struct Cli { + /// Name of the Node this pod runs on (downward API `spec.nodeName`). The + /// agent reads this Node's `5spot.finos.org/kata-config-ref` annotation. + #[arg(long, env = "NODE_NAME")] + node_name: String, + + /// Host filesystem mount root inside the pod. The drop-in's absolute + /// `destPath` is written under here (e.g. `/host` + `/etc/k0s/...`). + #[arg(long, env = "HOST_ROOT", default_value = "/host")] + host_root: PathBuf, + + /// Seconds between drift-watch sweeps. + #[arg(long, env = "POLL_INTERVAL_SECS", default_value_t = DEFAULT_POLL_INTERVAL_SECS)] + poll_interval_secs: u64, + + /// Run a single reconcile and exit (smoke-test / one-shot use). + #[arg(long, env = "ONESHOT", default_value_t = false)] + oneshot: bool, +} + +/// Resolve the absolute host `destPath` to the in-pod path under `host_root` +/// (e.g. `host_root=/host`, `dest=/etc/k0s/x.toml` → `/host/etc/k0s/x.toml`). +fn host_path(host_root: &std::path::Path, dest_path: &str) -> PathBuf { + host_root.join(dest_path.trim_start_matches('/')) +} + +/// The two kata annotations the agent reads off its own Node: the controller's +/// reference (present ⇒ deliver) and the agent's own applied-dest record (the +/// host path it last wrote, so it can clean up after tear-down). +struct NodeState { + kata_ref: Option, + applied_dest: Option, +} + +/// Read this Node's kata-config-ref and kata-config-applied annotations. +async fn read_node_state(nodes: &Api, node_name: &str) -> Result { + let node = nodes + .get(node_name) + .await + .with_context(|| format!("GET Node {node_name}"))?; + let anns = node.metadata.annotations.unwrap_or_default(); + let kata_ref = match anns.get(KATA_CONFIG_REF_ANNOTATION) { + Some(raw) => Some(parse_kata_ref(raw).with_context(|| { + format!("parsing {KATA_CONFIG_REF_ANNOTATION} on Node {node_name}") + })?), + None => None, + }; + Ok(NodeState { + kata_ref, + applied_dest: anns.get(KATA_CONFIG_APPLIED_ANNOTATION).cloned(), + }) +} + +/// Record the absolute host `destPath` this agent is responsible for, in the +/// kata-config-applied annotation on its own Node — the durable record cleanup +/// reads after the controller clears the reference annotation. +async fn record_applied(nodes: &Api, node_name: &str, dest_path: &str) -> Result<()> { + let patch = + json!({ "metadata": { "annotations": { KATA_CONFIG_APPLIED_ANNOTATION: dest_path } } }); + nodes + .patch(node_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .with_context(|| format!("recording applied-dest on Node {node_name}"))?; + Ok(()) +} + +/// Clear the applied annotation and remove the opt-in label from this Node so +/// the DaemonSet deschedules — the tear-down handshake completion (ADR 0002). +async fn clear_optin(nodes: &Api, node_name: &str) -> Result<()> { + let patch = json!({ + "metadata": { + "annotations": { KATA_CONFIG_APPLIED_ANNOTATION: null }, + "labels": { KATA_CONFIG_LABEL: null } + } + }); + nodes + .patch(node_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .with_context(|| format!("clearing kata opt-in on Node {node_name}"))?; + Ok(()) +} + +/// Fetch the drop-in content referenced by `kref` from the workload API. Returns +/// `None` when the object exists but lacks the key, or the object is absent +/// (404) — both map to a GitOps tear-down of the host file. +async fn fetch_content(client: &Client, kref: &KataRef) -> Result> { + match kref.kind.as_str() { + "ConfigMap" => { + let api: Api = Api::namespaced(client.clone(), &kref.namespace); + let Some(cm) = api.get_opt(&kref.name).await? else { + return Ok(None); + }; + Ok(cm.data.and_then(|d| d.get(&kref.key).cloned())) + } + "Secret" => { + let api: Api = Api::namespaced(client.clone(), &kref.namespace); + let Some(secret) = api.get_opt(&kref.name).await? else { + return Ok(None); + }; + Ok(secret.data.and_then(|d| { + d.get(&kref.key) + .map(|b| String::from_utf8_lossy(&b.0).into_owned()) + })) + } + other => anyhow::bail!("unknown kata source kind {other:?} (expected ConfigMap or Secret)"), + } +} + +/// One reconcile tick. +/// +/// - **reference present** → fetch the source, sync the host file, and record +/// the applied dest on the Node (so cleanup can find it later). +/// - **reference absent** → tear-down: unlink the host file the agent recorded +/// (if any), then clear the applied annotation and remove the opt-in label so +/// the DaemonSet deschedules. Doing the unlink here — while still scheduled — +/// is what closes the descheduled-before-cleanup gap (ADR 0002). +async fn reconcile_once(client: &Client, cli: &Cli) -> Result<()> { + let nodes: Api = Api::all(client.clone()); + let state = read_node_state(&nodes, &cli.node_name).await?; + + let Some(kref) = state.kata_ref else { + // Tear-down. If we previously applied a file, unlink it now. + if let Some(dest_path) = state.applied_dest.as_deref() { + let dest = host_path(&cli.host_root, dest_path); + if sync_content(None, &dest)? == SyncOutcome::Deleted { + info!(node = %cli.node_name, dest = %dest.display(), "removed kata drop-in from host (tear-down)"); + // Phase 4: bounce the host k0s service here (nsenter) so + // containerd drops the removed runtime. + } + } + clear_optin(&nodes, &cli.node_name).await?; + debug!(node = %cli.node_name, "kata tear-down complete; opt-in label removed (descheduling)"); + return Ok(()); + }; + + // Deliver. + let content = fetch_content(client, &kref).await?; + let dest = host_path(&cli.host_root, &kref.dest_path); + let bytes = content.as_ref().map(String::as_bytes); + + match sync_content(bytes, &dest)? { + SyncOutcome::Wrote(hash) => info!( + node = %cli.node_name, + dest = %dest.display(), + sha256 = %hash, + "wrote kata drop-in to host (containerd reload pending Phase 4 restart)" + ), + SyncOutcome::Deleted => info!( + node = %cli.node_name, + dest = %dest.display(), + "removed kata drop-in from host (source object/key cleared)" + ), + SyncOutcome::Unchanged(_) => { + debug!(node = %cli.node_name, dest = %dest.display(), "kata drop-in in sync") + } + } + + // Record the dest we manage so tear-down can find it after the controller + // clears the reference annotation. + record_applied(&nodes, &cli.node_name, &kref.dest_path).await?; + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), + ) + .init(); + + let cli = Cli::parse(); + info!( + node = %cli.node_name, + host_root = %cli.host_root.display(), + poll_interval_secs = cli.poll_interval_secs, + oneshot = cli.oneshot, + "kata-config-agent started", + ); + + let client = Client::try_default() + .await + .context("building in-cluster kube client")?; + + let interval = Duration::from_secs(cli.poll_interval_secs.max(1)); + loop { + if let Err(e) = reconcile_once(&client, &cli).await { + warn!( + node = %cli.node_name, + error = %e, + "kata drop-in reconcile failed; will retry next tick" + ); + } + if cli.oneshot { + break; + } + tokio::time::sleep(interval).await; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::host_path; + use std::path::Path; + + #[test] + fn test_host_path_joins_dest_under_root() { + assert_eq!( + host_path( + Path::new("/host"), + "/etc/k0s/container.d/kata-containers.toml" + ), + Path::new("/host/etc/k0s/container.d/kata-containers.toml") + ); + } + + #[test] + fn test_host_path_strips_leading_slashes() { + assert_eq!( + host_path(Path::new("/host"), "///a/b"), + Path::new("/host/a/b") + ); + } +} diff --git a/src/constants.rs b/src/constants.rs index 87e7a17..fbf353b 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -125,6 +125,49 @@ pub const RECLAIM_AGENT_CONFIGMAP_PREFIX: &str = "reclaim-agent-"; /// drifting if one side is renamed in isolation. pub const RECLAIM_CONFIG_DATA_KEY: &str = "reclaim.toml"; +// ============================================================================ +// Kata Config Delivery — per-node ConfigMap projection + opt-in label (ADR 0002) +// ============================================================================ +// +// When a `ScheduledMachine`'s `spec.kataConfigRef` is set, the controller +// resolves the referenced Secret/ConfigMap content and mirrors it onto the +// child cluster as a per-node `ConfigMap`, plus stamps an opt-in label on the +// backing Node so the `5spot-kata-config-agent` DaemonSet lands. This is a +// separate opt-in from emergency reclaim and from kata-deploy's +// `katacontainers.io/kata-runtime` label. See ADR 0002 / ADR 0003. + +/// Node label key used by the kata-config-agent `DaemonSet`'s `nodeSelector`. +/// The controller stamps this on every Node backing a `ScheduledMachine` whose +/// `spec.kata` source is resolved on the workload cluster; clearing `spec.kata` +/// (or a missing source) removes the label and tears the agent off the node. +/// Distinct from [`RECLAIM_AGENT_LABEL`] and from `kata-deploy`'s +/// `katacontainers.io/kata-runtime` label. +pub const KATA_CONFIG_LABEL: &str = "5spot.finos.org/kata-config"; + +/// Value for [`KATA_CONFIG_LABEL`] indicating delivery is enabled for this node. +pub const KATA_CONFIG_LABEL_ENABLED: &str = "enabled"; + +/// Annotation the controller stamps on a Node alongside [`KATA_CONFIG_LABEL`], +/// carrying a compact JSON object (`namespace`, `kind`, `name`, `key`, +/// `destPath`, `restartService`) that tells the node-side agent which workload +/// object to read and where to write it. Cleared (`null`) on tear-down / absent +/// source. See ADR 0002. +pub const KATA_CONFIG_REF_ANNOTATION: &str = "5spot.finos.org/kata-config-ref"; + +/// Annotation the **agent** records on its own Node after writing the host +/// drop-in, carrying the applied absolute `destPath`. It is the agent's only +/// durable record of what it wrote, so that on tear-down (the controller clears +/// [`KATA_CONFIG_REF_ANNOTATION`] but keeps the opt-in label) the still-scheduled +/// agent can unlink the right host file before removing its own label to +/// deschedule. See ADR 0002 / ADR 0003. +pub const KATA_CONFIG_APPLIED_ANNOTATION: &str = "5spot.finos.org/kata-config-applied"; + +/// Default workload-cluster namespace the agent reads the kata source object +/// from (`spec.kata.namespace` default) and where the kata-config-agent +/// `DaemonSet` runs. Same `5spot-system` namespace as the reclaim agent +/// ([`RECLAIM_AGENT_NAMESPACE`]). +pub const KATA_CONFIG_NAMESPACE: &str = "5spot-system"; + /// Kubernetes Event reason emitted on the `ScheduledMachine` when the /// emergency reclaim path fires. Operators see this in /// `kubectl describe scheduledmachine`. diff --git a/src/crd.rs b/src/crd.rs index 8b7d281..bdeb967 100644 --- a/src/crd.rs +++ b/src/crd.rs @@ -151,6 +151,29 @@ pub struct ScheduledMachineSpec { /// requirements and threat model. #[serde(default, skip_serializing_if = "Option::is_none")] pub kubeconfig_secret_ref: Option, + + /// Optional reference to a `Secret` or `ConfigMap` on the **workload + /// cluster** holding a Kata containerd drop-in to deliver to the node(s) this + /// resource owns. + /// + /// When set, the controller resolves the referenced object on the workload + /// cluster (via the `kubeconfig-` Secret) in `kata.namespace` + /// (default `5spot-system`). If it is present, the controller stamps the + /// `5spot.finos.org/kata-config=enabled` opt-in label **and** a reference + /// annotation on the backing Node; the `5spot-kata-config-agent` DaemonSet — + /// scheduled onto labelled nodes — reads the object from the workload API, + /// writes the drop-in to `destPath`, and restarts `restartService` so + /// containerd reloads it. If the object (or its namespace) is absent, the + /// controller does **not** label the Node and reports a fail-fast status + /// condition — 5-Spot never creates the object (it must pre-exist, + /// Flux-delivered). + /// + /// This is config *delivery*, not a Kata install: the `/opt/kata` binaries + /// remain `kata-deploy`'s responsibility, and the existing + /// `katacontainers.io/kata-runtime` opt-in label is unaffected. See ADR 0002 + /// and ADR 0003. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kata: Option, } fn default_priority() -> u8 { @@ -413,6 +436,91 @@ fn default_kubeconfig_secret_key() -> String { "value".to_string() } +/// Schema for `KataConfig.name` — bounded to RFC-1123 DNS subdomain length +/// (253 chars) with the Kubernetes-standard charset. The same rule governs both +/// `ConfigMap` and `Secret` names, so one schema covers both source kinds; a +/// value that fits is guaranteed to be a syntactically valid object name. +fn kata_config_name_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + }) +} + +/// Schema for `KataConfig.key` — the ConfigMap/Secret `data` key holding the +/// drop-in content. Bounded to the 253-char data-key cap with the charset the +/// Kubernetes API server enforces on `data` keys. +fn kata_config_key_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[A-Za-z0-9._-]+$" + }) +} + +/// Schema for `KataConfig.destPath` — the absolute host path the agent writes +/// the drop-in to. Must be absolute (anchored at `/`) and is bounded to the +/// Linux `PATH_MAX` (4096) so a malicious CR cannot inflate the host write path. +fn kata_config_dest_path_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^/[^\\u0000]*$" + }) +} + +/// Schema for `KataConfig.restartService` — the systemd unit the agent +/// restarts via `nsenter`. Constrained to a `*.service` unit name with the +/// systemd unit charset and bounded to systemd's 255-char unit-name cap. +fn kata_config_restart_service_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[A-Za-z0-9@._-]+\\.service$" + }) +} + +/// Schema for `KataConfig.namespace` — the workload-cluster namespace the agent +/// reads the source object from. RFC-1123 label (max 63 chars) with the +/// Kubernetes namespace charset. +fn kata_config_namespace_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + }) +} + +/// Default workload-cluster namespace for `KataConfig.namespace` — the agent's +/// own namespace, so it reads from its namespace with no cross-namespace RBAC. +fn default_kata_config_namespace() -> String { + crate::constants::KATA_CONFIG_NAMESPACE.to_string() +} + +/// Default `data` key for `KataConfig.key` — the containerd drop-in filename. +fn default_kata_config_key() -> String { + "kata-containers.toml".to_string() +} + +/// Default host path for `KataConfig.destPath` — the k0s containerd drop-in +/// directory. k0s loads drop-ins from `/etc/k0s/container.d/` on (re)start. +fn default_kata_config_dest_path() -> String { + "/etc/k0s/container.d/kata-containers.toml".to_string() +} + +/// Default systemd unit for `KataConfig.restartService` — the k0s worker +/// service. Single-node / controller-also-runs-workloads layouts override this +/// with `k0scontroller.service`. +fn default_kata_config_restart_service() -> String { + "k0sworker.service".to_string() +} + /// Schema for `EmbeddedResource` — requires apiVersion, kind, and spec fields. /// The `spec` field uses `x-kubernetes-preserve-unknown-fields` to allow any /// provider-specific fields (`K0sWorkerConfig`, `RemoteMachine`, `AWSMachine`, etc.). @@ -537,6 +645,76 @@ pub struct KubeconfigSecretRef { pub key: String, } +// ============================================================================ +// KataConfig - Reference to a Kata containerd drop-in source on the workload cluster +// ============================================================================ + +/// Source kind for a [`KataConfig`] — the drop-in content lives in either a +/// `ConfigMap` or a `Secret`. Variants serialize verbatim (`ConfigMap`, +/// `Secret`) so they line up with the Kubernetes object kinds and the +/// `KIND_CONFIG_MAP` / `KIND_SECRET` constants. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Hash)] +pub enum KataConfigSourceKind { + /// Drop-in content is stored in a `ConfigMap` on the workload cluster. + ConfigMap, + /// Drop-in content is stored in a `Secret` on the workload cluster. + Secret, +} + +/// Pointer to a `Secret` or `ConfigMap` on the **workload cluster** whose data +/// holds a Kata containerd drop-in to deliver to the node(s) this resource owns. +/// +/// See [`ScheduledMachineSpec::kata`] for the full semantics (workload-cluster +/// resolution, opt-in label + reference annotation, host write, and k0s +/// restart). Decisions: ADR 0002 (contract + resolution) and ADR 0003 (host +/// write + `nsenter` restart). +/// +/// `deny_unknown_fields` is intentional and mirrors [`KubeconfigSecretRef`]: a +/// typo must be a hard error, not a silent miss. The object is resolved on the +/// workload cluster in `namespace` (default `5spot-system`); 5-Spot **never +/// creates it** — it must pre-exist (Flux-delivered), or delivery fails fast +/// with a status condition. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct KataConfig { + /// Whether the source is a `ConfigMap` or a `Secret`. Required (no default). + pub kind: KataConfigSourceKind, + + /// Name of the source object on the workload cluster. RFC-1123 DNS subdomain + /// (max 253 chars). + #[schemars(schema_with = "kata_config_name_schema")] + pub name: String, + + /// Workload-cluster namespace the agent reads the object from. Defaults to + /// `5spot-system` (the agent's own namespace, so no cross-namespace agent + /// RBAC is needed). Override to place config in a per-tenant namespace. + #[serde(default = "default_kata_config_namespace")] + #[schemars(schema_with = "kata_config_namespace_schema")] + pub namespace: String, + + /// Key within the source's `data` map whose value is the drop-in content. + /// Defaults to `kata-containers.toml`. + #[serde(default = "default_kata_config_key")] + #[schemars(schema_with = "kata_config_key_schema")] + pub key: String, + + /// Absolute host path the agent writes the drop-in to. Defaults to + /// `/etc/k0s/container.d/kata-containers.toml` (the k0s containerd drop-in + /// directory). Configurable because layouts vary (k0s-in-docker, kairos, + /// vanilla containerd). + #[serde(default = "default_kata_config_dest_path")] + #[schemars(schema_with = "kata_config_dest_path_schema")] + pub dest_path: String, + + /// systemd unit the node agent restarts (via `nsenter`) so containerd + /// reloads the drop-in. Defaults to `k0sworker.service`; override with + /// `k0scontroller.service` on single-node / controller-runs-workloads + /// layouts. + #[serde(default = "default_kata_config_restart_service")] + #[schemars(schema_with = "kata_config_restart_service_schema")] + pub restart_service: String, +} + // ============================================================================ // NodeTaint / TaintEffect - User-declared taints on the provisioned Node // ============================================================================ diff --git a/src/crd_tests.rs b/src/crd_tests.rs index a32d86c..a58323c 100644 --- a/src/crd_tests.rs +++ b/src/crd_tests.rs @@ -315,6 +315,7 @@ mod tests { node_taints: vec![], kill_if_commands: None, kubeconfig_secret_ref: None, + kata: None, }; // Test that it serializes without errors @@ -537,6 +538,7 @@ mod tests { node_taints: vec![], kill_if_commands: None, kubeconfig_secret_ref: None, + kata: None, } } @@ -1182,6 +1184,7 @@ mod tests { name: "alpha-kubeconfig".to_string(), key: "value".to_string(), }), + kata: None, }; let s = serde_json::to_string(&spec).expect("must serialize"); assert!(s.contains("kubeconfigSecretRef")); @@ -1252,6 +1255,254 @@ mod tests { ); } + // ======================================================================== + // KataConfig — Kata config delivery reference tests (ADR 0002, TDD) + // + // These tests pin the contract for the optional `spec.kata` field that points + // a ScheduledMachine at a Secret or ConfigMap **on the workload cluster** + // holding a Kata containerd drop-in. The CRD type is the source of truth; + // serde defaults (incl. `namespace` → 5spot-system) and YAML schema bounds + // are asserted here so a regression flips a test. + // ======================================================================== + + #[test] + fn test_kata_defaults_when_optional_fields_omitted() { + // Posit: with only `kind` and `name`, serde fills the defaulted fields + // (namespace, key, destPath, restartService) so the common case is + // zero-config. + let json = serde_json::json!({ "kind": "ConfigMap", "name": "kata-drop-in" }); + let parsed: KataConfig = serde_json::from_value(json) + .expect("KataConfig must accept a JSON object with only kind + name"); + assert_eq!(parsed.kind, KataConfigSourceKind::ConfigMap); + assert_eq!(parsed.name, "kata-drop-in"); + assert_eq!( + parsed.namespace, "5spot-system", + "default namespace must be the agent's own namespace (5spot-system)" + ); + assert_eq!( + parsed.key, "kata-containers.toml", + "default key must be the containerd drop-in filename" + ); + assert_eq!( + parsed.dest_path, "/etc/k0s/container.d/kata-containers.toml", + "default destPath must be the k0s containerd drop-in path" + ); + assert_eq!( + parsed.restart_service, "k0sworker.service", + "default restartService must be the k0s worker unit" + ); + } + + #[test] + fn test_kata_explicit_round_trips() { + let original = KataConfig { + kind: KataConfigSourceKind::Secret, + name: "kata-secret".to_string(), + namespace: "team-alpha".to_string(), + key: "custom.toml".to_string(), + dest_path: "/etc/kata-containers/configuration.toml".to_string(), + restart_service: "k0scontroller.service".to_string(), + }; + let serialized = serde_json::to_value(&original).expect("must serialize"); + let parsed: KataConfig = + serde_json::from_value(serialized.clone()).expect("must round-trip"); + assert_eq!(parsed, original); + // camelCase JSON field names are required for kube-rs / OpenAPI conformance. + let s = serialized.to_string(); + assert!(s.contains("\"kind\""), "expected camelCase `kind`: {s}"); + assert!(s.contains("\"name\""), "expected camelCase `name`: {s}"); + assert!(s.contains("\"key\""), "expected camelCase `key`: {s}"); + assert!( + s.contains("\"destPath\""), + "expected camelCase `destPath`: {s}" + ); + assert!( + s.contains("\"restartService\""), + "expected camelCase `restartService`: {s}" + ); + } + + #[test] + fn test_kata_config_source_kind_serializes_pascalcase() { + // The two variants must serialize verbatim so they line up with the + // Kubernetes object kinds "ConfigMap" / "Secret". + assert_eq!( + serde_json::to_value(KataConfigSourceKind::ConfigMap).unwrap(), + serde_json::json!("ConfigMap") + ); + assert_eq!( + serde_json::to_value(KataConfigSourceKind::Secret).unwrap(), + serde_json::json!("Secret") + ); + } + + #[test] + fn test_kata_rejects_invalid_kind() { + // Only the two PascalCase kinds are valid; a lowercase typo must error. + let json = serde_json::json!({ "kind": "configmap", "name": "x" }); + let result: Result = serde_json::from_value(json); + assert!( + result.is_err(), + "KataConfig must reject a kind outside {{ConfigMap, Secret}}" + ); + } + + #[test] + fn test_kata_namespace_overrides_default() { + // `namespace` is a first-class field (the workload-cluster namespace the + // agent reads from); an explicit value must override the 5spot-system + // default and round-trip. + let json = serde_json::json!({ + "kind": "ConfigMap", + "name": "kata-drop-in", + "namespace": "team-alpha" + }); + let parsed: KataConfig = + serde_json::from_value(json).expect("KataConfig must accept an explicit namespace"); + assert_eq!(parsed.namespace, "team-alpha"); + } + + #[test] + fn test_kata_rejects_unknown_fields() { + // `deny_unknown_fields` blocks typos — an unrecognised field is a hard + // error, not a silent miss (ADR 0002). + let json = serde_json::json!({ + "kind": "ConfigMap", + "name": "kata-drop-in", + "bogusField": "x" // not a KataConfig field + }); + let result: Result = serde_json::from_value(json); + assert!( + result.is_err(), + "KataConfig must reject unknown fields (typos)" + ); + } + + #[test] + fn test_kata_kind_and_name_required() { + // Neither `kind` nor `name` has a default — omitting either must error. + let missing_kind = serde_json::json!({ "name": "x" }); + assert!( + serde_json::from_value::(missing_kind).is_err(), + "KataConfig must require `kind`" + ); + let missing_name = serde_json::json!({ "kind": "Secret" }); + assert!( + serde_json::from_value::(missing_name).is_err(), + "KataConfig must require `name`" + ); + } + + #[test] + fn test_spec_with_kata_round_trips() { + let mut spec = base_spec(); + spec.kata = Some(KataConfig { + kind: KataConfigSourceKind::ConfigMap, + name: "kata-drop-in".to_string(), + namespace: "5spot-system".to_string(), + key: "kata-containers.toml".to_string(), + dest_path: "/etc/k0s/container.d/kata-containers.toml".to_string(), + restart_service: "k0sworker.service".to_string(), + }); + let s = serde_json::to_string(&spec).expect("must serialize"); + assert!(s.contains("\"kata\""), "expected the `kata` field key: {s}"); + assert!(s.contains("kata-drop-in")); + let parsed: ScheduledMachineSpec = + serde_json::from_str(&s).expect("must round-trip with spec.kata"); + let r = parsed.kata.expect("ref must survive round-trip"); + assert_eq!(r.kind, KataConfigSourceKind::ConfigMap); + assert_eq!(r.name, "kata-drop-in"); + } + + #[test] + fn test_spec_without_kata_omits_field() { + // skip_serializing_if = Option::is_none must elide the field entirely so + // existing single-runtime SMs keep clean YAML. + let spec = base_spec(); + let s = serde_json::to_string(&spec).expect("must serialize"); + assert!( + !s.contains("\"kata\""), + "kata must be omitted when None: {s}" + ); + } + + // ---- Schema-bound assertions ---- + + #[test] + fn test_kata_name_schema_is_bounded() { + let schema = + serde_json::to_value(schemars::schema_for!(KataConfig)).expect("schema serializes"); + let name_schema = schema + .pointer("/properties/name") + .or_else(|| schema.pointer("/definitions/KataConfig/properties/name")) + .expect("KataConfig.name property must exist in schema"); + assert!( + name_schema.get("maxLength").is_some(), + "name must have maxLength bound (RFC-1123 DNS subdomain = 253): {name_schema}" + ); + assert!( + name_schema.get("pattern").is_some(), + "name must constrain charset via pattern: {name_schema}" + ); + } + + #[test] + fn test_kata_dest_path_schema_requires_absolute() { + let schema = + serde_json::to_value(schemars::schema_for!(KataConfig)).expect("schema serializes"); + let dest = schema + .pointer("/properties/destPath") + .or_else(|| schema.pointer("/definitions/KataConfig/properties/destPath")) + .expect("KataConfig.destPath property must exist in schema"); + let pattern = dest + .get("pattern") + .and_then(|v| v.as_str()) + .expect("destPath must constrain to an absolute path via pattern"); + assert!( + pattern.starts_with("^/"), + "destPath pattern must anchor to a leading slash (absolute path): {pattern}" + ); + assert!( + dest.get("maxLength").is_some(), + "destPath must have a maxLength bound: {dest}" + ); + } + + #[test] + fn test_kata_restart_service_schema_is_bounded() { + let schema = + serde_json::to_value(schemars::schema_for!(KataConfig)).expect("schema serializes"); + let svc = schema + .pointer("/properties/restartService") + .or_else(|| schema.pointer("/definitions/KataConfig/properties/restartService")) + .expect("KataConfig.restartService property must exist in schema"); + assert!( + svc.get("pattern").is_some(), + "restartService must constrain to a systemd unit via pattern: {svc}" + ); + assert!( + svc.get("maxLength").is_some(), + "restartService must have a maxLength bound: {svc}" + ); + } + + #[test] + fn test_spec_kata_is_optional_in_schema() { + // Backward compat: the new field must NOT be in the `required` list. + let schema = + serde_json::to_value(schemars::schema_for!(ScheduledMachineSpec)).expect("serializes"); + let required = schema + .pointer("/required") + .or_else(|| schema.pointer("/definitions/ScheduledMachineSpec/required")) + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert!( + !required.iter().any(|v| v == "kata"), + "kata MUST be optional to preserve backward compatibility: required={required:?}" + ); + } + // ======================================================================== // EmbeddedResource — metadata accessors // ======================================================================== diff --git a/src/kata_config_agent.rs b/src/kata_config_agent.rs new file mode 100644 index 0000000..b371e1a --- /dev/null +++ b/src/kata_config_agent.rs @@ -0,0 +1,259 @@ +// Copyright (c) 2026 Erick Bourgeois, 5-Spot +// SPDX-License-Identifier: Apache-2.0 +//! # Kata config agent — host-filesystem sync engine +//! +//! Core logic for the node-side `5spot-kata-config-agent` binary (ADR 0002 / +//! ADR 0003, roadmap `5spot-kata-config-per-node.md`, Phase 3). This module is +//! I/O-light around the filesystem and fully unit-testable; the binary entry +//! point in `src/bin/kata_config_agent.rs` wires it to a poll loop. +//! +//! ## Sync contract +//! +//! The agent reconciles a single host destination file to match the drop-in +//! content it reads from the workload-cluster `ConfigMap`/`Secret` named by its +//! Node's `5spot.finos.org/kata-config-ref` annotation (ADR 0002 — the agent +//! reads via the kube API, not a mounted file, because a cluster-wide DaemonSet +//! cannot template a `configMap.name` volume per replica): +//! +//! - content present, hashes differ → **atomically** write the host file +//! (temp-file in the destination directory + `rename`, mode `0644`). +//! - content present, hashes match → no-op (drift-watch). +//! - content absent (key removed / object deleted / annotation cleared) → +//! unlink the host file (GitOps: absent in source ⇒ absent on host). +//! +//! Writes are atomic so a crash mid-write can never leave a partially-written +//! drop-in that containerd would fail to parse. The applied-hash node +//! annotation and the host k0s-service restart (`nsenter`) are Phase 4 and +//! live in the binary, not here. + +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::fs; +use std::io; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +/// Parsed `5spot.finos.org/kata-config-ref` Node annotation — the compact JSON +/// object the controller stamps to tell the agent which workload object to read +/// and where to write it (ADR 0002). Field names match the controller's +/// `build_kata_config_ref_annotation_patch`. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct KataRef { + /// Workload-cluster namespace holding the source object. + pub namespace: String, + /// Source kind: `"ConfigMap"` or `"Secret"`. + pub kind: String, + /// Source object name. + pub name: String, + /// `data` key whose value is the drop-in content. + pub key: String, + /// Absolute host path to write the drop-in to. + #[serde(rename = "destPath")] + pub dest_path: String, + /// systemd unit to restart so containerd reloads the drop-in (Phase 4). + #[serde(rename = "restartService")] + pub restart_service: String, +} + +/// Parse the `5spot.finos.org/kata-config-ref` annotation value (a JSON string) +/// into a [`KataRef`]. +/// +/// # Errors +/// Returns the [`serde_json::Error`] if the value is not the expected JSON +/// object (missing/extra fields, wrong types). +pub fn parse_kata_ref(annotation: &str) -> Result { + serde_json::from_str(annotation) +} + +/// Default interval between drift-watch sweeps, in seconds. The agent reads +/// the projected source and reconciles the host file every tick; 30s caps +/// drift-correction latency while keeping the loop near-idle. +pub const DEFAULT_POLL_INTERVAL_SECS: u64 = 30; + +/// File mode applied to the written drop-in. `0644` is world-readable so +/// containerd (running as root) can read it; not writable by group/other. +const DEST_FILE_MODE: u32 = 0o644; + +/// Suffix for the in-flight temporary file written next to the destination +/// before the atomic `rename`. Includes the PID so a crashed prior run's +/// leftover never collides with a live write. +const TEMP_SUFFIX: &str = "5spot-tmp"; + +/// Compute the lowercase hex SHA-256 of `bytes`. Used for content/drift +/// comparison and (Phase 4) the applied-hash node annotation. +#[must_use] +pub fn sha256_hex(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let digest = hasher.finalize(); + let mut out = String::with_capacity(digest.len() * 2); + for byte in digest { + use std::fmt::Write as _; + let _ = write!(out, "{byte:02x}"); + } + out +} + +/// Return `Some(sha256_hex)` of the file at `path`, or `None` if the file does +/// not exist. +/// +/// # Errors +/// Returns the underlying [`io::Error`] for any failure other than the file +/// being absent (which is the expected steady state before first provision). +pub fn file_sha256(path: &Path) -> io::Result> { + match fs::read(path) { + Ok(bytes) => Ok(Some(sha256_hex(&bytes))), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } +} + +/// The reconcile action chosen by [`decide_action`] for one sync tick. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SyncAction { + /// Source present and differs from dest (or dest absent) → write it. + Write, + /// Source absent but dest present → unlink the host file (GitOps delete). + Delete, + /// Source and dest already agree (or both absent) → nothing to do. + NoOp, +} + +/// Decide what to do given the source bytes (if present) and the current +/// destination hash (if the host file exists). Pure — no I/O. +#[must_use] +pub fn decide_action(source: Option<&[u8]>, dest_hash: Option<&str>) -> SyncAction { + match source { + Some(bytes) => { + if dest_hash == Some(sha256_hex(bytes).as_str()) { + SyncAction::NoOp + } else { + SyncAction::Write + } + } + None => { + if dest_hash.is_some() { + SyncAction::Delete + } else { + SyncAction::NoOp + } + } + } +} + +/// Build the path of the temporary file written alongside `dest`. +fn temp_path_for(parent: &Path, dest: &Path) -> PathBuf { + let file_name = dest.file_name().map_or_else( + || "kata-config".to_string(), + |n| n.to_string_lossy().into_owned(), + ); + let pid = std::process::id(); + parent.join(format!("{file_name}.{TEMP_SUFFIX}.{pid}")) +} + +/// Atomically write `content` to `dest`: create any missing parent directories, +/// write a temp file in the destination directory, set mode `0644`, then +/// `rename` it over `dest`. The rename is atomic on a single filesystem, so a +/// crash can never leave a partially-written drop-in for containerd to choke +/// on. Ownership is the agent's process UID (root in the DaemonSet). +/// +/// # Errors +/// Returns the underlying [`io::Error`] if any directory creation, temp write, +/// permission set, or rename fails. +pub fn atomic_write(dest: &Path, content: &[u8]) -> io::Result<()> { + let parent = match dest.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("."), + }; + fs::create_dir_all(parent)?; + + let tmp = temp_path_for(parent, dest); + // Best-effort cleanup of a stale temp from a prior crashed run with the + // same PID (astronomically unlikely, but keeps the write deterministic). + let _ = fs::remove_file(&tmp); + fs::write(&tmp, content)?; + fs::set_permissions(&tmp, fs::Permissions::from_mode(DEST_FILE_MODE))?; + + match fs::rename(&tmp, dest) { + Ok(()) => Ok(()), + Err(e) => { + // Don't leak the temp file if the rename failed. + let _ = fs::remove_file(&tmp); + Err(e) + } + } +} + +/// Remove `dest` if it exists. Returns `Ok(true)` if a file was removed, +/// `Ok(false)` if it was already absent (idempotent tear-down). +/// +/// # Errors +/// Returns the underlying [`io::Error`] for any failure other than the file +/// being absent. +pub fn remove_if_present(dest: &Path) -> io::Result { + match fs::remove_file(dest) { + Ok(()) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + } +} + +/// What [`sync_once`] did this tick. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SyncOutcome { + /// Wrote the host file; carries the SHA-256 of the content now on disk. + Wrote(String), + /// Deleted the host file (source went away). + Deleted, + /// No change; carries the current dest hash (`None` if dest is absent). + Unchanged(Option), +} + +/// Reconcile `dest_path` to match in-memory `content` exactly once (the API-read +/// path — `content` is the drop-in body extracted from the workload object, or +/// `None` when the object/key/annotation is absent): +/// +/// - content present, differs/absent dest → [`atomic_write`], returns `Wrote`. +/// - content absent, dest present → [`remove_if_present`], returns `Deleted`. +/// - already in sync → returns `Unchanged`. +/// +/// # Errors +/// Returns an [`io::Error`] if hashing the destination, writing, or unlinking +/// fails. +pub fn sync_content(content: Option<&[u8]>, dest_path: &Path) -> io::Result { + let dest_hash = file_sha256(dest_path)?; + + match decide_action(content, dest_hash.as_deref()) { + SyncAction::Write => { + let bytes = content.expect("Write action implies the content is present"); + atomic_write(dest_path, bytes)?; + Ok(SyncOutcome::Wrote(sha256_hex(bytes))) + } + SyncAction::Delete => { + remove_if_present(dest_path)?; + Ok(SyncOutcome::Deleted) + } + SyncAction::NoOp => Ok(SyncOutcome::Unchanged(dest_hash)), + } +} + +/// Reconcile `dest_path` to match the file at `source_path` exactly once. +/// +/// Thin wrapper over [`sync_content`] that reads `source_path` first (absent +/// file ⇒ `None` content). Retained for the file-source path and its tests. +/// +/// # Errors +/// Returns an [`io::Error`] if reading the source (other than absent), hashing +/// the destination, writing, or unlinking fails. +pub fn sync_once(source_path: &Path, dest_path: &Path) -> io::Result { + let source = match fs::read(source_path) { + Ok(bytes) => Some(bytes), + Err(e) if e.kind() == io::ErrorKind::NotFound => None, + Err(e) => return Err(e), + }; + sync_content(source.as_deref(), dest_path) +} + +#[cfg(test)] +#[path = "kata_config_agent_tests.rs"] +mod tests; diff --git a/src/kata_config_agent_tests.rs b/src/kata_config_agent_tests.rs new file mode 100644 index 0000000..7fdc70a --- /dev/null +++ b/src/kata_config_agent_tests.rs @@ -0,0 +1,301 @@ +// Copyright (c) 2026 Erick Bourgeois, 5-Spot +// SPDX-License-Identifier: Apache-2.0 +#[cfg(test)] +#[allow(clippy::module_inception)] +mod tests { + use super::super::*; + use std::fs; + use std::os::unix::fs::PermissionsExt; + use std::path::Path; + + // ======================================================================== + // sha256_hex — content hashing + // ======================================================================== + + #[test] + fn test_sha256_hex_known_vectors() { + // NIST/standard SHA-256 vectors pin the implementation so a crate swap + // or encoding bug surfaces immediately. + assert_eq!( + sha256_hex(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn test_sha256_hex_differs_for_different_input() { + assert_ne!(sha256_hex(b"version = 1"), sha256_hex(b"version = 2")); + } + + // ======================================================================== + // file_sha256 — hash of an on-disk file (absent => None) + // ======================================================================== + + #[test] + fn test_file_sha256_absent_is_none() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("nope.toml"); + assert_eq!(file_sha256(&missing).unwrap(), None); + } + + #[test] + fn test_file_sha256_present_matches_sha256_hex() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("present.toml"); + fs::write(&path, b"hello-kata").unwrap(); + assert_eq!(file_sha256(&path).unwrap(), Some(sha256_hex(b"hello-kata"))); + } + + // ======================================================================== + // decide_action — pure sync decision + // ======================================================================== + + #[test] + fn test_decide_write_when_source_present_and_dest_absent() { + assert_eq!(decide_action(Some(b"x"), None), SyncAction::Write); + } + + #[test] + fn test_decide_noop_when_hashes_match() { + let h = sha256_hex(b"same"); + assert_eq!(decide_action(Some(b"same"), Some(&h)), SyncAction::NoOp); + } + + #[test] + fn test_decide_write_when_hashes_differ() { + let stale = sha256_hex(b"old"); + assert_eq!(decide_action(Some(b"new"), Some(&stale)), SyncAction::Write); + } + + #[test] + fn test_decide_delete_when_source_absent_and_dest_present() { + let h = sha256_hex(b"orphan"); + assert_eq!(decide_action(None, Some(&h)), SyncAction::Delete); + } + + #[test] + fn test_decide_noop_when_both_absent() { + assert_eq!(decide_action(None, None), SyncAction::NoOp); + } + + // ======================================================================== + // atomic_write — temp + rename, mode 0644, parent dirs + // ======================================================================== + + #[test] + fn test_atomic_write_creates_file_with_content() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + atomic_write(&dest, b"body").unwrap(); + assert_eq!(fs::read(&dest).unwrap(), b"body"); + } + + #[test] + fn test_atomic_write_sets_mode_0644() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + atomic_write(&dest, b"x").unwrap(); + let mode = fs::metadata(&dest).unwrap().permissions().mode() & 0o777; + assert_eq!( + mode, 0o644, + "drop-in must be world-readable 0644, got {mode:o}" + ); + } + + #[test] + fn test_atomic_write_overwrites_existing() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + fs::write(&dest, b"old").unwrap(); + atomic_write(&dest, b"new").unwrap(); + assert_eq!(fs::read(&dest).unwrap(), b"new"); + } + + #[test] + fn test_atomic_write_creates_parent_dirs() { + let dir = tempfile::tempdir().unwrap(); + // Mimic /etc/k0s/container.d/ not existing yet. + let dest = dir.path().join("etc/k0s/container.d/kata-containers.toml"); + atomic_write(&dest, b"deep").unwrap(); + assert_eq!(fs::read(&dest).unwrap(), b"deep"); + } + + #[test] + fn test_atomic_write_leaves_no_temp_files() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + atomic_write(&dest, b"x").unwrap(); + let entries: Vec<_> = fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + entries, + vec!["kata.toml".to_string()], + "atomic_write must rename its temp file away, leaving only the dest: {entries:?}" + ); + } + + #[test] + fn test_atomic_write_stray_temp_does_not_corrupt_dest() { + // Crash-safety: a leftover temp file from a prior interrupted write + // must not affect the destination — a fresh write renames over dest + // atomically regardless. + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + fs::write(dir.path().join("kata.toml.5spot-tmp.999999"), b"garbage").unwrap(); + atomic_write(&dest, b"clean").unwrap(); + assert_eq!(fs::read(&dest).unwrap(), b"clean"); + } + + // ======================================================================== + // remove_if_present — idempotent unlink + // ======================================================================== + + #[test] + fn test_remove_if_present_removes_existing_returns_true() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + fs::write(&dest, b"x").unwrap(); + assert!(remove_if_present(&dest).unwrap()); + assert!(!Path::new(&dest).exists()); + } + + #[test] + fn test_remove_if_present_absent_returns_false() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("nope.toml"); + assert!(!remove_if_present(&dest).unwrap()); + } + + // ======================================================================== + // sync_once — end-to-end reconcile of dest to source + // ======================================================================== + + #[test] + fn test_sync_once_writes_when_dest_absent() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src.toml"); + let dest = dir.path().join("dest.toml"); + fs::write(&src, b"version = 2\n").unwrap(); + let outcome = sync_once(&src, &dest).unwrap(); + assert_eq!(outcome, SyncOutcome::Wrote(sha256_hex(b"version = 2\n"))); + assert_eq!(fs::read(&dest).unwrap(), b"version = 2\n"); + } + + #[test] + fn test_sync_once_noop_when_dest_matches() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src.toml"); + let dest = dir.path().join("dest.toml"); + fs::write(&src, b"same").unwrap(); + fs::write(&dest, b"same").unwrap(); + let outcome = sync_once(&src, &dest).unwrap(); + assert_eq!(outcome, SyncOutcome::Unchanged(Some(sha256_hex(b"same")))); + } + + #[test] + fn test_sync_once_rewrites_on_drift() { + // Host file edited out-of-band must be restored to the source content. + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src.toml"); + let dest = dir.path().join("dest.toml"); + fs::write(&src, b"canonical").unwrap(); + fs::write(&dest, b"tampered").unwrap(); + let outcome = sync_once(&src, &dest).unwrap(); + assert_eq!(outcome, SyncOutcome::Wrote(sha256_hex(b"canonical"))); + assert_eq!(fs::read(&dest).unwrap(), b"canonical"); + } + + #[test] + fn test_sync_once_deletes_when_source_absent() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src.toml"); // never created + let dest = dir.path().join("dest.toml"); + fs::write(&dest, b"orphan").unwrap(); + let outcome = sync_once(&src, &dest).unwrap(); + assert_eq!(outcome, SyncOutcome::Deleted); + assert!(!Path::new(&dest).exists()); + } + + #[test] + fn test_sync_once_noop_when_both_absent() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src.toml"); + let dest = dir.path().join("dest.toml"); + let outcome = sync_once(&src, &dest).unwrap(); + assert_eq!(outcome, SyncOutcome::Unchanged(None)); + } + + // ---- parse_kata_ref: the 5spot.finos.org/kata-config-ref annotation ---- + + #[test] + fn test_parse_kata_ref_full_object() { + let json = r#"{ + "namespace": "5spot-system", + "kind": "ConfigMap", + "name": "kata-drop-in", + "key": "kata-containers.toml", + "destPath": "/etc/k0s/container.d/kata-containers.toml", + "restartService": "k0sworker.service" + }"#; + let r = parse_kata_ref(json).expect("must parse the controller-stamped annotation"); + assert_eq!(r.namespace, "5spot-system"); + assert_eq!(r.kind, "ConfigMap"); + assert_eq!(r.name, "kata-drop-in"); + assert_eq!(r.key, "kata-containers.toml"); + assert_eq!(r.dest_path, "/etc/k0s/container.d/kata-containers.toml"); + assert_eq!(r.restart_service, "k0sworker.service"); + } + + #[test] + fn test_parse_kata_ref_rejects_missing_field() { + // destPath omitted → hard error (the camelCase rename is required). + let json = r#"{"namespace":"ns","kind":"Secret","name":"n","key":"k","restartService":"s.service"}"#; + assert!( + parse_kata_ref(json).is_err(), + "a missing destPath must be a parse error, not a silent default" + ); + } + + #[test] + fn test_parse_kata_ref_rejects_non_json() { + assert!(parse_kata_ref("not json").is_err()); + } + + // ---- sync_content: the API-read sync path ---- + + #[test] + fn test_sync_content_writes_when_present_and_dest_absent() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + let outcome = sync_content(Some(b"version = 2\n"), &dest).unwrap(); + assert!(matches!(outcome, SyncOutcome::Wrote(_))); + assert_eq!(fs::read(&dest).unwrap(), b"version = 2\n"); + let mode = fs::metadata(&dest).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o644); + } + + #[test] + fn test_sync_content_deletes_when_content_absent() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + fs::write(&dest, b"orphan").unwrap(); + let outcome = sync_content(None, &dest).unwrap(); + assert_eq!(outcome, SyncOutcome::Deleted); + assert!(!dest.exists()); + } + + #[test] + fn test_sync_content_noop_when_hashes_match() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("kata.toml"); + fs::write(&dest, b"same").unwrap(); + let outcome = sync_content(Some(b"same"), &dest).unwrap(); + assert!(matches!(outcome, SyncOutcome::Unchanged(Some(_)))); + } +} diff --git a/src/lib.rs b/src/lib.rs index 6c8593a..7596cd1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,7 @@ pub mod auto_vex_reachability; pub mod constants; pub mod crd; pub mod health; +pub mod kata_config_agent; pub mod labels; pub mod loop_protection; pub mod metrics; diff --git a/src/reconcilers/child_client_tests.rs b/src/reconcilers/child_client_tests.rs index f9776b1..4e6aaa6 100644 --- a/src/reconcilers/child_client_tests.rs +++ b/src/reconcilers/child_client_tests.rs @@ -96,6 +96,7 @@ current-context: child node_taints: vec![], kill_if_commands: None, kubeconfig_secret_ref: ref_, + kata: None, }, status: None, }; diff --git a/src/reconcilers/helpers.rs b/src/reconcilers/helpers.rs index 5b357b8..c4187c4 100644 --- a/src/reconcilers/helpers.rs +++ b/src/reconcilers/helpers.rs @@ -2945,6 +2945,246 @@ pub async fn reconcile_reclaim_agent_provision( Ok(()) } +// ============================================================================ +// Kata config delivery — controller-side projection (ADR 0002, Phase 2) +// ============================================================================ +// +// When a `ScheduledMachine`'s `spec.kataConfigRef` is set, the controller +// resolves the referenced Secret/ConfigMap content (in the SM namespace, on +// the management cluster) and mirrors it onto the **child** cluster as: +// +// 1. A label `KATA_CONFIG_LABEL=KATA_CONFIG_LABEL_ENABLED` on the backing Node +// so the opt-in DaemonSet's `nodeSelector` matches and the agent lands. +// 2. A per-node `ConfigMap` named `kata-config-` in +// `KATA_CONFIG_NAMESPACE` carrying the drop-in body under +// `KATA_CONFIG_DATA_KEY`. +// +// Clearing `kataConfigRef` strips the label (evicting the DaemonSet pod) and +// deletes the ConfigMap. The pure builders/extractors below are unit-tested; +// the async orchestrator underneath wires them to the kube API and is covered +// by mock-client tests mirroring the reclaim-agent projection. + +/// Outcome of a kata-config delivery reconcile for a single Node (ADR 0002). +/// +/// The controller performs no writes for kata delivery beyond the Node +/// label/annotation patch — it resolves the source object on the workload +/// cluster (read-only) and either opts the Node in or fails fast. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum KataDeliveryOutcome { + /// `spec.kata` unset — opt-in label and reference annotation cleared so the + /// agent does not land (or is evicted). + Disabled, + /// Source object resolved on the workload cluster; Node opted in. + Ready, + /// `kata.namespace` exists but the named object is absent — fail fast. + SourceNotFound, + /// `kata.namespace` itself is absent on the workload cluster — fail fast. + TargetNamespaceMissing, +} + +/// Build the merge-patch that sets (`Some`) or clears (`None`) the kata-config +/// **reference annotation** on a Node. The value is a compact JSON object the +/// node-side agent reads to locate and apply its drop-in (namespace, kind, name, +/// key, destPath, restartService). Clearing uses JSON `null` so merge-patch +/// removes the key. +#[must_use] +pub fn build_kata_config_ref_annotation_patch( + kata: Option<&crate::crd::KataConfig>, +) -> serde_json::Value { + let value = match kata { + Some(k) => serde_json::Value::String( + json!({ + "namespace": k.namespace, + "kind": k.kind, + "name": k.name, + "key": k.key, + "destPath": k.dest_path, + "restartService": k.restart_service, + }) + .to_string(), + ), + None => serde_json::Value::Null, + }; + json!({ + "metadata": { + "annotations": { + crate::constants::KATA_CONFIG_REF_ANNOTATION: value, + } + } + }) +} + +/// Build the merge-patch body that sets (when `enable == true`) or clears (when +/// `enable == false`) the kata-config opt-in label on a `Node`. Clearing uses +/// JSON `null` so merge-patch deletes the key — an empty string would leave the +/// label set and keep the DaemonSet pod scheduled. +#[must_use] +pub fn build_kata_config_label_patch(enable: bool) -> serde_json::Value { + let value = if enable { + serde_json::Value::String(crate::constants::KATA_CONFIG_LABEL_ENABLED.to_string()) + } else { + serde_json::Value::Null + }; + json!({ + "metadata": { + "labels": { + crate::constants::KATA_CONFIG_LABEL: value, + } + } + }) +} + +/// Reconcile kata-config delivery for a single Node (ADR 0002). +/// +/// 5-Spot performs **no writes** for kata delivery beyond the Node +/// label/annotation patch; the config object must already exist on the workload +/// cluster (Flux-delivered). For `kata = Some`: +/// - resolve the named `ConfigMap`/`Secret` in `kata.namespace` on the workload +/// cluster (read-only `get_opt`); +/// - **present** → stamp the opt-in label + the reference annotation on the Node +/// → [`KataDeliveryOutcome::Ready`]; +/// - **absent** → clear ONLY the reference annotation (leaving the opt-in label) +/// and return [`KataDeliveryOutcome::SourceNotFound`] or +/// [`KataDeliveryOutcome::TargetNamespaceMissing`] (distinguished by probing +/// the namespace) for a fail-fast status condition. +/// +/// For `kata = None`, the reference annotation is cleared +/// ([`KataDeliveryOutcome::Disabled`]). +/// +/// The opt-in label is **never removed by the controller** — the still-scheduled +/// agent removes it itself after cleaning up the host drop-in, so it is not +/// descheduled before it can tear the file down (ADR 0002). +/// +/// `workload_client` is the resolved child-cluster client; all reads and the +/// Node patch target the workload cluster where the agent runs. +/// +/// # Errors +/// Returns [`ReconcilerError::KubeError`] when a `get`/probe fails with a +/// non-404 error or when the Node PATCH fails. +pub async fn reconcile_kata_config_delivery( + workload_client: &Client, + node_name: &str, + kata: Option<&crate::crd::KataConfig>, +) -> Result { + use crate::crd::KataConfigSourceKind; + use k8s_openapi::api::core::v1::{ConfigMap, Namespace, Node, Secret}; + + let nodes: Api = Api::all(workload_client.clone()); + + // Deliver patch: stamp the opt-in label + reference annotation in one + // merge-patch (both under metadata). + let deliver_patch = |kata: &crate::crd::KataConfig| { + let mut patch = build_kata_config_label_patch(true); + let ann = build_kata_config_ref_annotation_patch(Some(kata)); + if let (Some(meta), Some(ann_meta)) = ( + patch.get_mut("metadata").and_then(|m| m.as_object_mut()), + ann.get("metadata").and_then(|m| m.as_object()), + ) { + for (k, v) in ann_meta { + meta.insert(k.clone(), v.clone()); + } + } + patch + }; + + // Tear-down patch clears ONLY the reference annotation and deliberately + // leaves the opt-in label in place. The still-scheduled agent sees the + // annotation gone, unlinks the host file it recorded (in the + // kata-config-applied annotation), then removes its own label to deschedule + // (ADR 0002). Removing the label here instead would race the agent off the + // Node before it could clean up the host drop-in. + let teardown_patch = build_kata_config_ref_annotation_patch(None); + + let Some(kata) = kata else { + nodes + .patch( + node_name, + &PatchParams::default(), + &Patch::Merge(&teardown_patch), + ) + .await + .map_err(|e| { + error!(node = %node_name, error = %e, "Failed to clear kata-config reference annotation"); + ReconcilerError::KubeError(e) + })?; + return Ok(KataDeliveryOutcome::Disabled); + }; + + // Read-only existence check on the workload cluster. + let present = match kata.kind { + KataConfigSourceKind::ConfigMap => { + let api: Api = Api::namespaced(workload_client.clone(), &kata.namespace); + api.get_opt(&kata.name) + .await + .map_err(ReconcilerError::KubeError)? + .is_some() + } + KataConfigSourceKind::Secret => { + let api: Api = Api::namespaced(workload_client.clone(), &kata.namespace); + api.get_opt(&kata.name) + .await + .map_err(ReconcilerError::KubeError)? + .is_some() + } + }; + + if present { + let patch = deliver_patch(kata); + nodes + .patch(node_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .map_err(|e| { + error!(node = %node_name, error = %e, "Failed to stamp kata-config Node opt-in"); + ReconcilerError::KubeError(e) + })?; + debug!( + node = %node_name, + namespace = %kata.namespace, + name = %kata.name, + "kata-config source resolved on workload cluster; Node opted in" + ); + return Ok(KataDeliveryOutcome::Ready); + } + + // Absent: distinguish a missing namespace from a missing object for a precise + // status reason, then clear the reference annotation (the agent self-cleans + // and deschedules; the label is left for it to remove). + let namespaces: Api = Api::all(workload_client.clone()); + let ns_missing = namespaces + .get_opt(&kata.namespace) + .await + .map_err(ReconcilerError::KubeError)? + .is_none(); + + nodes + .patch( + node_name, + &PatchParams::default(), + &Patch::Merge(&teardown_patch), + ) + .await + .map_err(|e| { + error!(node = %node_name, error = %e, "Failed to clear kata-config reference annotation"); + ReconcilerError::KubeError(e) + })?; + + if ns_missing { + warn!( + node = %node_name, + namespace = %kata.namespace, + "kata-config target namespace missing on workload cluster (fail-fast)" + ); + return Ok(KataDeliveryOutcome::TargetNamespaceMissing); + } + warn!( + node = %node_name, + namespace = %kata.namespace, + name = %kata.name, + "kata-config source object not found on workload cluster (fail-fast)" + ); + Ok(KataDeliveryOutcome::SourceNotFound) +} + // ============================================================================ // Node taint diff + apply (Phase 3 of user-defined-node-taints roadmap) // ============================================================================ diff --git a/src/reconcilers/helpers_tests.rs b/src/reconcilers/helpers_tests.rs index 0926c4d..cbb59f7 100644 --- a/src/reconcilers/helpers_tests.rs +++ b/src/reconcilers/helpers_tests.rs @@ -1945,6 +1945,7 @@ mod tests { node_taints: vec![], kill_if_commands: None, kubeconfig_secret_ref: None, + kata: None, }, status: None, }; @@ -2872,6 +2873,247 @@ mod tests { srv.await.unwrap(); } + // ======================================================================== + // Kata config delivery — label/annotation patch builders + workload-read + // reconcile (ADR 0002). 5-Spot performs NO ConfigMap/Secret writes: it reads + // the source object on the workload cluster and stamps (or clears) the Node + // opt-in, failing fast when the object/namespace is absent. + // ======================================================================== + + fn kata_json_resp(value: serde_json::Value) -> http::Response { + Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&value).unwrap())) + .unwrap() + } + fn kata_node_resp(name: &str) -> http::Response { + kata_json_resp(serde_json::json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": { "name": name, "resourceVersion": "2" } + })) + } + fn kata_cm_resp(name: &str, ns: &str) -> http::Response { + kata_json_resp(serde_json::json!({ + "apiVersion": "v1", "kind": "ConfigMap", + "metadata": { "name": name, "namespace": ns } + })) + } + fn kata_ns_resp(name: &str) -> http::Response { + kata_json_resp(serde_json::json!({ + "apiVersion": "v1", "kind": "Namespace", + "metadata": { "name": name } + })) + } + fn kata_404() -> http::Response { + Response::builder() + .status(404) + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "apiVersion": "v1", "kind": "Status", "status": "Failure", + "code": 404, "reason": "NotFound" + })) + .unwrap(), + )) + .unwrap() + } + fn sample_kata() -> crate::crd::KataConfig { + crate::crd::KataConfig { + kind: crate::crd::KataConfigSourceKind::ConfigMap, + name: "kata-drop-in".to_string(), + namespace: "5spot-system".to_string(), + key: "kata-containers.toml".to_string(), + dest_path: "/etc/k0s/container.d/kata-containers.toml".to_string(), + restart_service: "k0sworker.service".to_string(), + } + } + + const KATA_LABEL_PTR: &str = "/metadata/labels/5spot.finos.org~1kata-config"; + const KATA_ANN_PTR: &str = "/metadata/annotations/5spot.finos.org~1kata-config-ref"; + + #[test] + fn test_build_kata_config_label_patch_enable_writes_enabled_value() { + let patch = build_kata_config_label_patch(true); + assert_eq!( + patch.pointer(KATA_LABEL_PTR).and_then(|v| v.as_str()), + Some("enabled") + ); + } + + #[test] + fn test_build_kata_config_label_patch_disable_writes_null() { + let patch = build_kata_config_label_patch(false); + assert!( + patch + .pointer(KATA_LABEL_PTR) + .map(serde_json::Value::is_null) + .unwrap_or(false), + "disable must write JSON null so merge-patch removes the label" + ); + } + + #[test] + fn test_build_kata_config_ref_annotation_patch_some_carries_compact_json() { + let kata = sample_kata(); + let patch = build_kata_config_ref_annotation_patch(Some(&kata)); + let ann = patch + .pointer(KATA_ANN_PTR) + .and_then(|v| v.as_str()) + .expect("annotation must be a JSON string"); + let parsed: serde_json::Value = + serde_json::from_str(ann).expect("annotation value is JSON"); + assert_eq!(parsed["namespace"], "5spot-system"); + assert_eq!(parsed["kind"], "ConfigMap"); + assert_eq!(parsed["name"], "kata-drop-in"); + assert_eq!(parsed["key"], "kata-containers.toml"); + assert_eq!( + parsed["destPath"], + "/etc/k0s/container.d/kata-containers.toml" + ); + assert_eq!(parsed["restartService"], "k0sworker.service"); + } + + #[test] + fn test_build_kata_config_ref_annotation_patch_none_clears() { + let patch = build_kata_config_ref_annotation_patch(None); + assert!( + patch + .pointer(KATA_ANN_PTR) + .map(serde_json::Value::is_null) + .unwrap_or(false), + "None must write JSON null so merge-patch removes the annotation" + ); + } + + #[tokio::test] + async fn test_reconcile_kata_config_delivery_disabled_clears_optin() { + let (client, handle) = mock_client_pair(); + let srv = tokio::spawn(async move { + let mut h = pin!(handle); + let (req, send) = h.next_request().await.expect("expected Node PATCH (clear)"); + assert_eq!(req.method(), http::Method::PATCH); + assert!(req.uri().path().ends_with("/api/v1/nodes/node-a")); + let body = collect_json_body(req.into_body()).await; + assert!( + body.pointer(KATA_LABEL_PTR).is_none(), + "tear-down must NOT touch the opt-in label (the agent removes it after cleanup)" + ); + assert!( + body.pointer(KATA_ANN_PTR) + .map(serde_json::Value::is_null) + .unwrap_or(false), + "tear-down must clear the reference annotation" + ); + send.send_response(kata_node_resp("node-a")); + }); + let outcome = reconcile_kata_config_delivery(&client, "node-a", None).await; + assert_eq!(outcome.unwrap(), KataDeliveryOutcome::Disabled); + srv.await.unwrap(); + } + + #[tokio::test] + async fn test_reconcile_kata_config_delivery_present_opts_in() { + let (client, handle) = mock_client_pair(); + let srv = tokio::spawn(async move { + let mut h = pin!(handle); + let (req, send) = h.next_request().await.expect("expected source CM GET"); + assert_eq!(req.method(), http::Method::GET); + assert!(req + .uri() + .path() + .ends_with("/api/v1/namespaces/5spot-system/configmaps/kata-drop-in")); + send.send_response(kata_cm_resp("kata-drop-in", "5spot-system")); + + let (req, send) = h.next_request().await.expect("expected Node PATCH"); + assert_eq!(req.method(), http::Method::PATCH); + let body = collect_json_body(req.into_body()).await; + assert_eq!( + body.pointer(KATA_LABEL_PTR).and_then(|v| v.as_str()), + Some("enabled"), + "present source must set the opt-in label" + ); + assert!( + body.pointer(KATA_ANN_PTR) + .and_then(|v| v.as_str()) + .is_some(), + "present source must stamp the reference annotation" + ); + send.send_response(kata_node_resp("node-a")); + }); + let kata = sample_kata(); + let outcome = reconcile_kata_config_delivery(&client, "node-a", Some(&kata)).await; + assert_eq!(outcome.unwrap(), KataDeliveryOutcome::Ready); + srv.await.unwrap(); + } + + #[tokio::test] + async fn test_reconcile_kata_config_delivery_source_missing_namespace_present() { + let (client, handle) = mock_client_pair(); + let srv = tokio::spawn(async move { + let mut h = pin!(handle); + let (req, send) = h.next_request().await.expect("expected source CM GET"); + assert!(req + .uri() + .path() + .ends_with("/api/v1/namespaces/5spot-system/configmaps/kata-drop-in")); + send.send_response(kata_404()); + + let (req, send) = h.next_request().await.expect("expected namespace GET"); + assert!(req + .uri() + .path() + .ends_with("/api/v1/namespaces/5spot-system")); + send.send_response(kata_ns_resp("5spot-system")); + + let (req, send) = h.next_request().await.expect("expected Node PATCH (clear)"); + let body = collect_json_body(req.into_body()).await; + assert!( + body.pointer(KATA_LABEL_PTR).is_none(), + "absent source must NOT touch the opt-in label (agent removes it)" + ); + assert!( + body.pointer(KATA_ANN_PTR) + .map(serde_json::Value::is_null) + .unwrap_or(false), + "absent source must clear the reference annotation" + ); + send.send_response(kata_node_resp("node-a")); + }); + let kata = sample_kata(); + let outcome = reconcile_kata_config_delivery(&client, "node-a", Some(&kata)).await; + assert_eq!(outcome.unwrap(), KataDeliveryOutcome::SourceNotFound); + srv.await.unwrap(); + } + + #[tokio::test] + async fn test_reconcile_kata_config_delivery_namespace_missing() { + let (client, handle) = mock_client_pair(); + let srv = tokio::spawn(async move { + let mut h = pin!(handle); + let (_req, send) = h.next_request().await.expect("expected source CM GET"); + send.send_response(kata_404()); + + let (req, send) = h.next_request().await.expect("expected namespace GET"); + assert!(req + .uri() + .path() + .ends_with("/api/v1/namespaces/5spot-system")); + send.send_response(kata_404()); + + let (req, send) = h.next_request().await.expect("expected Node PATCH (clear)"); + let _ = collect_json_body(req.into_body()).await; + send.send_response(kata_node_resp("node-a")); + }); + let kata = sample_kata(); + let outcome = reconcile_kata_config_delivery(&client, "node-a", Some(&kata)).await; + assert_eq!( + outcome.unwrap(), + KataDeliveryOutcome::TargetNamespaceMissing + ); + srv.await.unwrap(); + } + // ======================================================================== // Phase 3 — diff_node_taints pure helper // ======================================================================== @@ -3527,6 +3769,7 @@ mod tests { node_taints: vec![], kill_if_commands: None, kubeconfig_secret_ref: None, + kata: None, }, status: Some(crate::crd::ScheduledMachineStatus { phase: Some("ShuttingDown".to_string()), diff --git a/src/reconcilers/scheduled_machine.rs b/src/reconcilers/scheduled_machine.rs index d66a005..4a86e9e 100644 --- a/src/reconcilers/scheduled_machine.rs +++ b/src/reconcilers/scheduled_machine.rs @@ -863,6 +863,7 @@ async fn handle_active_phase( } provision_reclaim_agent_best_effort(&resource, &ctx, &node_ref).await; reconcile_node_taints_best_effort(&resource, &ctx, &node_ref).await; + provision_kata_config_best_effort(&resource, &ctx, &node_ref).await; } Ok(None) => { debug!(resource = %name, "CAPI Machine not found yet — skipping status enrichment"); @@ -933,6 +934,79 @@ async fn provision_reclaim_agent_best_effort( } } +/// Best-effort projection of the kata-config opt-in label + per-node +/// `ConfigMap` based on the current `spec.kataConfigRef`. +/// +/// Runs from the `Active` phase once a `nodeRef` is known so the projection +/// follows the Node the machine is bound to. The source Secret/ConfigMap is +/// read from the **management** cluster (the SM namespace); the label and +/// per-node ConfigMap are written to the **child** cluster where the +/// `5spot-kata-config-agent` DaemonSet runs. A failure here must not block the +/// reconcile — a missing or stale projection degrades kata config delivery but +/// does not break day-to-day scheduling. Non-fatal errors are logged. +/// +/// `kataConfigRef` cleared (or never set) runs the tear-down path so a removed +/// reference strips the label and deletes the ConfigMap (GitOps semantics), +/// mirroring the reclaim-agent projection. +async fn provision_kata_config_best_effort( + resource: &Arc, + ctx: &Arc, + node_ref: &Option, +) { + let Some(node_name) = node_ref.as_ref().map(|n| n.name.as_str()) else { + return; + }; + if node_name.is_empty() { + return; + } + let kata = resource.spec.kata.as_ref(); + + // Resolve the workload-cluster client. The kata source object and the + // kata-config agent both live on the workload cluster; the controller reads + // the object (read-only) and stamps the Node opt-in there. 5-Spot writes no + // ConfigMap/Secret and creates no namespace (ADR 0002). + let resolved = match ctx.child_clients.resolve(&ctx.client, resource).await { + Ok(c) => c, + Err(e) => { + warn!( + resource = %resource.name_any(), + node = %node_name, + error = %e, + "Failed to resolve workload-cluster client for kata-config delivery (non-fatal)" + ); + return; + } + }; + match super::helpers::reconcile_kata_config_delivery(resolved.client(), node_name, kata).await { + Ok(outcome) => { + use super::helpers::KataDeliveryOutcome::{ + Disabled, Ready, SourceNotFound, TargetNamespaceMissing, + }; + match outcome { + Ready | Disabled => {} + SourceNotFound => warn!( + resource = %resource.name_any(), + node = %node_name, + "kata-config source object not found on workload cluster — Node not opted in (fail-fast)" + ), + TargetNamespaceMissing => warn!( + resource = %resource.name_any(), + node = %node_name, + "kata-config target namespace missing on workload cluster — Node not opted in (fail-fast)" + ), + } + } + Err(e) => { + warn!( + resource = %resource.name_any(), + node = %node_name, + error = %e, + "Failed to reconcile kata-config delivery (non-fatal)" + ); + } + } +} + /// Best-effort reconciliation of user-defined Node taints from /// `spec.nodeTaints` onto the bound Node. /// diff --git a/src/reconcilers/scheduled_machine_tests.rs b/src/reconcilers/scheduled_machine_tests.rs index 3d203c3..9bf00fe 100644 --- a/src/reconcilers/scheduled_machine_tests.rs +++ b/src/reconcilers/scheduled_machine_tests.rs @@ -40,6 +40,7 @@ mod tests { node_taints: vec![], kill_if_commands: None, kubeconfig_secret_ref: None, + kata: None, } } diff --git a/tests/integration_child_kubeconfig.rs b/tests/integration_child_kubeconfig.rs index 814065f..ee49cd9 100644 --- a/tests/integration_child_kubeconfig.rs +++ b/tests/integration_child_kubeconfig.rs @@ -115,6 +115,7 @@ fn make_sm( node_taints: vec![], kill_if_commands: None, kubeconfig_secret_ref: ref_, + kata: None, }, status: None, })