From a86dfe902a7d3c1649e58096d5bde383ff3df77c Mon Sep 17 00:00:00 2001 From: Erick Bourgeois Date: Tue, 30 Jun 2026 05:33:55 -0400 Subject: [PATCH] Remove the mTLS check and resumption change, as this was a red herring Signed-off-by: Erick Bourgeois --- .claude/CHANGELOG.md | 41 +++++----- Cargo.lock | 111 +++++++-------------------- Cargo.toml | 8 +- deploy/deployment/networkpolicy.yaml | 20 ++++- docs/src/security/index.md | 2 +- docs/src/security/threat-model.md | 2 +- src/reconcilers/helpers.rs | 5 ++ src/reconcilers/helpers_tests.rs | 8 ++ 8 files changed, 83 insertions(+), 114 deletions(-) diff --git a/.claude/CHANGELOG.md b/.claude/CHANGELOG.md index 86e00ba..0d523f3 100644 --- a/.claude/CHANGELOG.md +++ b/.claude/CHANGELOG.md @@ -9,29 +9,32 @@ The format is based on the regulated environment requirements: --- -## [2026-06-26 10:00] - Switch rustls crypto provider ring -> aws-lc-rs (fixes mTLS to k0smotron-hosted control planes) +## [2026-06-27 14:00] - THE ACTUAL k0smotron taint/drain fix: NetworkPolicy egress + server-side-apply type meta **Author:** Erick Bourgeois ### Changed -- `Cargo.toml`: `kube` now uses `default-features = false` + `["…, "rustls-tls", "aws-lc-rs"]` - instead of the default that pulls the `ring` crypto provider. `ring` leaves the lock; the - rustls TLS stack now runs on aws-lc-rs (BoringSSL-derived). 682 lib tests pass, fmt/clippy clean. - -### Why -5-Spot's child client could not complete the mTLS TLS-1.3 handshake against a k0smotron-hosted -control plane — it stalled after the client-certificate step and timed out -(`client error (Connect) -> deadline has elapsed`), so node taints/drains never happened on the -hosted-CP tier. Proven on a live cluster that `curl --cert/--key` (OpenSSL) and `client-go` -(kubelet/kubectl) complete the *identical* handshake to the *same* endpoint in ~13ms, while -kube-rs/rustls-with-`ring` hangs (captured via `rustls=trace`). aws-lc-rs shares OpenSSL/BoringSSL's -crypto lineage, so it behaves like the clients that work. - -### Build note -CI builds **natively per arch** (`ubuntu-24.04` + `ubuntu-24.04-arm`, plain `cargo build --release`), -so aws-lc-rs compiles natively (cmake + cc, present on GitHub runners) — no cross-toolchain and no -OpenSSL runtime lib in the distroless image, i.e. it avoids the cross-compile problem that motivated -`ring` in the first place. +- `src/reconcilers/helpers.rs` (`apply_node_taints`): the server-side-apply patch was missing + `apiVersion: v1` + `kind: Node`. SSA REQUIRES the object's type meta in the body; without it the + API server rejects with `invalid object type: /, Kind=: BadRequest (400)` and the taint never lands. +- `src/reconcilers/helpers_tests.rs`: the apply-taints test now asserts `apiVersion`+`kind` are in the + patch body (would have caught this). +- `deploy/deployment/networkpolicy.yaml`: egress allowed only `53` + `6443`, which **silently dropped + every child-cluster connection at the network layer** — k0smotron's hosted CP is on the NodePort + `apiPort` (default **30443**), not 6443. Added 30443 to the egress (with guidance for other ports). +- `docs/src/security/index.md`, `docs/src/security/threat-model.md`: the egress-posture rows now + state the policy permits the mgmt apiserver (6443) + child/k0smotron-hosted CPs (30443) + DNS, + matching the updated NetworkPolicy. + +### Why — and a correction +On the k0smotron tier the child client could never taint/drain. Live diagnosis on the playground proved +**two** independent root causes, both above were confirmed by experiment (delete the NetworkPolicy → +connection succeeds; a properly-typed `kubectl apply --server-side` → taint lands). The earlier +TLS-centric changes — `insecure-skip-tls-verify`, the `ring`→`aws-lc-rs` provider swap, and disabling +rustls session resumption — were **misdiagnoses**: the `rustls=trace` "resumption stall" was the +*management* client's normal TLS, while the child client was firewalled before it ever reached TLS. Those +TLS experiments have been **reverted** (back to the default `ring` provider + `Client::try_from`); +this NetworkPolicy + server-side-apply change is the real fix. ### Impact - [ ] Breaking change diff --git a/Cargo.lock b/Cargo.lock index 1bdb6ce..de1a399 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,9 +91,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "async-broadcast" @@ -152,28 +152,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "aws-lc-rs" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "backon" version = "1.6.0" @@ -240,8 +218,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -321,15 +297,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -540,12 +507,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.20" @@ -572,18 +533,18 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "enum-ordinalize" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +checksum = "07f808d588c10e464ea6f7d3eaed500049eff30aaac103460f61828c2d65b3eb" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", @@ -592,18 +553,19 @@ dependencies = [ [[package]] name = "env_filter" -version = "1.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", + "regex", ] [[package]] name = "env_logger" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -732,12 +694,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.32" @@ -1012,9 +968,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "typenum", ] @@ -1157,9 +1113,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" dependencies = [ "defmt", "jiff-static", @@ -1171,30 +1127,20 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -1981,9 +1927,9 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ - "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2017,7 +1963,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2853,9 +2798,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -2866,9 +2811,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2876,9 +2821,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -2889,9 +2834,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] diff --git a/Cargo.toml b/Cargo.toml index 3932833..db1073a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,13 +45,7 @@ path = "src/bin/spot_schedule_capital_markets.rs" # Controller. The feature is "unstable" in the SemVer sense (the API # may change between minor kube-runtime versions) but is the canonical # way to push external triggers into the controller loop. -# TLS: use rustls with the aws-lc-rs crypto provider (BoringSSL-derived) instead of the -# `ring` provider kube pulls by default. `ring` cannot complete the mTLS TLS-1.3 handshake -# against a k0smotron-hosted control plane (it stalls after the client cert step), whereas -# aws-lc-rs — same crypto lineage as OpenSSL/BoringSSL, which client-go and curl use -# successfully against that exact endpoint — does. default-features=false drops the default -# `ring`; we re-add `client` + `rustls-tls` explicitly alongside `aws-lc-rs`. -kube = { version = "3.1", default-features = false, features = ["runtime", "derive", "client", "ws", "unstable-runtime", "rustls-tls", "aws-lc-rs"] } +kube = { version = "3.1", features = ["runtime", "derive", "client", "ws", "unstable-runtime"] } kube-lease-manager = "0.11" k8s-openapi = { version = "0.27", features = ["latest", "schemars"] } tokio = { version = "1", features = ["full"] } diff --git a/deploy/deployment/networkpolicy.yaml b/deploy/deployment/networkpolicy.yaml index e02d50b..703ff35 100644 --- a/deploy/deployment/networkpolicy.yaml +++ b/deploy/deployment/networkpolicy.yaml @@ -5,8 +5,10 @@ # # Ingress: Only Prometheus (monitoring namespace) may scrape metrics on 8080. # Kubelet health probes are allowed on 8081 from any source (k8s internal). -# Egress: Only Kubernetes API server (6443) and DNS (53) are permitted. -# All other egress is denied, preventing data exfiltration. +# Egress: DNS (53) and Kubernetes API servers (6443 management + standard child +# clusters, 30443 k0smotron-hosted control planes) are permitted. All other +# egress is denied, preventing data exfiltration. Child clusters on other +# API ports must add that port to the egress rule below. # # NIST SP 800-53 SC-7: Boundary Protection apiVersion: networking.k8s.io/v1 @@ -45,7 +47,19 @@ spec: - protocol: TCP port: 53 - # Kubernetes API server + # Kubernetes API servers — BOTH the management cluster AND any child/workload + # cluster the controller drives (taint/drain in CAPI / k0smotron topologies). + # * 6443 — the management apiserver (the kube API ClusterIP DNATs to it) and + # standard child clusters. + # * 30443 — a k0smotron-hosted control plane's default NodePort apiPort; the + # child kubeconfig's server is `…:30443`, and the connection is + # evaluated on that pod port, so it MUST be allowed here. + # + # ⚠ A child cluster on any OTHER API port must add it here, or the child-cluster + # client is silently dropped at the network layer ("client error (Connect) -> + # deadline has elapsed") and taints/drains never run. - ports: - protocol: TCP port: 6443 + - protocol: TCP + port: 30443 diff --git a/docs/src/security/index.md b/docs/src/security/index.md index 4ff7756..ce0c302 100644 --- a/docs/src/security/index.md +++ b/docs/src/security/index.md @@ -34,7 +34,7 @@ Per-field validation status and downstream sinks for every attacker-controllable |---|---|---| | Non-root container, read-only rootfs, all caps dropped | ✅ | `deploy/deployment/deployment.yaml` | | Least-privilege RBAC — explicit resources, no wildcards | ✅ | `deploy/deployment/rbac/clusterrole.yaml` | -| NetworkPolicy — egress to API server only | ✅ | `deploy/deployment/networkpolicy.yaml` | +| NetworkPolicy — egress to API servers only (mgmt 6443 + child/k0smotron 30443) and DNS | ✅ | `deploy/deployment/networkpolicy.yaml` | | Admission validation — 13 CEL rules, `failurePolicy: Fail` | ✅ | `deploy/admission/` | | Finalizer cleanup timeout (10 min) — prevents deletion hangs | ✅ | `src/constants.rs` | | Cross-namespace resource creation prevented | ✅ | `namespace` field removed from `EmbeddedResource` | diff --git a/docs/src/security/threat-model.md b/docs/src/security/threat-model.md index 082b8f9..670cded 100644 --- a/docs/src/security/threat-model.md +++ b/docs/src/security/threat-model.md @@ -325,7 +325,7 @@ in `5spot-system`. |---|---| | Kubernetes ResourceQuota | Limit `ScheduledMachine` count per namespace (e.g., max 50) | | `ValidatingAdmissionPolicy` ✅ deployed 2026-04-08 | Validates `bootstrapSpec.apiVersion`, `infrastructureSpec.apiVersion`, `kind` fields, duration format, and day/hour item format at admission time — see `deploy/admission/` | -| NetworkPolicy | Restrict controller pod egress to Kubernetes API server only | +| NetworkPolicy | Restrict controller pod egress to the management API server (6443), child/k0smotron-hosted control planes (NodePort apiPort 30443), and DNS — all other egress denied | | Audit logging | Enable API server audit log at `RequestResponse` level for `scheduledmachines` resources | | RBAC for SM creation | Only grant `create` on `scheduledmachines` to trusted identities; do not grant to end users directly | | Secrets for bootstrap data | Move sensitive bootstrap config out of CR spec into Secrets; reference from spec | diff --git a/src/reconcilers/helpers.rs b/src/reconcilers/helpers.rs index c0979f7..b148c0b 100644 --- a/src/reconcilers/helpers.rs +++ b/src/reconcilers/helpers.rs @@ -3485,7 +3485,12 @@ pub async fn apply_node_taints( .collect(); let annotation_value = serde_json::to_string(&owned).unwrap_or_else(|_| "[]".to_string()); + // Server-side apply REQUIRES the object's type meta in the body; without + // `apiVersion` + `kind` the API server rejects the patch with + // `invalid object type: /, Kind=: BadRequest (400)` and the taint never lands. let patch = json!({ + "apiVersion": "v1", + "kind": "Node", "metadata": { "annotations": { APPLIED_TAINTS_ANNOTATION: annotation_value, diff --git a/src/reconcilers/helpers_tests.rs b/src/reconcilers/helpers_tests.rs index 6123ad7..95c75c9 100644 --- a/src/reconcilers/helpers_tests.rs +++ b/src/reconcilers/helpers_tests.rs @@ -3733,6 +3733,14 @@ mod tests { annotations.get("5spot.finos.org/applied-taints").is_some(), "ownership annotation must be set: {annotations}" ); + // Server-side apply REQUIRES the object's type meta in the body; without + // apiVersion + kind the API server rejects with + // `invalid object type: /, Kind=: BadRequest (400)`. + assert_eq!( + json["apiVersion"], "v1", + "SSA patch must carry apiVersion: {json}" + ); + assert_eq!(json["kind"], "Node", "SSA patch must carry kind: {json}"); send.send_response( Response::builder() .status(200)