diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 345325a..2a39b72 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -12,4 +12,13 @@ ignore = [ # 2. Calls rand::rng() from within that logger # Neither condition applies to 5-spot. "RUSTSEC-2026-0097", + + # RUSTSEC-2026-0173: proc-macro-error2 is UNMAINTAINED (a maintenance-status + # advisory, not a vulnerability — no known exploit). It is a build-time proc-macro + # pulled transitively via jiff 0.2.29 -> defmt -> defmt-macros, and defmt is jiff's + # OPTIONAL embedded (no_std) feature: `cargo tree -e features -i defmt` reports + # "nothing to print" for the host target, so proc-macro-error2 is never compiled + # into the 5-Spot binary — it exists only as a Cargo.lock entry. Removal awaits + # upstream (kube bumping jiff, or defmt-macros dropping proc-macro-error2). + "RUSTSEC-2026-0173", ] diff --git a/.claude/CHANGELOG.md b/.claude/CHANGELOG.md index db2e286..d9541a8 100644 --- a/.claude/CHANGELOG.md +++ b/.claude/CHANGELOG.md @@ -9,6 +9,126 @@ The format is based on the regulated environment requirements: --- +## [2026-06-24 11:00] - Silence RUSTSEC-2026-0173 (proc-macro-error2 unmaintained) with justification + +**Author:** Erick Bourgeois + +### Changed +- `.cargo/audit.toml` and `deny.toml`: ignore `RUSTSEC-2026-0173` with a documented + justification (kept in sync across both tools). + +### Why +CI `cargo audit` started failing on `proc-macro-error2` being **unmaintained** (a +maintenance-status advisory, not a vulnerability). It is reached only transitively via +`jiff 0.2.29 -> defmt -> defmt-macros`, and `defmt` is jiff's optional embedded/no_std +feature — `cargo tree -e features -i defmt` prints "nothing to print" for the host +target, so the crate is never compiled into the 5-Spot binary; it exists only as a +`Cargo.lock` entry. No runtime or build exposure. Removal awaits upstream (kube bumping +jiff, or defmt-macros dropping proc-macro-error2). Verified `cargo audit` passes. + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [x] Config change only +- [ ] Documentation only + +## [2026-06-24 10:30] - Idempotency sweep: 409 AlreadyExists / duplicate finalizers no longer Error the ScheduledMachine + +**Author:** Erick Bourgeois + +### Changed +- `src/reconcilers/helpers.rs` (`create_dynamic_resource`): treat a `409 AlreadyExists` + from `api.create(...)` as success instead of propagating it. The bootstrap config, + infrastructure object, and CAPI Machine are all created through this helper; a prior + reconcile having created them IS the desired state. +- `src/reconcilers/helpers.rs` (`add_finalizer`): only append our finalizer if it isn't + already present — a double-call can no longer write a duplicate (which would wedge GC). +- `src/reconcilers/helpers_tests.rs`: +3 unit tests (create 409 → Ok; create 500 still + propagates; add_finalizer writes exactly one copy). +- Audited the rest of the codebase for idempotency and confirmed it already holds: the + only other `.create()` is a `SelfSubjectAccessReview` (a virtual auth check, never + 409s); all `.delete()` sites swallow 404; the kata ConfigMap uses `Patch::Apply` (SSA); + Events go through the kube `recorder`; status/spec writes use `Patch::Merge`/`Apply`. + +### Why +On every reconcile after the first, `add_machine_to_cluster` re-issues the create and +got `409 AlreadyExists` (e.g. `k0sworkerconfigs "…" already exists`). That bubbled up as +`MachineCreationFailed`, flipping the ScheduledMachine into the **Error** phase — which +skips ALL Active-phase work, including node-taint reconciliation. On the k0smotron +workshop tier this presented as **Flag 1 (worker joins) passing but Flag 2 (the Node +carries the spot taint) never happening**: the SM was wedged in Error, so it never ran +`handle_active_phase`. Idempotent create is standard controller behavior and fixes it. + +### Impact +- [ ] Breaking change +- [x] Requires cluster rollout +- [ ] Config change only +- [ ] Documentation only + +## [2026-06-24 03:10] - Fix Mermaid flow diagrams broken by special characters + +**Author:** Erick Bourgeois + +### Changed +- `docs/architecture/calm/architecture.json`: escaped Mermaid-hostile characters in + three flow transition descriptions using Mermaid entity codes (which render as the + literal characters): + - `flow-emergency-reclaim` t1: `reclaim-requested="true"` → `reclaim-requested=#quot;true#quot;`. + The embedded double quotes terminated the Mermaid label string early, causing a parse + error that left the **Emergency Reclaim** diagram blank on the docs site. + - `flow-kata-config-delivery` t2: `/etc/5spot/kata-config/` → `…/#lt;key#gt;`. + - `flow-kata-config-delivery` t5: `restart ` → `restart #lt;restartService#gt;`. + Raw `<…>` placeholders were parsed as HTML tags and silently dropped from the rendered label. +- `docs/src/architecture/flows.md`: regenerated via `make calm-diagrams` (auto-generated; + not hand-edited). +- `docs/src/concepts/kata-config-delivery.md`: hand-written `sequenceDiagram` — replaced a + `;` with `,` inside a `Note over Agent,Host: …` label. Mermaid treats `;` as a statement + separator even inside note text, so it split the note mid-string and failed the whole + diagram to parse. + +### Why +The `flows.md.hbs` template emits transition descriptions verbatim into `flowchart` node +labels (`t1["…"]`). Mermaid cannot contain an unescaped `"` inside a quoted label, and +treats `<…>` as HTML — so these characters either broke parsing (Emergency Reclaim) or +ate label text (Kata Config Delivery). Fixing at the source-of-truth `architecture.json` +keeps the diagrams correct after every regeneration. The `phase -> …` arrows in other +flows contain only `>` and render fine, so they were left unchanged. + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [ ] Config change only +- [x] Documentation only + +--- + +## [2026-06-24 02:30] - Surface the real cause of child-cluster connection failures (error_chain) + +**Author:** Erick Bourgeois + +### Changed +- `src/reconcilers/helpers.rs`: new `error_chain(&dyn Error)` helper that flattens an + error's `source()` chain into one string, and wired it into all six child-cluster + failure logs (taint GET/PATCH/reconcile, kata clear/stamp). These previously logged + only the terse top-level `Display` ("client error (Connect)"), hiding the real cause. +- `src/reconcilers/helpers_tests.rs`: +2 unit tests for `error_chain` (multi-level + source chain; single error with no source). + +### Why +A k0smotron-hosted control plane (workshop k0smotron tier) exposes a self-signed API +server cert that **rustls** (5-Spot's TLS backend, kube 3.1 default) rejects where +OpenSSL/curl `-k` accept it — so 5-Spot's child client fails to taint/drain the worker +Node with `client error (Connect)`, with no visible cause. Logging the full chain makes +the exact rustls reason (unknown issuer / SAN mismatch / unsupported signature) visible +so the fix can target it (a proper cert, `tls_server_name`, or `insecure-skip-tls-verify` +on the child kubeconfig — kube already honors the latter via `Config::accept_invalid_certs`). + +### Impact +- [ ] Breaking change +- [ ] Requires cluster rollout +- [ ] Config change only +- [x] Diagnostics only (no behavior change) + ## [2026-06-23 11:45] - Ship the spot-schedule provider binaries in the image (ADR 0009 providers were unrunnable) **Author:** Erick Bourgeois diff --git a/Cargo.lock b/Cargo.lock index e1d5176..a80970a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,6 +169,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.0" @@ -201,15 +207,15 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "cc" -version = "1.2.64" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "shlex", @@ -421,6 +427,38 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -585,7 +623,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "five_spot" -version = "0.1.1" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -631,12 +669,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foldhash" version = "0.2.0" @@ -784,15 +816,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "wasip2", - "wasip3", ] [[package]] @@ -809,9 +839,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -826,15 +856,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -843,7 +864,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -1055,12 +1076,6 @@ dependencies = [ "cc", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1075,8 +1090,6 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", - "serde", - "serde_core", ] [[package]] @@ -1099,10 +1112,11 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" dependencies = [ + "defmt", "jiff-static", "log", "portable-atomic", @@ -1112,9 +1126,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" dependencies = [ "proc-macro2", "quote", @@ -1309,12 +1323,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" version = "0.2.186" @@ -1344,9 +1352,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "matchers" @@ -1431,7 +1439,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "libc", @@ -1666,12 +1674,24 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.37" +name = "proc-macro-error-attr2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", "syn", ] @@ -1690,7 +1710,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags", + "bitflags 2.13.0", "hex", "procfs-core", "rustix 0.38.44", @@ -1702,7 +1722,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags", + "bitflags 2.13.0", "hex", ] @@ -1745,9 +1765,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -1799,7 +1819,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.0", ] [[package]] @@ -1880,7 +1900,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -1893,7 +1913,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.12.1", @@ -1902,9 +1922,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "log", "once_cell", @@ -2020,7 +2040,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2241,9 +2261,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -2262,7 +2282,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2284,7 +2304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -2526,7 +2546,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", - "bitflags", + "bitflags 2.13.0", "bytes", "http", "http-body", @@ -2686,12 +2706,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -2778,16 +2792,7 @@ version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -2835,40 +2840,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -3036,100 +3007,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "zerocopy" version = "0.8.52" diff --git a/deny.toml b/deny.toml index 1361a12..fd6b3ab 100644 --- a/deny.toml +++ b/deny.toml @@ -43,7 +43,13 @@ name = "unicode-ident" [advisories] version = 2 yanked = "deny" -ignore = [] +ignore = [ + # See .cargo/audit.toml for the full justification. proc-macro-error2 is + # UNMAINTAINED (not a vulnerability), reached only via jiff's optional `defmt` + # feature which is never compiled for the host target — kept in sync with + # .cargo/audit.toml so `cargo deny` and `cargo audit` agree. + "RUSTSEC-2026-0173", +] # ───────────────────────────────────────────────────────────────────────────── # Bans — prevent multiple versions of the same crate and wildcard deps. diff --git a/docs/architecture/calm/architecture.json b/docs/architecture/calm/architecture.json index 58a5588..8b1517c 100644 --- a/docs/architecture/calm/architecture.json +++ b/docs/architecture/calm/architecture.json @@ -832,7 +832,7 @@ { "sequence-number": 1, "relationship-unique-id": "rel-reclaim-agent-workload-kube-api", - "description": "Agent detects a process matching spec.killIfCommands via /proc scan and PATCHes the reclaim annotation triple (5spot.finos.org/reclaim-requested=\"true\", /reclaim-reason, /reclaim-requested-at) onto its own Node via the kubelet node-scoped token. Field manager: 5spot-reclaim-agent." + "description": "Agent detects a process matching spec.killIfCommands via /proc scan and PATCHes the reclaim annotation triple (5spot.finos.org/reclaim-requested=#quot;true#quot;, /reclaim-reason, /reclaim-requested-at) onto its own Node via the kubelet node-scoped token. Field manager: 5spot-reclaim-agent." }, { "sequence-number": 2, @@ -895,7 +895,7 @@ { "sequence-number": 2, "relationship-unique-id": "rel-kata-agent-workload-kube-api", - "description": "The kata config agent DaemonSet (scheduled onto the node by the opt-in label) watches its per-node ConfigMap and reads the projected drop-in content at /etc/5spot/kata-config/.", + "description": "The kata config agent DaemonSet (scheduled onto the node by the opt-in label) watches its per-node ConfigMap and reads the projected drop-in content at /etc/5spot/kata-config/#lt;key#gt;.", "direction": "destination-to-source" }, { @@ -911,7 +911,7 @@ { "sequence-number": 5, "relationship-unique-id": "rel-kata-agent-writes-host", - "description": "Agent runs `nsenter -t 1 -m -u -i -n -p -- systemctl restart ` to restart the host k0s service; containerd reloads the new drop-in. The restart SIGKILLs the agent pod; systemd still completes the D-Bus job. Kubelet restarts the agent; on the next loop hashes match the applied annotation, so it is a no-op. Single-cycle convergence." + "description": "Agent runs `nsenter -t 1 -m -u -i -n -p -- systemctl restart #lt;restartService#gt;` to restart the host k0s service; containerd reloads the new drop-in. The restart SIGKILLs the agent pod; systemd still completes the D-Bus job. Kubelet restarts the agent; on the next loop hashes match the applied annotation, so it is a no-op. Single-cycle convergence." } ], "controls": { diff --git a/docs/src/concepts/kata-config-delivery.md b/docs/src/concepts/kata-config-delivery.md index 5f53840..4b191e1 100644 --- a/docs/src/concepts/kata-config-delivery.md +++ b/docs/src/concepts/kata-config-delivery.md @@ -86,7 +86,7 @@ sequenceDiagram Note over Agent: Applied record written BEFORE
the restart — the restart-loop guard Agent->>Host: nsenter -t 1 -m -u -i -n -p --
systemctl restart k0sworker.service Host-->>Agent: containerd bounces → pod SIGKILLed - Note over Agent,Host: kubelet restarts the agent;
next tick: hash matches → no-op.
Single-cycle convergence. + Note over Agent,Host: kubelet restarts the agent,
next tick: hash matches → no-op.
Single-cycle convergence. else hashes match Agent->>Agent: no-op (sleep 30s) end diff --git a/src/reconcilers/helpers.rs b/src/reconcilers/helpers.rs index 3ee38dc..fbcfd30 100644 --- a/src/reconcilers/helpers.rs +++ b/src/reconcilers/helpers.rs @@ -180,7 +180,12 @@ pub async fn add_finalizer( let api: Api = Api::namespaced(ctx.client.clone(), &namespace); let mut finalizers = resource.meta().finalizers.clone().unwrap_or_default(); - finalizers.push(FINALIZER_SCHEDULED_MACHINE.to_string()); + // Idempotent: only add ours if it is not already present, so a double-call (or a + // caller that forgets the has_finalizer() guard) can never write a duplicate + // finalizer — which would otherwise wedge deletion (the GC waits for every copy). + if !finalizers.iter().any(|f| f == FINALIZER_SCHEDULED_MACHINE) { + finalizers.push(FINALIZER_SCHEDULED_MACHINE.to_string()); + } let patch = json!({ "metadata": { @@ -1557,9 +1562,24 @@ async fn create_dynamic_resource( let dyn_obj: kube::core::DynamicObject = serde_json::from_value(obj).map_err(kube::Error::SerdeError)?; - api.create(&kube::api::PostParams::default(), &dyn_obj) - .await?; - Ok(()) + match api + .create(&kube::api::PostParams::default(), &dyn_obj) + .await + { + Ok(_) => Ok(()), + // Idempotent reconciliation: a prior reconcile already created this object + // (bootstrap / infrastructure / Machine), which IS the desired state. Treat + // 409 AlreadyExists as success — otherwise every re-reconcile after the first + // create fails the ScheduledMachine into the Error phase, which then skips all + // Active-phase work (node-taint reconciliation, status projection, …). This + // surfaced on the k0smotron tier as Flag 1 (worker joins) passing but Flag 2 + // (node carries the spot taint) never happening. + Err(kube::Error::Api(ae)) if ae.code == 409 => { + debug!(kind = %kind, "resource already exists; treating create as idempotent success"); + Ok(()) + } + Err(e) => Err(e), + } } /// Split a Kubernetes `apiVersion` string into `(group, version)`. @@ -3139,7 +3159,7 @@ pub async fn reconcile_kata_config_delivery( ) .await .map_err(|e| { - error!(node = %node_name, error = %e, "Failed to clear kata-config reference annotation"); + error!(node = %node_name, error = %e, cause = %error_chain(&e), "Failed to clear kata-config reference annotation"); ReconcilerError::KubeError(e) })?; return Ok(KataDeliveryOutcome::Disabled); @@ -3169,7 +3189,7 @@ pub async fn reconcile_kata_config_delivery( .patch(node_name, &PatchParams::default(), &Patch::Merge(&patch)) .await .map_err(|e| { - error!(node = %node_name, error = %e, "Failed to stamp kata-config Node opt-in"); + error!(node = %node_name, error = %e, cause = %error_chain(&e), "Failed to stamp kata-config Node opt-in"); ReconcilerError::KubeError(e) })?; debug!( @@ -3199,7 +3219,7 @@ pub async fn reconcile_kata_config_delivery( ) .await .map_err(|e| { - error!(node = %node_name, error = %e, "Failed to clear kata-config reference annotation"); + error!(node = %node_name, error = %e, cause = %error_chain(&e), "Failed to clear kata-config reference annotation"); ReconcilerError::KubeError(e) })?; @@ -3354,6 +3374,26 @@ pub fn diff_node_taints( plan } +/// Flatten an error's [`source`](std::error::Error::source) chain into one string. +/// +/// kube/hyper surface child-cluster connection failures with a terse top-level +/// `Display` ("client error (Connect)") that hides the real cause — most often a +/// rustls certificate rejection (unknown issuer, name/SAN mismatch, unsupported +/// signature scheme) when talking to a hosted control plane whose self-signed cert +/// OpenSSL/curl would accept but rustls (stricter) will not. Logging this chain at +/// the failure site makes that cause visible instead of guessing. +#[must_use] +pub fn error_chain(err: &dyn std::error::Error) -> String { + let mut out = err.to_string(); + let mut source = err.source(); + while let Some(cause) = source { + out.push_str(" -> "); + out.push_str(&cause.to_string()); + source = cause.source(); + } + out +} + /// Apply a [`NodeTaintPlan`] to the named Node via a single server-side apply /// PATCH, and write the `5spot.finos.org/applied-taints` ownership annotation. /// @@ -3383,7 +3423,7 @@ pub async fn apply_node_taints( let nodes: Api = Api::all(client.clone()); let current_node = nodes.get(node_name).await.map_err(|e| { - error!(node = %node_name, error = %e, "Failed to GET Node for taint apply"); + error!(node = %node_name, error = %e, cause = %error_chain(&e), "Failed to GET Node for taint apply"); ReconcilerError::KubeError(e) })?; @@ -3460,7 +3500,7 @@ pub async fn apply_node_taints( .patch(node_name, ¶ms, &Patch::Apply(&patch)) .await .map_err(|e| { - error!(node = %node_name, error = %e, "Failed to PATCH Node taints"); + error!(node = %node_name, error = %e, cause = %error_chain(&e), "Failed to PATCH Node taints"); ReconcilerError::KubeError(e) })?; @@ -3539,7 +3579,7 @@ pub async fn reconcile_node_taints( return Ok(NodeTaintReconcileOutcome::NoNodeYet); } Err(e) => { - error!(node = %input.node_name, error = %e, "Failed to GET Node for taint reconcile"); + error!(node = %input.node_name, error = %e, cause = %error_chain(&e), "Failed to GET Node for taint reconcile"); return Err(ReconcilerError::KubeError(e)); } }; diff --git a/src/reconcilers/helpers_tests.rs b/src/reconcilers/helpers_tests.rs index ffedd7d..6123ad7 100644 --- a/src/reconcilers/helpers_tests.rs +++ b/src/reconcilers/helpers_tests.rs @@ -1502,6 +1502,66 @@ mod tests { srv.await.unwrap(); } + // ======================================================================== + // error_chain — flatten an error's source chain for diagnostics + // ======================================================================== + + #[test] + fn test_error_chain_walks_full_source_chain() { + use std::fmt; + #[derive(Debug)] + struct Leaf; + impl fmt::Display for Leaf { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "invalid peer certificate: UnknownIssuer") + } + } + impl std::error::Error for Leaf {} + #[derive(Debug)] + struct Mid(Leaf); + impl fmt::Display for Mid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "tls handshake eof") + } + } + impl std::error::Error for Mid { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } + } + #[derive(Debug)] + struct Top(Mid); + impl fmt::Display for Top { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "client error (Connect)") + } + } + impl std::error::Error for Top { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } + } + // The terse top-level message that hid the real cause, now followed by it. + assert_eq!( + error_chain(&Top(Mid(Leaf))), + "client error (Connect) -> tls handshake eof -> invalid peer certificate: UnknownIssuer" + ); + } + + #[test] + fn test_error_chain_single_error_no_source() { + use std::fmt; + #[derive(Debug)] + struct Solo; + impl fmt::Display for Solo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "boom") + } + } + impl std::error::Error for Solo {} + assert_eq!(error_chain(&Solo), "boom"); + } + // ======================================================================== // machine_api_version_for — bootstrapSpec passthrough + discovery fallback // ======================================================================== @@ -1545,6 +1605,128 @@ mod tests { ); } + // ======================================================================== + // create_dynamic_resource — 409 AlreadyExists is idempotent success + // ======================================================================== + + fn k0sworkerconfig_create_body() -> serde_json::Value { + serde_json::json!({ + "apiVersion": "bootstrap.cluster.x-k8s.io/v1beta1", + "kind": "K0sWorkerConfig", + "metadata": { "name": "business-hours-worker", "namespace": "default" }, + "spec": {} + }) + } + + #[tokio::test] + async fn test_create_dynamic_resource_409_already_exists_is_ok() { + // A re-reconcile re-creates an object a prior reconcile already made. The API + // returns 409 AlreadyExists — which IS the desired state, so create must return + // Ok (otherwise the ScheduledMachine flips to Error and skips Active-phase work + // like node-taint reconciliation). + let (client, handle) = mock_client_pair(); + let srv = tokio::spawn(async move { + let mut h = pin!(handle); + let (req, send) = h.next_request().await.expect("expected POST create"); + assert_eq!(req.method(), http::Method::POST); + send.send_response( + Response::builder() + .status(409) + .header("content-type", "application/json") + .body(Body::from(k8s_error_body( + 409, + "k0sworkerconfigs \"business-hours-worker\" already exists", + ))) + .unwrap(), + ); + }); + let result = create_dynamic_resource( + &client, + "default", + "bootstrap.cluster.x-k8s.io/v1beta1", + "K0sWorkerConfig", + k0sworkerconfig_create_body(), + ) + .await; + assert!( + result.is_ok(), + "409 AlreadyExists must be idempotent success, got: {result:?}" + ); + srv.await.unwrap(); + } + + #[tokio::test] + async fn test_create_dynamic_resource_non_409_propagates() { + // A genuine failure (500) must still surface as an error. + let (client, handle) = mock_client_pair(); + let srv = tokio::spawn(async move { + let mut h = pin!(handle); + let (_req, send) = h.next_request().await.expect("expected POST create"); + send.send_response( + Response::builder() + .status(500) + .header("content-type", "application/json") + .body(Body::from(k8s_error_body(500, "internal error"))) + .unwrap(), + ); + }); + let result = create_dynamic_resource( + &client, + "default", + "bootstrap.cluster.x-k8s.io/v1beta1", + "K0sWorkerConfig", + k0sworkerconfig_create_body(), + ) + .await; + assert!(result.is_err(), "non-409 errors must propagate"); + srv.await.unwrap(); + } + + // ======================================================================== + // add_finalizer — idempotent: never writes a duplicate finalizer + // ======================================================================== + + #[tokio::test] + async fn test_add_finalizer_does_not_duplicate() { + // Called on an SM that ALREADY carries our finalizer, add_finalizer must PATCH + // a finalizers list with exactly one copy — a duplicate would wedge deletion + // (the garbage collector waits for every entry to clear). + let (client, handle) = mock_client_pair(); + let ctx = std::sync::Arc::new(crate::reconcilers::Context::new(client, 0, 1)); + let mut sm = sm_for_delete("sm-fin", "default", "bootstrap.cluster.x-k8s.io/v1beta1"); + sm.metadata.finalizers = Some(vec![ + crate::constants::FINALIZER_SCHEDULED_MACHINE.to_string() + ]); + let resp_body = serde_json::to_vec(&sm).expect("serialize SM"); + let srv = tokio::spawn(async move { + let mut h = pin!(handle); + let (req, send) = h.next_request().await.expect("expected finalizer PATCH"); + assert_eq!(req.method(), http::Method::PATCH); + let body = collect_json_body(req.into_body()).await; + let fins = body["metadata"]["finalizers"] + .as_array() + .cloned() + .unwrap_or_default(); + let count = fins + .iter() + .filter(|f| f.as_str() == Some(crate::constants::FINALIZER_SCHEDULED_MACHINE)) + .count(); + assert_eq!( + count, 1, + "finalizer must appear exactly once, got: {fins:?}" + ); + send.send_response( + Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(resp_body)) + .unwrap(), + ); + }); + let _ = add_finalizer(std::sync::Arc::new(sm), ctx).await; + srv.await.unwrap(); + } + // ======================================================================== // remove_machine_from_cluster — deletes the Machine at the bootstrapSpec version // ========================================================================