diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..345325a --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,15 @@ +# cargo-audit configuration +# https://rustsec.org/ + +[advisories] +# Advisories to ignore (with justification) +ignore = [ + # RUSTSEC-2026-0097: rand unsound with custom logger using rand::rng() + # This is a transitive dependency from warp/tungstenite/kube-client. + # We do not use rand::rng() with a custom logger in our codebase. + # The vulnerability only affects code that: + # 1. Uses a custom global logger (not tracing-subscriber) + # 2. Calls rand::rng() from within that logger + # Neither condition applies to 5-spot. + "RUSTSEC-2026-0097", +] diff --git a/.claude/CHANGELOG.md b/.claude/CHANGELOG.md index 2cc192c..ac35715 100644 --- a/.claude/CHANGELOG.md +++ b/.claude/CHANGELOG.md @@ -462,7 +462,7 @@ Establish baseline CI/CD pipeline for the 5-spot operator using the same firesto - `src/crd.rs`: Removed `namespace` field from `EmbeddedResource` — bootstrap and infrastructure resources are now always created in the ScheduledMachine's own namespace, preventing cross-namespace attacks - `src/crd.rs`: Added `timezone_schema()` with `maxLength: 64` and character-class pattern constraint to block log injection via the timezone field - `src/reconcilers/helpers.rs`: Fixed integer overflow in `parse_duration()` — now uses `checked_mul` and rejects durations exceeding 24 hours (`MAX_DURATION_SECS`) -- `src/reconcilers/helpers.rs`: Added `validate_labels()` — rejects label/annotation keys using reserved prefixes (`kubernetes.io/`, `k8s.io/`, `cluster.x-k8s.io/`, `5spot.io/`) before merging into CAPI Machine resources +- `src/reconcilers/helpers.rs`: Added `validate_labels()` — rejects label/annotation keys using reserved prefixes (`kubernetes.io/`, `k8s.io/`, `cluster.x-k8s.io/`, `5spot.finos.org/`) before merging into CAPI Machine resources - `src/reconcilers/helpers.rs`: Added `validate_api_group()` — enforces an allowlist of permitted API groups for bootstrap and infrastructure embedded resources; blocks core Kubernetes APIs (`v1`, `rbac.authorization.k8s.io/v1`, etc.) - `src/reconcilers/helpers.rs`: Wrapped `remove_machine_from_cluster` in `tokio::time::timeout` inside `handle_deletion` — finalizer cleanup now has a hard 10-minute deadline, preventing indefinite namespace deletion blocks - `src/reconcilers/scheduled_machine.rs`: Added `ValidationError` and `TimeoutError` variants to `ReconcilerError` diff --git a/CHANGELOG.md b/CHANGELOG.md index ea3ea77..fd9f9ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,7 +107,7 @@ Having `version` in MachineSpec created a conceptual mismatch with CAPI architec - `CAPI_MACHINE_API_VERSION_FULL`: "cluster.x-k8s.io/v1beta1" - `CAPI_CLUSTER_NAME_LABEL`: "cluster.x-k8s.io/cluster-name" - `CAPI_RESOURCE_MACHINES`: "machines" - - `API_VERSION_FULL`: "capi.5spot.io/v1alpha1" + - `API_VERSION_FULL`: "5spot.finos.org/v1alpha1" - `src/main.rs`: Implemented actual Prometheus metrics - **Replaced** stub metrics endpoint with proper `prometheus::gather()` integration @@ -204,7 +204,7 @@ where the cluster name is used by bootstrap and infrastructure providers. ### Changed - `src/crd.rs`: **BREAKING** - Converted from k0smotron to CAPI (Cluster API) based architecture - - Changed API group from `5spot.eribourg.dev` to `capi.5spot.io` + - Changed API group from `5spot.eribourg.dev` to `5spot.finos.org` - Added `bootstrap_ref` and `infrastructure_ref` for CAPI Machine creation - Made `files` field non-optional in `MachineSpec` - Renamed `FileContentFrom` types to `ContentSource` and `KeySelector` for consistency @@ -214,7 +214,7 @@ where the cluster name is used by bootstrap and infrastructure providers. - Changed `machine_ref` to use `ObjectReference` instead of custom `MachineRef` type - `src/constants.rs`: Updated all constants for CAPI architecture - - Changed API group constants to `capi.5spot.io` + - Changed API group constants to `5spot.finos.org` - Updated finalizer name to use new API group - Added CAPI Machine phase constants (Pending, Active, ShuttingDown, Inactive, Disabled, Terminated, Error) - Added CAPI API version constants (`cluster.x-k8s.io/v1beta1`) @@ -256,7 +256,7 @@ Complete architectural shift from k0smotron-specific machine management to stand 2. **Delete old CRD**: `kubectl delete crd scheduledmachines.5spot.eribourg.dev` 3. **Deploy new CRD**: `kubectl apply -f deploy/crds/scheduledmachine.yaml` 4. **Update all ScheduledMachine manifests** to new schema: - - Change `apiVersion` from `5spot.eribourg.dev/v1alpha1` to `capi.5spot.io/v1alpha1` + - Change `apiVersion` from `5spot.eribourg.dev/v1alpha1` to `5spot.finos.org/v1alpha1` - Add `bootstrapRef` and `infrastructureRef` fields - Update `files` structure (now required, not optional) - Ensure all file paths are absolute and start with `/` diff --git a/Cargo.lock b/Cargo.lock index d643a25..b6aafad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,9 +210,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.59" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "shlex", @@ -842,6 +842,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "headers" version = "0.3.9" @@ -1004,9 +1010,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "c2b52f86d1d4bc0d6b4e6826d960b1b333217e07d36b882dca570a5e1c48895b" dependencies = [ "http 1.4.0", "hyper 1.9.0", @@ -1014,7 +1020,6 @@ dependencies = [ "log", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -1200,12 +1205,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -1254,9 +1259,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.94" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ "once_cell", "wasm-bindgen", @@ -1400,7 +1405,7 @@ checksum = "159a11dbcf655bb619f6479081e565ea889bb860c5d7d830f3ad90adc74fb90a" dependencies = [ "k8s-openapi", "kube", - "rand 0.9.2", + "rand 0.9.4", "thiserror 2.0.18", "tokio", "tracing", @@ -1447,9 +1452,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.184" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "linux-raw-sys" @@ -1798,9 +1803,9 @@ checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" [[package]] name = "predicates-tree" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" dependencies = [ "predicates-core", "termtree", @@ -1918,9 +1923,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2073,9 +2078,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "log", "once_cell", @@ -2109,9 +2114,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" dependencies = [ "ring", "rustls-pki-types", @@ -2563,9 +2568,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.51.0" +version = "1.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" +checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" dependencies = [ "bytes", "libc", @@ -2832,7 +2837,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "sha1", "thiserror 2.0.18", "utf-8", @@ -2986,9 +2991,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -2999,9 +3004,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3009,9 +3014,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -3022,9 +3027,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] diff --git a/README.md b/README.md index 34b6c20..f0e7f73 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![FINOS - Incubating](https://cdn.jsdelivr.net/gh/finos/contrib-toolbox@master/images/badge-incubating.svg)](https://community.finos.org/docs/governance/Software-Projects/stages/incubating) -# 5-Spot Machine Scheduler +# 5-Spot Logo 5-Spot Machine Scheduler A cloud-native Kubernetes controller for managing time-based machine scheduling on physical nodes using Cluster API (CAPI). @@ -81,7 +81,7 @@ kubectl apply -f examples/scheduledmachine-basic.yaml ## Example ```yaml -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: business-hours-machine @@ -334,7 +334,7 @@ make security-scan-local This creates a `.git/hooks/pre-commit` hook that scans staged changes before each commit. 3. **Configure allowlists** (for false positives): - + Edit `.gitleaks.toml` to add exceptions: ```toml [allowlist] diff --git a/deploy/admission/validatingadmissionpolicy.yaml b/deploy/admission/validatingadmissionpolicy.yaml index 3616057..577d953 100644 --- a/deploy/admission/validatingadmissionpolicy.yaml +++ b/deploy/admission/validatingadmissionpolicy.yaml @@ -13,7 +13,6 @@ # Apply with: # kubectl apply -f deploy/admission/validatingadmissionpolicy.yaml # kubectl apply -f deploy/admission/validatingadmissionpolicybinding.yaml ---- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy metadata: @@ -28,10 +27,10 @@ spec: matchConstraints: resourceRules: - - apiGroups: ["capi.5spot.io"] + - apiGroups: ["5spot.finos.org"] apiVersions: ["v1alpha1"] - resources: ["scheduledmachines"] - operations: ["CREATE", "UPDATE"] + resources: ["scheduledmachines"] + operations: ["CREATE", "UPDATE"] validations: diff --git a/deploy/admission/validatingadmissionpolicybinding.yaml b/deploy/admission/validatingadmissionpolicybinding.yaml index 3c3c972..05955b9 100644 --- a/deploy/admission/validatingadmissionpolicybinding.yaml +++ b/deploy/admission/validatingadmissionpolicybinding.yaml @@ -9,7 +9,6 @@ # Apply after the ValidatingAdmissionPolicy: # kubectl apply -f deploy/admission/validatingadmissionpolicy.yaml # kubectl apply -f deploy/admission/validatingadmissionpolicybinding.yaml ---- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding metadata: diff --git a/deploy/crds/scheduledmachine.yaml b/deploy/crds/scheduledmachine.yaml index 2a2d5ac..24e31b0 100644 --- a/deploy/crds/scheduledmachine.yaml +++ b/deploy/crds/scheduledmachine.yaml @@ -1,323 +1,316 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: scheduledmachines.5spot.io + name: scheduledmachines.5spot.finos.org spec: - group: 5spot.io + group: 5spot.finos.org names: categories: [] kind: ScheduledMachine plural: scheduledmachines shortNames: - - sm + - sm singular: scheduledmachine scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .status.phase - name: Phase - type: string - - jsonPath: .status.inSchedule - name: InSchedule - type: boolean - - jsonPath: .spec.schedule.enabled - name: Enabled - type: boolean - - jsonPath: .spec.killSwitch - name: KillSwitch - type: boolean - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: Auto-generated derived type for ScheduledMachineSpec via `CustomResource` - properties: - spec: - properties: - bootstrapSpec: - description: |- - Inline bootstrap configuration spec (e.g., `K0sWorkerConfig`) - This resource will be created when the schedule is active - properties: - apiVersion: - description: API version of the resource (e.g., "bootstrap.cluster.x-k8s.io/v1beta1") - type: string - kind: - description: Kind of the resource (e.g., `K0sWorkerConfig`, `RemoteMachine`) - type: string - spec: - additionalProperties: true - description: |- - The spec of the resource (provider-specific) - This is an arbitrary JSON object whose schema depends on the kind - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - apiVersion - - kind - - spec - type: object - clusterName: - description: Name of the CAPI cluster this machine belongs to - type: string - gracefulShutdownTimeout: - default: 5m - description: Timeout for graceful machine shutdown (e.g., "5m", "10s") - type: string - infrastructureSpec: - description: |- - Inline infrastructure configuration spec (e.g., `RemoteMachine`) - This resource will be created when the schedule is active - properties: - apiVersion: - description: API version of the resource (e.g., "bootstrap.cluster.x-k8s.io/v1beta1") - type: string - kind: - description: Kind of the resource (e.g., `K0sWorkerConfig`, `RemoteMachine`) - type: string - spec: - additionalProperties: true - description: |- - The spec of the resource (provider-specific) - This is an arbitrary JSON object whose schema depends on the kind - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - apiVersion - - kind - - spec - type: object - killSwitch: - default: false - description: When true, immediately removes the machine from cluster - type: boolean - machineTemplate: - description: |- - Optional configuration for the created CAPI Machine - If not specified, creates a Machine with default labels/annotations - nullable: true - properties: - annotations: - additionalProperties: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.inSchedule + name: InSchedule + type: boolean + - jsonPath: .spec.schedule.enabled + name: Enabled + type: boolean + - jsonPath: .spec.killSwitch + name: KillSwitch + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for ScheduledMachineSpec via `CustomResource` + properties: + spec: + properties: + bootstrapSpec: + description: |- + Inline bootstrap configuration spec (e.g., `K0sWorkerConfig`) + This resource will be created when the schedule is active + properties: + apiVersion: + description: API version of the resource (e.g., 'bootstrap.cluster.x-k8s.io/v1beta1') type: string - default: {} - description: Annotations to apply to the created Machine - type: object - labels: - additionalProperties: + kind: + description: Kind of the resource (e.g., 'K0sWorkerConfig', 'RemoteMachine') + type: string + spec: + description: Provider-specific configuration + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - apiVersion + - kind + - spec + type: object + clusterName: + description: Name of the CAPI cluster this machine belongs to + type: string + gracefulShutdownTimeout: + default: 5m + description: Timeout for graceful machine shutdown (e.g., "5m", "10s") + type: string + infrastructureSpec: + description: |- + Inline infrastructure configuration spec (e.g., `RemoteMachine`) + This resource will be created when the schedule is active + properties: + apiVersion: + description: API version of the resource (e.g., 'bootstrap.cluster.x-k8s.io/v1beta1') + type: string + kind: + description: Kind of the resource (e.g., 'K0sWorkerConfig', 'RemoteMachine') + type: string + spec: + description: Provider-specific configuration + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - apiVersion + - kind + - spec + type: object + killSwitch: + default: false + description: When true, immediately removes the machine from cluster + type: boolean + machineTemplate: + description: |- + Optional configuration for the created CAPI Machine + If not specified, creates a Machine with default labels/annotations + nullable: true + properties: + annotations: + additionalProperties: + type: string + default: {} + description: Annotations to apply to the created Machine + type: object + labels: + additionalProperties: + type: string + default: {} + description: Labels to apply to the created Machine + type: object + type: object + nodeDrainTimeout: + default: 5m + description: Timeout for draining the node before deletion (e.g., "5m", "10m") + type: string + priority: + default: 50 + description: Priority for machine scheduling (higher values = higher priority) + format: uint8 + maximum: 255.0 + minimum: 0.0 + type: integer + schedule: + description: Machine scheduling configuration + properties: + cron: + description: |- + Cron expression for the schedule (e.g., "0 9-17 * * 1-5" for Mon-Fri 9am-5pm) + If specified, takes precedence over daysOfWeek/hoursOfDay + nullable: true + type: string + daysOfWeek: + default: [] + description: |- + Days of the week when machine should be active + Supports: individual days (mon), ranges (mon-fri), combinations (mon-wed,fri-sun) + Ignored if cron is specified + items: + type: string + type: array + enabled: + default: true + description: Whether the schedule is enabled + type: boolean + hoursOfDay: + default: [] + description: |- + Hours when machine should be active (0-23) + Supports: individual hours (9), ranges (9-17), combinations (0-9,18-23) + Ignored if cron is specified + items: + type: string + type: array + timezone: + default: UTC + description: |- + Timezone for the schedule (e.g., "UTC", "America/New\_York") + Maximum length of 64 characters. + maxLength: 64 + pattern: ^[A-Za-z][A-Za-z0-9_+\-/]*$ type: string - default: {} - description: Labels to apply to the created Machine + type: object + required: + - bootstrapSpec + - clusterName + - infrastructureSpec + - schedule + type: object + status: + nullable: true + properties: + bootstrapRef: + description: Reference to the created bootstrap resource + nullable: true + properties: + apiVersion: + description: API version of the referenced object + type: string + kind: + description: Kind of the referenced object + type: string + name: + description: Name of the referenced object + type: string + namespace: + description: Namespace of the referenced object + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + conditions: + default: [] + description: Standard Kubernetes conditions + items: + properties: + lastTransitionTime: + description: Last transition time (RFC3339 format) + type: string + message: + description: Human-readable message + type: string + reason: + description: One-word reason in CamelCase + type: string + status: + description: 'Status: "True", "False", or "Unknown"' + enum: + - 'True' + - 'False' + - Unknown + type: string + type: + description: Type of condition (e.g., "Ready", "`MachineReady`", "`ReferencesValid`") + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type type: object - type: object - nodeDrainTimeout: - default: 5m - description: Timeout for draining the node before deletion (e.g., "5m", "10m") - type: string - priority: - default: 50 - description: Priority for machine scheduling (higher values = higher priority) - format: uint8 - maximum: 255.0 - minimum: 0.0 - type: integer - schedule: - description: Machine scheduling configuration - properties: - cron: - description: |- - Cron expression for the schedule (e.g., "0 9-17 * * 1-5" for Mon-Fri 9am-5pm) - If specified, takes precedence over daysOfWeek/hoursOfDay - nullable: true - type: string - daysOfWeek: - default: [] - description: |- - Days of the week when machine should be active - Supports: individual days (mon), ranges (mon-fri), combinations (mon-wed,fri-sun) - Ignored if cron is specified - items: + type: array + inSchedule: + default: false + description: Whether machine is currently in scheduled window + type: boolean + infrastructureRef: + description: Reference to the created infrastructure resource + nullable: true + properties: + apiVersion: + description: API version of the referenced object + type: string + kind: + description: Kind of the referenced object type: string - type: array - enabled: - default: true - description: Whether the schedule is enabled - type: boolean - hoursOfDay: - default: [] - description: |- - Hours when machine should be active (0-23) - Supports: individual hours (9), ranges (9-17), combinations (0-9,18-23) - Ignored if cron is specified - items: + name: + description: Name of the referenced object type: string - type: array - timezone: - default: UTC - description: |- - Timezone for the schedule (e.g., "UTC", "America/New\_York") - Maximum length of 64 characters. - maxLength: 64 - pattern: ^[A-Za-z][A-Za-z0-9_+\-/]*$ - type: string - type: object - required: - - bootstrapSpec - - clusterName - - infrastructureSpec - - schedule - type: object - status: - nullable: true - properties: - bootstrapRef: - description: Reference to the created bootstrap resource - nullable: true - properties: - apiVersion: - description: API version of the referenced object - type: string - kind: - description: Kind of the referenced object - type: string - name: - description: Name of the referenced object - type: string - namespace: - description: Namespace of the referenced object - nullable: true - type: string - required: - - apiVersion - - kind - - name - type: object - conditions: - default: [] - description: Standard Kubernetes conditions - items: + namespace: + description: Namespace of the referenced object + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + lastScheduledTime: + description: Last time a machine was created (RFC3339 format) + nullable: true + type: string + machineRef: + description: Reference to the created CAPI Machine + nullable: true properties: - lastTransitionTime: - description: Last transition time (RFC3339 format) + apiVersion: + description: API version of the referenced object type: string - message: - description: Human-readable message + kind: + description: Kind of the referenced object type: string - reason: - description: One-word reason in CamelCase + name: + description: Name of the referenced object type: string - status: - description: 'Status: "True", "False", or "Unknown"' - enum: - - 'True' - - 'False' - - Unknown + namespace: + description: Namespace of the referenced object + nullable: true type: string - type: - description: Type of condition (e.g., "Ready", "`MachineReady`", "`ReferencesValid`") + required: + - apiVersion + - kind + - name + type: object + message: + description: Human-readable status message + nullable: true + type: string + nextActivation: + description: Next scheduled activation time (RFC3339 format) + nullable: true + type: string + nextCleanup: + description: Time when machine will be cleaned up (RFC3339 format) + nullable: true + type: string + nodeRef: + description: Reference to the Kubernetes Node (once provisioned) + nullable: true + properties: + name: + description: Name of the referenced object type: string required: - - lastTransitionTime - - message - - reason - - status - - type + - name type: object - type: array - inSchedule: - default: false - description: Whether machine is currently in scheduled window - type: boolean - infrastructureRef: - description: Reference to the created infrastructure resource - nullable: true - properties: - apiVersion: - description: API version of the referenced object - type: string - kind: - description: Kind of the referenced object - type: string - name: - description: Name of the referenced object - type: string - namespace: - description: Namespace of the referenced object - nullable: true - type: string - required: - - apiVersion - - kind - - name - type: object - lastScheduledTime: - description: Last time a machine was created (RFC3339 format) - nullable: true - type: string - machineRef: - description: Reference to the created CAPI Machine - nullable: true - properties: - apiVersion: - description: API version of the referenced object - type: string - kind: - description: Kind of the referenced object - type: string - name: - description: Name of the referenced object - type: string - namespace: - description: Namespace of the referenced object - nullable: true - type: string - required: - - apiVersion - - kind - - name - type: object - message: - description: Human-readable status message - nullable: true - type: string - nextActivation: - description: Next scheduled activation time (RFC3339 format) - nullable: true - type: string - nextCleanup: - description: Time when machine will be cleaned up (RFC3339 format) - nullable: true - type: string - nodeRef: - description: Reference to the Kubernetes Node (once provisioned) - nullable: true - properties: - name: - description: Name of the referenced object - type: string - required: - - name - type: object - observedGeneration: - description: Observed generation for change detection - format: int64 - nullable: true - type: integer - phase: - description: |- - Current phase of the machine lifecycle - Values: Pending, Active, `ShuttingDown`, Inactive, Disabled, Terminated, Error - nullable: true - type: string - type: object - required: - - spec - title: ScheduledMachine - type: object - served: true - storage: true - subresources: - status: {} - + observedGeneration: + description: Observed generation for change detection + format: int64 + nullable: true + type: integer + phase: + description: |- + Current phase of the machine lifecycle + Values: Pending, Active, `ShuttingDown`, Inactive, Disabled, Terminated, Error + nullable: true + type: string + type: object + required: + - spec + title: ScheduledMachine + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/deployment/configmap.yaml b/deploy/deployment/configmap.yaml index 106b9aa..7289bca 100644 --- a/deploy/deployment/configmap.yaml +++ b/deploy/deployment/configmap.yaml @@ -1,4 +1,3 @@ ---- # 5-Spot Controller ConfigMap # Configuration options for the controller apiVersion: v1 @@ -14,7 +13,7 @@ data: # Logging configuration RUST_LOG: "info,kube=warn,hyper=warn,tower=warn" RUST_LOG_FORMAT: "json" - + # Metrics and health ports METRICS_PORT: "8080" HEALTH_PORT: "8081" diff --git a/deploy/deployment/pdb.yaml b/deploy/deployment/pdb.yaml index d947a36..7b54fbe 100644 --- a/deploy/deployment/pdb.yaml +++ b/deploy/deployment/pdb.yaml @@ -1,4 +1,3 @@ ---- # 5-Spot Controller PodDisruptionBudget # Ensures at least one replica is available during voluntary disruptions apiVersion: policy/v1 diff --git a/deploy/deployment/rbac/clusterrole.yaml b/deploy/deployment/rbac/clusterrole.yaml index 2246473..178c0cb 100644 --- a/deploy/deployment/rbac/clusterrole.yaml +++ b/deploy/deployment/rbac/clusterrole.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/component: controller rules: # ScheduledMachine CRD management - - apiGroups: ["5spot.io"] + - apiGroups: ["5spot.finos.org"] resources: ["scheduledmachines", "scheduledmachines/status"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/deploy/deployment/service.yaml b/deploy/deployment/service.yaml index aa7123c..814b30f 100644 --- a/deploy/deployment/service.yaml +++ b/deploy/deployment/service.yaml @@ -1,4 +1,3 @@ ---- # 5-Spot Controller Service # Exposes metrics endpoint for Prometheus scraping apiVersion: v1 diff --git a/deploy/monitoring/servicemonitor.yaml b/deploy/monitoring/servicemonitor.yaml index 40955fe..82a4ab7 100644 --- a/deploy/monitoring/servicemonitor.yaml +++ b/deploy/monitoring/servicemonitor.yaml @@ -1,4 +1,3 @@ ---- # 5-Spot Controller ServiceMonitor # For Prometheus Operator to automatically discover and scrape metrics apiVersion: monitoring.coreos.com/v1 diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 71a0f07..cd376c5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -87,8 +87,8 @@ theme: - content.tooltips # Improved tooltips # Logo and favicon - logo: images/favicons/favicon.png - favicon: images/favicons/favicon.png + logo: images/5-spot-icon.svg + favicon: images/favicon.ico icon: repo: fontawesome/brands/github edit: material/pencil diff --git a/docs/reference/api.md b/docs/reference/api.md index 17cbd4d..6979bf3 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -7,14 +7,14 @@ automatically added to and removed from a k0smotron cluster based on a time sche ### API Group and Version -- **API Group**: `capi.5spot.io` +- **API Group**: `5spot.finos.org` - **API Version**: `v1alpha1` - **Kind**: `ScheduledMachine` ### Example ```yaml -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: example-machine diff --git a/docs/src/advanced/capi-integration.md b/docs/src/advanced/capi-integration.md index b120099..d982a76 100644 --- a/docs/src/advanced/capi-integration.md +++ b/docs/src/advanced/capi-integration.md @@ -29,7 +29,7 @@ metadata: name: scheduled-machine-worker-xyz namespace: default ownerReferences: - - apiVersion: capi.5spot.io/v1alpha1 + - apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine name: scheduled-machine spec: diff --git a/docs/src/concepts/scheduled-machine.md b/docs/src/concepts/scheduled-machine.md index 8ddf068..1c084c8 100644 --- a/docs/src/concepts/scheduled-machine.md +++ b/docs/src/concepts/scheduled-machine.md @@ -13,7 +13,7 @@ A ScheduledMachine defines: ## Example ```yaml -apiVersion: 5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: business-hours-worker diff --git a/docs/src/images/5-spot-icon.svg b/docs/src/images/5-spot-icon.svg new file mode 100644 index 0000000..dfab8e9 --- /dev/null +++ b/docs/src/images/5-spot-icon.svg @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/images/favicon.ico b/docs/src/images/favicon.ico new file mode 100644 index 0000000..1911422 Binary files /dev/null and b/docs/src/images/favicon.ico differ diff --git a/docs/src/index.md b/docs/src/index.md index 05e66bf..63e27b5 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,4 +1,4 @@ -# Introduction +# 5-Spot Logo Introduction ## CI/CD @@ -56,7 +56,7 @@ Traditional infrastructure management involves: Here's how simple it is to create a scheduled machine: ```yaml -apiVersion: 5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: business-hours-machine @@ -111,27 +111,27 @@ kubectl apply -f scheduled-machine.yaml ```mermaid stateDiagram-v2 [*] --> Pending: Resource Created - + Pending --> Active: Schedule active & machine created Pending --> Inactive: Outside schedule window Pending --> Disabled: Schedule disabled - + Active --> ShuttingDown: Schedule ends Active --> Terminated: Kill switch activated Active --> Error: Provisioning error - + ShuttingDown --> Inactive: Grace period complete ShuttingDown --> Error: Shutdown error - + Inactive --> Active: Schedule window starts Inactive --> Disabled: Schedule disabled - + Disabled --> Pending: Schedule re-enabled - + Terminated --> Pending: Kill switch deactivated - + Error --> Pending: Recovery / Retry - + note right of Pending: Initial state after creation note right of Active: Machine is part of cluster note right of ShuttingDown: Grace period in progress diff --git a/docs/src/installation/controller.md b/docs/src/installation/controller.md index 1d92c00..6a4651a 100644 --- a/docs/src/installation/controller.md +++ b/docs/src/installation/controller.md @@ -39,7 +39,7 @@ kind: ClusterRole metadata: name: 5spot-controller rules: - - apiGroups: ["5spot.io"] + - apiGroups: ["5spot.finos.org"] resources: ["scheduledmachines", "scheduledmachines/status"] verbs: ["*"] - apiGroups: ["cluster.x-k8s.io"] diff --git a/docs/src/installation/crds.md b/docs/src/installation/crds.md index c7838f4..0094022 100644 --- a/docs/src/installation/crds.md +++ b/docs/src/installation/crds.md @@ -21,21 +21,21 @@ kubectl apply -f https://raw.githubusercontent.com/finos/5-spot/main/deploy/crds ### Verify Installation ```bash -kubectl get crds scheduledmachines.capi.5spot.io +kubectl get crds scheduledmachines.5spot.finos.org ``` Expected output: ``` NAME CREATED AT -scheduledmachines.capi.5spot.io 2025-01-01T00:00:00Z +scheduledmachines.5spot.finos.org 2025-01-01T00:00:00Z ``` ## CRD Schema The CRD defines the following structure: -- **apiVersion**: `capi.5spot.io/v1alpha1` +- **apiVersion**: `5spot.finos.org/v1alpha1` - **kind**: `ScheduledMachine` - **spec**: Configuration for scheduling and machine management - **status**: Current state and conditions diff --git a/docs/src/installation/quickstart.md b/docs/src/installation/quickstart.md index d6a4412..84ff598 100644 --- a/docs/src/installation/quickstart.md +++ b/docs/src/installation/quickstart.md @@ -34,7 +34,7 @@ kubectl get crds | grep 5spot Create a file named `my-scheduled-machine.yaml`: ```yaml -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: my-first-scheduled-machine diff --git a/docs/src/operations/configuration.md b/docs/src/operations/configuration.md index e165e79..2a50f45 100644 --- a/docs/src/operations/configuration.md +++ b/docs/src/operations/configuration.md @@ -126,10 +126,10 @@ metadata: name: 5spot-controller rules: # ScheduledMachine resources - - apiGroups: ["capi.5spot.io"] + - apiGroups: ["5spot.finos.org"] resources: ["scheduledmachines"] verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["capi.5spot.io"] + - apiGroups: ["5spot.finos.org"] resources: ["scheduledmachines/status"] verbs: ["get", "update", "patch"] diff --git a/docs/src/operations/troubleshooting.md b/docs/src/operations/troubleshooting.md index 4c8d071..f874822 100644 --- a/docs/src/operations/troubleshooting.md +++ b/docs/src/operations/troubleshooting.md @@ -199,7 +199,7 @@ kubectl logs -n 5spot-system -l app=5spot-controller -f | \ - Once the root cause is fixed, the next successful reconciliation resets the counter - If the resource is stuck at max backoff (300 s), fix the underlying issue and patch the resource to trigger an immediate reconcile: ```bash - kubectl annotate scheduledmachine 5spot.io/force-reconcile="$(date -u +%s)" --overwrite + kubectl annotate scheduledmachine 5spot.finos.org/force-reconcile="$(date -u +%s)" --overwrite ``` ## Error Messages diff --git a/docs/src/reference/api.md b/docs/src/reference/api.md index 17cbd4d..ea481e6 100644 --- a/docs/src/reference/api.md +++ b/docs/src/reference/api.md @@ -7,19 +7,20 @@ automatically added to and removed from a k0smotron cluster based on a time sche ### API Group and Version -- **API Group**: `capi.5spot.io` +- **API Group**: `5spot.finos.org` - **API Version**: `v1alpha1` - **Kind**: `ScheduledMachine` ### Example ```yaml -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: - name: example-machine + name: example-spot-machine namespace: default spec: + clusterName: my-cluster schedule: daysOfWeek: - mon-fri @@ -27,25 +28,29 @@ spec: - 9-17 timezone: America/New_York enabled: true - machine: - address: 192.168.1.100 - user: admin - port: 22 - useSudo: false - files: [] - bootstrapRef: + bootstrapSpec: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: worker-bootstrap-config - namespace: default - infrastructureRef: + kind: K0sWorkerConfig + spec: + version: v1.32.8+k0s.0 + downloadURL: https://github.com/k0sproject/k0s/releases/download/v1.32.8+k0s.0/k0s-v1.32.8+k0s.0-amd64 + infrastructureSpec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: MachineTemplate - name: worker-machine-template - namespace: default - clusterName: my-cluster + kind: RemoteMachine + spec: + address: 192.168.1.100 + port: 22 + user: root + sshKeyRef: + name: my-ssh-key + machineTemplate: + labels: + node-role.kubernetes.io/worker: spot + annotations: + example.com/scheduled-by: 5spot priority: 50 gracefulShutdownTimeout: 5m + nodeDrainTimeout: 5m killSwitch: false ``` @@ -66,24 +71,40 @@ Machine scheduling configuration. - **enabled** (optional, boolean, default: `true`): Whether the schedule is enabled. -#### machine +#### clusterName -Machine specification for k0smotron. +(required, string) Name of the CAPI cluster this machine belongs to. -- **address** (required, string): IP address of the machine. +#### bootstrapSpec -- **user** (required, string): Username for SSH connection. +(required, object) Inline bootstrap configuration that will be created when the schedule is active. +This is a fully unstructured object that must contain: -- **port** (optional, integer, default: `22`): SSH port. +- **apiVersion** (required, string): API version of the bootstrap resource (e.g., `bootstrap.cluster.x-k8s.io/v1beta1`) +- **kind** (required, string): Kind of the bootstrap resource (e.g., `K0sWorkerConfig`, `KubeadmConfig`) +- **spec** (required, object): Provider-specific configuration for the bootstrap resource -- **useSudo** (optional, boolean, default: `false`): Whether to use sudo for commands. +The controller validates that the apiVersion belongs to an allowed bootstrap API group. -- **files** (optional, array): Files to be passed to user_data upon creation. +#### infrastructureSpec -#### clusterName +(required, object) Inline infrastructure configuration that will be created when the schedule is active. +This is a fully unstructured object that must contain: -(required, string) Name of the CAPI cluster this machine belongs to. -The bootstrap and infrastructure refs must be configured for this cluster. +- **apiVersion** (required, string): API version of the infrastructure resource (e.g., `infrastructure.cluster.x-k8s.io/v1beta1`) +- **kind** (required, string): Kind of the infrastructure resource (e.g., `RemoteMachine`, `AWSMachine`) +- **spec** (required, object): Provider-specific configuration for the infrastructure resource + +The controller validates that the apiVersion belongs to an allowed infrastructure API group. + +#### machineTemplate + +(optional, object) Configuration for the created CAPI Machine resource. + +- **labels** (optional, map of string to string): Labels to apply to the created Machine +- **annotations** (optional, map of string to string): Annotations to apply to the created Machine + +Note: Labels and annotations using reserved prefixes (`5spot.finos.org/`, `cluster.x-k8s.io/`) are rejected. #### priority @@ -96,6 +117,11 @@ operator instances. (optional, string, default: `5m`) Timeout for graceful machine shutdown. Format: `` where unit is `s` (seconds), `m` (minutes), or `h` (hours). +#### nodeDrainTimeout + +(optional, string, default: `5m`) Timeout for draining the node before deletion. +Format: `` where unit is `s` (seconds), `m` (minutes), or `h` (hours). + #### killSwitch (optional, boolean, default: `false`) When true, immediately removes the machine @@ -107,12 +133,12 @@ from the cluster and takes it out of rotation, bypassing the grace period. Current phase of the machine lifecycle. Possible values: -- **Pending**: Initial state, schedule not yet evaluated -- **Scheduled**: Machine is within scheduled time window but not yet active +- **Pending**: Initial state, awaiting schedule evaluation - **Active**: Machine is running and part of the cluster -- **UnScheduled**: Machine is outside scheduled time window -- **Removing**: Machine is being removed from cluster -- **Inactive**: Machine has been removed and is inactive +- **ShuttingDown**: Machine is being gracefully removed (draining, etc.) +- **Inactive**: Machine is outside scheduled time window and has been removed +- **Disabled**: Schedule is disabled, machine is not active +- **Terminated**: Machine has been permanently removed - **Error**: An error occurred during processing #### conditions @@ -125,22 +151,14 @@ Array of condition objects with the following fields: - **message**: Human-readable message - **lastTransitionTime**: Last time the condition transitioned -#### machineRef - -Reference to the actual Machine resource: - -- **name**: Machine name -- **namespace**: Machine namespace -- **uid**: Machine UID - -#### lastScheduleTime +#### inSchedule -Last time the machine was scheduled and activated. +(boolean) Whether the machine is currently within its scheduled time window. -#### nextScheduleTime +#### message -Next time the machine will be scheduled (if calculable). +(string) Human-readable message describing the current state. #### observedGeneration -The generation observed by the controller. Used for change detection. +(integer) The generation observed by the controller. Used for change detection. diff --git a/docs/src/security/admission-validation.md b/docs/src/security/admission-validation.md index 7532df8..f31ee57 100644 --- a/docs/src/security/admission-validation.md +++ b/docs/src/security/admission-validation.md @@ -278,7 +278,7 @@ attempted violation, which is useful for SIEM alerting. ```bash kubectl apply -f - <<'EOF' -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: test-valid @@ -303,13 +303,13 @@ spec: EOF ``` -Expected: `scheduledmachine.capi.5spot.io/test-valid created` +Expected: `scheduledmachine.5spot.finos.org/test-valid created` ### Test invalid duration format (rules 2–3) ```bash kubectl apply -f - <<'EOF' -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: test-bad-duration @@ -346,7 +346,7 @@ The ScheduledMachine "test-bad-duration" is invalid: ```bash kubectl apply -f - <<'EOF' -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: test-bad-apigroup @@ -383,7 +383,7 @@ The ScheduledMachine "test-bad-apigroup" is invalid: ```bash kubectl apply -f - <<'EOF' -apiVersion: capi.5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: test-cron-conflict diff --git a/examples/scheduledmachine-basic.yaml b/examples/scheduledmachine-basic.yaml index 90da929..8ca1c41 100644 --- a/examples/scheduledmachine-basic.yaml +++ b/examples/scheduledmachine-basic.yaml @@ -5,7 +5,7 @@ # 2. A RemoteMachine infrastructure resource # 3. A CAPI Machine referencing both -apiVersion: 5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: business-hours-worker @@ -24,7 +24,7 @@ spec: enabled: true # Inline bootstrap specification (creates K0sWorkerConfig) - # Name is auto-generated as: business-hours-worker-bootstrap + # Name is auto-generated as: business-hours-worker bootstrapSpec: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: K0sWorkerConfig @@ -33,7 +33,7 @@ spec: # Add any K0sWorkerConfig-specific fields here # Inline infrastructure specification (creates RemoteMachine) - # Name is auto-generated as: business-hours-worker-infra + # Name is auto-generated as: business-hours-worker infrastructureSpec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: RemoteMachine @@ -49,7 +49,7 @@ spec: node-role.kubernetes.io/worker: "" topology.kubernetes.io/zone: zone-a annotations: - 5spot.io/scheduled-by: business-hours-worker + 5spot.finos.org/scheduled-by: business-hours-worker # Lifecycle settings priority: 50 diff --git a/examples/scheduledmachine-weekend.yaml b/examples/scheduledmachine-weekend.yaml index e933496..9148783 100644 --- a/examples/scheduledmachine-weekend.yaml +++ b/examples/scheduledmachine-weekend.yaml @@ -2,7 +2,7 @@ # This machine will be active Saturday-Sunday, 24 hours # Uses cron expression format as an alternative to daysOfWeek/hoursOfDay -apiVersion: 5spot.io/v1alpha1 +apiVersion: 5spot.finos.org/v1alpha1 kind: ScheduledMachine metadata: name: weekend-worker diff --git a/src/bin/crddoc.rs b/src/bin/crddoc.rs index 741e792..a640659 100644 --- a/src/bin/crddoc.rs +++ b/src/bin/crddoc.rs @@ -41,19 +41,20 @@ fn main() { println!(); println!("### API Group and Version"); println!(); - println!("- **API Group**: `capi.5spot.io`"); + println!("- **API Group**: `5spot.finos.org`"); println!("- **API Version**: `v1alpha1`"); println!("- **Kind**: `ScheduledMachine`"); println!(); println!("### Example"); println!(); println!("```yaml"); - println!("apiVersion: capi.5spot.io/v1alpha1"); + println!("apiVersion: 5spot.finos.org/v1alpha1"); println!("kind: ScheduledMachine"); println!("metadata:"); - println!(" name: example-machine"); + println!(" name: example-spot-machine"); println!(" namespace: default"); println!("spec:"); + println!(" clusterName: my-cluster"); println!(" schedule:"); println!(" daysOfWeek:"); println!(" - mon-fri"); @@ -61,25 +62,29 @@ fn main() { println!(" - 9-17"); println!(" timezone: America/New_York"); println!(" enabled: true"); - println!(" machine:"); - println!(" address: 192.168.1.100"); - println!(" user: admin"); - println!(" port: 22"); - println!(" useSudo: false"); - println!(" files: []"); - println!(" bootstrapRef:"); + println!(" bootstrapSpec:"); println!(" apiVersion: bootstrap.cluster.x-k8s.io/v1beta1"); - println!(" kind: KubeadmConfigTemplate"); - println!(" name: worker-bootstrap-config"); - println!(" namespace: default"); - println!(" infrastructureRef:"); + println!(" kind: K0sWorkerConfig"); + println!(" spec:"); + println!(" version: v1.32.8+k0s.0"); + println!(" downloadURL: https://github.com/k0sproject/k0s/releases/download/v1.32.8+k0s.0/k0s-v1.32.8+k0s.0-amd64"); + println!(" infrastructureSpec:"); println!(" apiVersion: infrastructure.cluster.x-k8s.io/v1beta1"); - println!(" kind: MachineTemplate"); - println!(" name: worker-machine-template"); - println!(" namespace: default"); - println!(" clusterName: my-cluster"); + println!(" kind: RemoteMachine"); + println!(" spec:"); + println!(" address: 192.168.1.100"); + println!(" port: 22"); + println!(" user: root"); + println!(" sshKeyRef:"); + println!(" name: my-ssh-key"); + println!(" machineTemplate:"); + println!(" labels:"); + println!(" node-role.kubernetes.io/worker: spot"); + println!(" annotations:"); + println!(" example.com/scheduled-by: 5spot"); println!(" priority: 50"); println!(" gracefulShutdownTimeout: 5m"); + println!(" nodeDrainTimeout: 5m"); println!(" killSwitch: false"); println!("```"); println!(); @@ -102,26 +107,46 @@ fn main() { "- **enabled** (optional, boolean, default: `true`): Whether the schedule is enabled." ); println!(); - println!("#### machine"); + println!("#### clusterName"); println!(); - println!("Machine specification for k0smotron."); + println!("(required, string) Name of the CAPI cluster this machine belongs to."); println!(); - println!("- **address** (required, string): IP address of the machine."); + println!("#### bootstrapSpec"); println!(); - println!("- **user** (required, string): Username for SSH connection."); + println!("(required, object) Inline bootstrap configuration that will be created when the schedule is active."); + println!("This is a fully unstructured object that must contain:"); println!(); - println!("- **port** (optional, integer, default: `22`): SSH port."); + println!("- **apiVersion** (required, string): API version of the bootstrap resource (e.g., `bootstrap.cluster.x-k8s.io/v1beta1`)"); + println!("- **kind** (required, string): Kind of the bootstrap resource (e.g., `K0sWorkerConfig`, `KubeadmConfig`)"); + println!( + "- **spec** (required, object): Provider-specific configuration for the bootstrap resource" + ); println!(); println!( - "- **useSudo** (optional, boolean, default: `false`): Whether to use sudo for commands." + "The controller validates that the apiVersion belongs to an allowed bootstrap API group." ); println!(); - println!("- **files** (optional, array): Files to be passed to user_data upon creation."); + println!("#### infrastructureSpec"); println!(); - println!("#### clusterName"); + println!("(required, object) Inline infrastructure configuration that will be created when the schedule is active."); + println!("This is a fully unstructured object that must contain:"); println!(); - println!("(required, string) Name of the CAPI cluster this machine belongs to."); - println!("The bootstrap and infrastructure refs must be configured for this cluster."); + println!("- **apiVersion** (required, string): API version of the infrastructure resource (e.g., `infrastructure.cluster.x-k8s.io/v1beta1`)"); + println!("- **kind** (required, string): Kind of the infrastructure resource (e.g., `RemoteMachine`, `AWSMachine`)"); + println!("- **spec** (required, object): Provider-specific configuration for the infrastructure resource"); + println!(); + println!("The controller validates that the apiVersion belongs to an allowed infrastructure API group."); + println!(); + println!("#### machineTemplate"); + println!(); + println!("(optional, object) Configuration for the created CAPI Machine resource."); + println!(); + println!( + "- **labels** (optional, map of string to string): Labels to apply to the created Machine" + ); + println!("- **annotations** (optional, map of string to string): Annotations to apply to the created Machine"); + println!(); + println!("Note: Labels and annotations using reserved prefixes (`5spot.finos.org/`, `cluster.x-k8s.io/`) are rejected."); println!(); println!("#### priority"); println!(); @@ -136,6 +161,13 @@ fn main() { "Format: `` where unit is `s` (seconds), `m` (minutes), or `h` (hours)." ); println!(); + println!("#### nodeDrainTimeout"); + println!(); + println!("(optional, string, default: `5m`) Timeout for draining the node before deletion."); + println!( + "Format: `` where unit is `s` (seconds), `m` (minutes), or `h` (hours)." + ); + println!(); println!("#### killSwitch"); println!(); println!("(optional, boolean, default: `false`) When true, immediately removes the machine"); @@ -147,12 +179,12 @@ fn main() { println!(); println!("Current phase of the machine lifecycle. Possible values:"); println!(); - println!("- **Pending**: Initial state, schedule not yet evaluated"); - println!("- **Scheduled**: Machine is within scheduled time window but not yet active"); + println!("- **Pending**: Initial state, awaiting schedule evaluation"); println!("- **Active**: Machine is running and part of the cluster"); - println!("- **UnScheduled**: Machine is outside scheduled time window"); - println!("- **Removing**: Machine is being removed from cluster"); - println!("- **Inactive**: Machine has been removed and is inactive"); + println!("- **ShuttingDown**: Machine is being gracefully removed (draining, etc.)"); + println!("- **Inactive**: Machine is outside scheduled time window and has been removed"); + println!("- **Disabled**: Schedule is disabled, machine is not active"); + println!("- **Terminated**: Machine has been permanently removed"); println!("- **Error**: An error occurred during processing"); println!(); println!("#### conditions"); @@ -165,23 +197,15 @@ fn main() { println!("- **message**: Human-readable message"); println!("- **lastTransitionTime**: Last time the condition transitioned"); println!(); - println!("#### machineRef"); - println!(); - println!("Reference to the actual Machine resource:"); - println!(); - println!("- **name**: Machine name"); - println!("- **namespace**: Machine namespace"); - println!("- **uid**: Machine UID"); - println!(); - println!("#### lastScheduleTime"); + println!("#### inSchedule"); println!(); - println!("Last time the machine was scheduled and activated."); + println!("(boolean) Whether the machine is currently within its scheduled time window."); println!(); - println!("#### nextScheduleTime"); + println!("#### message"); println!(); - println!("Next time the machine will be scheduled (if calculable)."); + println!("(string) Human-readable message describing the current state."); println!(); println!("#### observedGeneration"); println!(); - println!("The generation observed by the controller. Used for change detection."); + println!("(integer) The generation observed by the controller. Used for change detection."); } diff --git a/src/constants.rs b/src/constants.rs index 4479b52..6da9fae 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -22,13 +22,13 @@ // ============================================================================ /// API group for 5-Spot CRDs -pub const API_GROUP: &str = "5spot.io"; +pub const API_GROUP: &str = "5spot.finos.org"; /// API version for 5-Spot CRDs pub const API_VERSION: &str = "v1alpha1"; /// Full API version string -pub const API_VERSION_FULL: &str = "5spot.io/v1alpha1"; +pub const API_VERSION_FULL: &str = "5spot.finos.org/v1alpha1"; // ============================================================================ // Resource Names @@ -54,7 +54,7 @@ pub const KIND_SECRET: &str = "Secret"; // ============================================================================ /// Finalizer for `ScheduledMachine` resources -pub const FINALIZER_SCHEDULED_MACHINE: &str = "5spot.io/scheduledmachine"; +pub const FINALIZER_SCHEDULED_MACHINE: &str = "5spot.finos.org/scheduledmachine"; // ============================================================================ // Timing Constants (in seconds) @@ -100,7 +100,7 @@ pub const DEFAULT_LEASE_RENEW_DEADLINE_SECS: u64 = 10; /// Default retry period in seconds — documented for ops; not a direct `kube-lease-manager` parameter pub const DEFAULT_LEASE_RETRY_PERIOD_SECS: u64 = 2; -/// Computed default grace period (duration − renew_deadline) in seconds +/// Computed default grace period (duration − `renew_deadline`) in seconds pub const DEFAULT_LEASE_GRACE_SECS: u64 = DEFAULT_LEASE_DURATION_SECS - DEFAULT_LEASE_RENEW_DEADLINE_SECS; @@ -349,7 +349,7 @@ pub const RESERVED_LABEL_PREFIXES: &[&str] = &[ "kubernetes.io/", "k8s.io/", "cluster.x-k8s.io/", - "5spot.io/", + "5spot.finos.org/", ]; /// Allowed API groups for bootstrap embedded resources diff --git a/src/crd.rs b/src/crd.rs index 1c445ac..eec6d98 100644 --- a/src/crd.rs +++ b/src/crd.rs @@ -26,7 +26,7 @@ use std::collections::{BTreeMap, HashSet}; #[derive(CustomResource, Clone, Debug, Serialize, Deserialize, JsonSchema)] #[kube( - group = "5spot.io", + group = "5spot.finos.org", version = "v1alpha1", kind = "ScheduledMachine", namespaced, @@ -167,21 +167,42 @@ impl ScheduleSpec { // EmbeddedResource - Inline resource specification for CAPI resources // ============================================================================ -/// An embedded Kubernetes resource specification -/// Used for inline bootstrap and infrastructure specs +/// An embedded Kubernetes resource specification. +/// +/// Used for inline bootstrap and infrastructure specs. This is intentionally +/// unstructured to support any provider type (`K0sWorkerConfig`, `KubeadmConfig`, +/// `RemoteMachine`, `AWSMachine`, etc.) without requiring schema knowledge. +/// +/// Must contain at minimum `apiVersion` and `kind` fields. The controller +/// will extract these to create the appropriate dynamic resource. #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct EmbeddedResource { - /// API version of the resource (e.g., "bootstrap.cluster.x-k8s.io/v1beta1") - pub api_version: String, +#[schemars(schema_with = "embedded_resource_schema")] +pub struct EmbeddedResource(pub Value); - /// Kind of the resource (e.g., `K0sWorkerConfig`, `RemoteMachine`) - pub kind: String, +impl EmbeddedResource { + /// Get the apiVersion field from the embedded resource + #[must_use] + pub fn api_version(&self) -> Option<&str> { + self.0.get("apiVersion").and_then(Value::as_str) + } + + /// Get the kind field from the embedded resource + #[must_use] + pub fn kind(&self) -> Option<&str> { + self.0.get("kind").and_then(Value::as_str) + } + + /// Get the spec field from the embedded resource + #[must_use] + pub fn spec(&self) -> Option<&Value> { + self.0.get("spec") + } - /// The spec of the resource (provider-specific) - /// This is an arbitrary JSON object whose schema depends on the kind - #[schemars(schema_with = "arbitrary_object_schema")] - pub spec: Value, + /// Get the inner JSON value + #[must_use] + pub fn inner(&self) -> &Value { + &self.0 + } } /// Schema for the timezone field — bounded string to prevent log injection @@ -193,12 +214,29 @@ fn timezone_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { }) } -/// Schema for arbitrary JSON object -fn arbitrary_object_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { +/// Schema for `EmbeddedResource` — requires apiVersion, kind, and spec fields. +/// The `spec` field uses `x-kubernetes-preserve-unknown-fields` to allow any +/// provider-specific fields (`K0sWorkerConfig`, `RemoteMachine`, `AWSMachine`, etc.). +fn embedded_resource_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { schemars::json_schema!({ "type": "object", - "additionalProperties": true, - "x-kubernetes-preserve-unknown-fields": true + "required": ["apiVersion", "kind", "spec"], + "properties": { + "apiVersion": { + "type": "string", + "description": "API version of the resource (e.g., 'bootstrap.cluster.x-k8s.io/v1beta1')" + }, + "kind": { + "type": "string", + "description": "Kind of the resource (e.g., 'K0sWorkerConfig', 'RemoteMachine')" + }, + "spec": { + "type": "object", + "x-kubernetes-preserve-unknown-fields": true, + "description": "Provider-specific configuration" + } + }, + "additionalProperties": false }) } diff --git a/src/crd_tests.rs b/src/crd_tests.rs index faa6faf..4562cfd 100644 --- a/src/crd_tests.rs +++ b/src/crd_tests.rs @@ -217,16 +217,16 @@ mod tests { enabled: true, }, cluster_name: "test-cluster".to_string(), - bootstrap_spec: EmbeddedResource { - api_version: "bootstrap.cluster.x-k8s.io/v1beta1".to_string(), - kind: "K0sWorkerConfig".to_string(), - spec: json!({"args": []}), - }, - infrastructure_spec: EmbeddedResource { - api_version: "infrastructure.cluster.x-k8s.io/v1beta1".to_string(), - kind: "RemoteMachine".to_string(), - spec: json!({"address": "192.168.1.100", "port": 22}), - }, + bootstrap_spec: EmbeddedResource(json!({ + "apiVersion": "bootstrap.cluster.x-k8s.io/v1beta1", + "kind": "K0sWorkerConfig", + "spec": {"args": []} + })), + infrastructure_spec: EmbeddedResource(json!({ + "apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1", + "kind": "RemoteMachine", + "spec": {"address": "192.168.1.100", "port": 22} + })), machine_template: None, priority: 50, graceful_shutdown_timeout: "5m".to_string(), diff --git a/src/main.rs b/src/main.rs index e7da12d..45ec7dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -94,7 +94,7 @@ struct Cli { #[clap(long, env = "LEASE_RENEW_DEADLINE_SECONDS", default_value_t = DEFAULT_LEASE_RENEW_DEADLINE_SECS)] lease_renew_deadline_secs: u64, - /// Retry period in seconds — documented for ops parity; not a direct LeaseManager parameter + /// Retry period in seconds — documented for ops parity; not a direct `LeaseManager` parameter #[clap(long, env = "LEASE_RETRY_PERIOD_SECONDS", default_value_t = DEFAULT_LEASE_RETRY_PERIOD_SECS)] _lease_retry_period_secs: u64, } @@ -106,6 +106,7 @@ struct Cli { /// - The Kubernetes client cannot be configured /// - The `ScheduledMachine` CRD is not present in the cluster #[tokio::main] +#[allow(clippy::too_many_lines)] async fn main() -> Result<()> { let cli = Cli::parse(); @@ -208,20 +209,17 @@ async fn main() -> Result<()> { tokio::spawn(async move { let (mut channel, task) = manager.watch().await; loop { - match channel.changed().await { - Ok(()) => { - let acquired = *channel.borrow_and_update(); - is_leader.store(acquired, Ordering::Release); - if acquired { - info!(holder_id = %holder_id, "Acquired leadership lease"); - } else { - info!(holder_id = %holder_id, "Lost leadership lease — standby"); - } - } - Err(_) => { - error!("Leader election watch channel closed unexpectedly"); - break; + if let Ok(()) = channel.changed().await { + let acquired = *channel.borrow_and_update(); + is_leader.store(acquired, Ordering::Release); + if acquired { + info!(holder_id = %holder_id, "Acquired leadership lease"); + } else { + info!(holder_id = %holder_id, "Lost leadership lease — standby"); } + } else { + error!("Leader election watch channel closed unexpectedly"); + break; } } drop(channel); @@ -245,7 +243,7 @@ async fn main() -> Result<()> { error!(" kubectl apply -f deploy/crds/scheduledmachine.yaml"); error!("Or generate and apply: cargo run --bin crdgen | kubectl apply -f -"); return Err(anyhow::anyhow!( - "Required CRD 'scheduledmachines.5spot.io' is not installed in the cluster" + "Required CRD 'scheduledmachines.5spot.finos.org' is not installed in the cluster" )); } info!("ScheduledMachine CRD verified"); diff --git a/src/reconcilers/helpers.rs b/src/reconcilers/helpers.rs index ce541a9..f51061b 100644 --- a/src/reconcilers/helpers.rs +++ b/src/reconcilers/helpers.rs @@ -366,6 +366,7 @@ pub async fn handle_kill_switch( PHASE_TERMINATED, Some(REASON_KILL_SWITCH), Some("Machine terminated due to kill switch"), + false, // in_schedule: kill switch overrides schedule ) .await?; @@ -516,6 +517,7 @@ pub fn build_phase_transition_event( /// The `from_phase` parameter captures the previous phase for before/after logging. /// Event recording is best-effort — a failure to publish the event is logged as a /// warning but does not abort the phase transition. +#[allow(clippy::too_many_arguments)] pub async fn update_phase( ctx: &Context, namespace: &str, @@ -524,6 +526,7 @@ pub async fn update_phase( phase: &str, reason: Option<&str>, message: Option<&str>, + in_schedule: bool, ) -> Result<(), ReconcilerError> { let resolved_reason = reason.unwrap_or(REASON_RECONCILE_SUCCEEDED); let resolved_message = message.unwrap_or("Phase transition completed"); @@ -568,6 +571,7 @@ pub async fn update_phase( phase: Some(phase.to_string()), message: Some(resolved_message.to_string()), conditions: vec![condition], + in_schedule, ..Default::default() }; @@ -591,6 +595,7 @@ pub async fn update_phase( /// # Errors /// Same as [`update_phase`]. #[allow(dead_code)] // TODO: Use this when machine creation is implemented +#[allow(clippy::too_many_arguments)] pub async fn update_phase_with_last_schedule( ctx: &Context, namespace: &str, @@ -599,6 +604,7 @@ pub async fn update_phase_with_last_schedule( phase: &str, reason: Option<&str>, message: Option<&str>, + in_schedule: bool, ) -> Result<(), ReconcilerError> { let resolved_reason = reason.unwrap_or(REASON_RECONCILE_SUCCEEDED); let resolved_message = message.unwrap_or("Phase transition completed"); @@ -638,6 +644,7 @@ pub async fn update_phase_with_last_schedule( message: Some(resolved_message.to_string()), conditions: vec![condition], last_scheduled_time: Some(Utc::now().to_rfc3339()), + in_schedule, ..Default::default() }; @@ -661,6 +668,7 @@ pub async fn update_phase_with_last_schedule( /// /// # Errors /// Same as [`update_phase`]. +#[allow(clippy::too_many_arguments)] pub async fn update_phase_with_grace_period( ctx: &Context, namespace: &str, @@ -669,6 +677,7 @@ pub async fn update_phase_with_grace_period( phase: &str, reason: Option<&str>, message: Option<&str>, + in_schedule: bool, ) -> Result<(), ReconcilerError> { let resolved_reason = reason.unwrap_or(REASON_GRACE_PERIOD); let resolved_message = message.unwrap_or("Grace period started"); @@ -707,6 +716,7 @@ pub async fn update_phase_with_grace_period( phase: Some(phase.to_string()), message: Some(resolved_message.to_string()), conditions: vec![condition], + in_schedule, ..Default::default() }; @@ -770,32 +780,6 @@ pub fn validate_api_group( // CAPI Resource Creation // ============================================================================ -/// Derive the Kubernetes name for the bootstrap config resource. -/// -/// The name is `-bootstrap`, which makes the -/// child resource easy to identify in `kubectl get` output and ties its -/// lifecycle to the parent `ScheduledMachine` via name-based correlation -/// (in addition to `ownerReferences`). -fn bootstrap_resource_name(scheduled_machine_name: &str) -> String { - format!("{scheduled_machine_name}-bootstrap") -} - -/// Derive the Kubernetes name for the infrastructure resource. -/// -/// The name is `-infra`. See [`bootstrap_resource_name`] -/// for the naming rationale. -fn infrastructure_resource_name(scheduled_machine_name: &str) -> String { - format!("{scheduled_machine_name}-infra") -} - -/// Derive the Kubernetes name for the CAPI `Machine` resource. -/// -/// The name is `-machine`. See [`bootstrap_resource_name`] -/// for the naming rationale. -fn machine_resource_name(scheduled_machine_name: &str) -> String { - format!("{scheduled_machine_name}-machine") -} - /// Add machine to cluster by creating bootstrap, infrastructure, and Machine resources /// /// This function: @@ -811,28 +795,58 @@ pub async fn add_machine_to_cluster( let name = resource.name_any(); let cluster_name = &resource.spec.cluster_name; - let bootstrap_name = bootstrap_resource_name(&name); - let infra_name = infrastructure_resource_name(&name); - let machine_name = machine_resource_name(&name); - info!( resource = %name, namespace = %namespace, cluster = %cluster_name, - bootstrap = %bootstrap_name, - infrastructure = %infra_name, - machine = %machine_name, "Creating CAPI resources from inline specs" ); + // Extract required fields from embedded resources + let bootstrap_api_version = resource.spec.bootstrap_spec.api_version().ok_or_else(|| { + ReconcilerError::InvalidConfig( + "bootstrapSpec missing required field 'apiVersion'".to_string(), + ) + })?; + let bootstrap_kind = resource.spec.bootstrap_spec.kind().ok_or_else(|| { + ReconcilerError::InvalidConfig("bootstrapSpec missing required field 'kind'".to_string()) + })?; + let bootstrap_spec_inner = resource + .spec + .bootstrap_spec + .spec() + .cloned() + .unwrap_or_default(); + + let infra_api_version = resource + .spec + .infrastructure_spec + .api_version() + .ok_or_else(|| { + ReconcilerError::InvalidConfig( + "infrastructureSpec missing required field 'apiVersion'".to_string(), + ) + })?; + let infra_kind = resource.spec.infrastructure_spec.kind().ok_or_else(|| { + ReconcilerError::InvalidConfig( + "infrastructureSpec missing required field 'kind'".to_string(), + ) + })?; + let infra_spec_inner = resource + .spec + .infrastructure_spec + .spec() + .cloned() + .unwrap_or_default(); + // Validate API groups before creating any resources validate_api_group( - &resource.spec.bootstrap_spec.api_version, + bootstrap_api_version, ALLOWED_BOOTSTRAP_API_GROUPS, "bootstrap", )?; validate_api_group( - &resource.spec.infrastructure_spec.api_version, + infra_api_version, ALLOWED_INFRASTRUCTURE_API_GROUPS, "infrastructure", )?; @@ -860,61 +874,68 @@ pub async fn add_machine_to_cluster( let infra_ns = namespace; // 1. Create bootstrap resource - let bootstrap_spec = &resource.spec.bootstrap_spec; + // NOTE: No ownerReferences here - the bootstrap controller (e.g., k0smotron) needs to + // process this resource. We use labels for tracking instead, and the CAPI Machine's + // bootstrap.configRef provides the logical relationship. let bootstrap_obj = json!({ - "apiVersion": bootstrap_spec.api_version, - "kind": bootstrap_spec.kind, + "apiVersion": bootstrap_api_version, + "kind": bootstrap_kind, "metadata": { - "name": bootstrap_name, + "name": name, "namespace": bootstrap_ns, - "ownerReferences": [owner_ref], + "labels": { + "5spot.finos.org/scheduled-machine": name, + CAPI_CLUSTER_NAME_LABEL: cluster_name, + }, }, - "spec": bootstrap_spec.spec, + "spec": bootstrap_spec_inner, }); create_dynamic_resource( client, bootstrap_ns, - &bootstrap_spec.api_version, - &bootstrap_spec.kind, + bootstrap_api_version, + bootstrap_kind, bootstrap_obj, ) .await .map_err(|e| ReconcilerError::CapiError(format!("Failed to create bootstrap resource: {e}")))?; - info!(bootstrap = %bootstrap_name, "Bootstrap resource created"); + info!(kind = %bootstrap_kind, "Bootstrap resource created"); // 2. Create infrastructure resource - let infra_spec = &resource.spec.infrastructure_spec; + // NOTE: No ownerReferences here - the infrastructure controller (e.g., CAPM3, CAPA) needs to + // process this resource. We use labels for tracking instead, and the CAPI Machine's + // infrastructureRef provides the logical relationship. let infra_obj = json!({ - "apiVersion": infra_spec.api_version, - "kind": infra_spec.kind, + "apiVersion": infra_api_version, + "kind": infra_kind, "metadata": { - "name": infra_name, + "name": name, "namespace": infra_ns, - "ownerReferences": [owner_ref], + "labels": { + "5spot.finos.org/scheduled-machine": name, + CAPI_CLUSTER_NAME_LABEL: cluster_name, + }, }, - "spec": infra_spec.spec, + "spec": infra_spec_inner, }); - create_dynamic_resource( - client, - infra_ns, - &infra_spec.api_version, - &infra_spec.kind, - infra_obj, - ) - .await - .map_err(|e| { - ReconcilerError::CapiError(format!("Failed to create infrastructure resource: {e}")) - })?; + create_dynamic_resource(client, infra_ns, infra_api_version, infra_kind, infra_obj) + .await + .map_err(|e| { + ReconcilerError::CapiError(format!("Failed to create infrastructure resource: {e}")) + })?; - info!(infrastructure = %infra_name, "Infrastructure resource created"); + info!(kind = %infra_kind, "Infrastructure resource created"); // 3. Create CAPI Machine referencing both let mut machine_labels = std::collections::BTreeMap::new(); machine_labels.insert(CAPI_CLUSTER_NAME_LABEL.to_string(), cluster_name.clone()); - machine_labels.insert("5spot.io/scheduled-machine".to_string(), name.clone()); + machine_labels.insert( + "5spot.finos.org/scheduled-machine".to_string(), + name.clone(), + ); // Merge in user-provided labels if let Some(template) = &resource.spec.machine_template { @@ -935,7 +956,7 @@ pub async fn add_machine_to_cluster( "apiVersion": CAPI_MACHINE_API_VERSION_FULL, "kind": "Machine", "metadata": { - "name": machine_name, + "name": name, "namespace": namespace, "labels": machine_labels, "annotations": machine_annotations, @@ -945,16 +966,16 @@ pub async fn add_machine_to_cluster( "clusterName": cluster_name, "bootstrap": { "configRef": { - "apiVersion": bootstrap_spec.api_version, - "kind": bootstrap_spec.kind, - "name": bootstrap_name, + "apiVersion": bootstrap_api_version, + "kind": bootstrap_kind, + "name": name, "namespace": bootstrap_ns, } }, "infrastructureRef": { - "apiVersion": infra_spec.api_version, - "kind": infra_spec.kind, - "name": infra_name, + "apiVersion": infra_api_version, + "kind": infra_kind, + "name": name, "namespace": infra_ns, }, } @@ -970,11 +991,7 @@ pub async fn add_machine_to_cluster( .await .map_err(|e| ReconcilerError::CapiError(format!("Failed to create Machine: {e}")))?; - info!( - resource = %name, - machine_name = %machine_name, - "CAPI Machine created successfully" - ); + info!(resource = %name, "CAPI Machine created successfully"); Ok(()) } @@ -1033,6 +1050,44 @@ fn parse_api_version(api_version: &str) -> (String, String) { } } +/// Delete a Kubernetes resource via the dynamic API client. +/// +/// A 404 response is treated as success (resource already deleted). +/// +/// # Errors +/// Returns `ReconcilerError::CapiError` if the API call fails with a non-404 status. +async fn delete_dynamic_resource( + client: &Client, + namespace: &str, + api_version: &str, + kind: &str, + name: &str, +) -> Result<(), ReconcilerError> { + let (group, version) = parse_api_version(api_version); + let plural = format!("{}s", kind.to_lowercase()); + + let ar = kube::api::ApiResource::from_gvk_with_plural( + &kube::api::GroupVersionKind::gvk(&group, &version, kind), + &plural, + ); + + let api: Api = Api::namespaced_with(client.clone(), namespace, &ar); + + match api.delete(name, &kube::api::DeleteParams::default()).await { + Ok(_) => { + info!(kind = %kind, name = %name, "Resource deletion initiated"); + Ok(()) + } + Err(kube::Error::Api(e)) if e.code == 404 => { + debug!(kind = %kind, name = %name, "Resource already deleted or does not exist"); + Ok(()) + } + Err(e) => Err(ReconcilerError::CapiError(format!( + "Failed to delete {kind} {name}: {e}" + ))), + } +} + /// Delete the CAPI `Machine` resource that represents this node in the cluster. /// /// Deletion is initiated by issuing a `DELETE` to the `Machine` resource. @@ -1040,6 +1095,9 @@ fn parse_api_version(api_version: &str) -> (String, String) { /// (deprovision, drain, etc.) asynchronously. A 404 response is treated as /// success because it means the machine was already removed. /// +/// Also deletes the associated bootstrap and infrastructure resources since they +/// no longer have ownerReferences (to allow their respective controllers to process them). +/// /// # Errors /// - [`ReconcilerError::CapiError`] — API call failed with a non-404 status pub async fn remove_machine_from_cluster( @@ -1049,17 +1107,15 @@ pub async fn remove_machine_from_cluster( ) -> Result<(), ReconcilerError> { let name = resource.name_any(); let cluster_name = &resource.spec.cluster_name; - let machine_name = format!("{name}-machine"); info!( resource = %name, namespace = %namespace, cluster = %cluster_name, - machine_name = %machine_name, - "Deleting CAPI Machine resource" + "Deleting CAPI resources" ); - // Get the Machine API + // 1. Delete the Machine resource first (this triggers CAPI cleanup) let ar = kube::api::ApiResource::from_gvk_with_plural( &kube::api::GroupVersionKind::gvk(CAPI_GROUP, CAPI_MACHINE_API_VERSION, "Machine"), CAPI_RESOURCE_MACHINES, @@ -1067,30 +1123,38 @@ pub async fn remove_machine_from_cluster( let machines: Api = Api::namespaced_with(client.clone(), namespace, &ar); - // Delete the Machine resource if it exists match machines - .delete(&machine_name, &kube::api::DeleteParams::default()) + .delete(&name, &kube::api::DeleteParams::default()) .await { Ok(_) => { - info!( - machine_name = %machine_name, - "CAPI Machine deletion initiated" - ); - Ok(()) + info!(resource = %name, "CAPI Machine deletion initiated"); } Err(kube::Error::Api(e)) if e.code == 404 => { - // Machine already deleted or doesn't exist - debug!( - machine_name = %machine_name, - "CAPI Machine already deleted or does not exist" - ); - Ok(()) + debug!(resource = %name, "CAPI Machine already deleted or does not exist"); } - Err(e) => Err(ReconcilerError::CapiError(format!( - "Failed to delete Machine {machine_name}: {e}" - ))), + Err(e) => { + return Err(ReconcilerError::CapiError(format!( + "Failed to delete Machine {name}: {e}" + ))); + } + } + + // 2. Delete bootstrap resource (no ownerReference, must delete explicitly) + let bootstrap_api_version = resource.spec.bootstrap_spec.api_version(); + let bootstrap_kind = resource.spec.bootstrap_spec.kind(); + if let (Some(api_version), Some(kind)) = (bootstrap_api_version, bootstrap_kind) { + delete_dynamic_resource(client, namespace, api_version, kind, &name).await?; + } + + // 3. Delete infrastructure resource (no ownerReference, must delete explicitly) + let infra_api_version = resource.spec.infrastructure_spec.api_version(); + let infra_kind = resource.spec.infrastructure_spec.kind(); + if let (Some(api_version), Some(kind)) = (infra_api_version, infra_kind) { + delete_dynamic_resource(client, namespace, api_version, kind, &name).await?; } + + Ok(()) } // ============================================================================ @@ -1345,6 +1409,7 @@ pub(crate) fn compute_backoff_secs(retry_count: u32) -> u64 { /// /// The retry count is cleared when reconciliation succeeds (see /// `reconcile_guarded` in `scheduled_machine.rs`). +#[allow(clippy::needless_pass_by_value)] // kube-rs Controller API requires Arc by value pub fn error_policy( resource: Arc, err: &ReconcilerError, @@ -1356,7 +1421,10 @@ pub fn error_policy( resource.name_any() ); let retry_count = { - let mut counts = ctx.retry_counts.lock().unwrap_or_else(|p| p.into_inner()); + let mut counts = ctx + .retry_counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let count = counts.entry(key).or_insert(0); *count = count.saturating_add(1); *count diff --git a/src/reconcilers/helpers_tests.rs b/src/reconcilers/helpers_tests.rs index 49df674..ba28713 100644 --- a/src/reconcilers/helpers_tests.rs +++ b/src/reconcilers/helpers_tests.rs @@ -138,7 +138,7 @@ mod tests { fn test_validate_labels_rejects_5spot_io() { let mut labels = BTreeMap::new(); labels.insert( - "5spot.io/scheduled-machine".to_string(), + "5spot.finos.org/scheduled-machine".to_string(), "injected".to_string(), ); assert!(validate_labels(&labels, "labels").is_err()); @@ -397,7 +397,7 @@ mod tests { /// Minimal `ScheduledMachine` JSON for `patch_status` responses. fn sm_response_body(name: &str, namespace: &str, phase: &str) -> Vec { serde_json::to_vec(&serde_json::json!({ - "apiVersion": "5spot.io/v1alpha1", + "apiVersion": "5spot.finos.org/v1alpha1", "kind": "ScheduledMachine", "metadata": { "name": name, @@ -447,7 +447,7 @@ mod tests { "reason": "MachineCreated", "type": "Normal", "regarding": { - "apiVersion": "5spot.io/v1alpha1", + "apiVersion": "5spot.finos.org/v1alpha1", "kind": "ScheduledMachine", "name": "test-sm", "namespace": "default" @@ -510,6 +510,7 @@ mod tests { "Active", Some("MachineCreated"), Some("CAPI Machine created"), + true, ) .await .expect("update_phase should return Ok on success"); @@ -545,9 +546,11 @@ mod tests { }); // Passing None for reason and message — should use defaults without panicking - update_phase(&ctx, "default", "test-sm", None, "Inactive", None, None) - .await - .expect("should succeed with default reason/message"); + update_phase( + &ctx, "default", "test-sm", None, "Inactive", None, None, false, + ) + .await + .expect("should succeed with default reason/message"); srv.await.unwrap(); } @@ -589,6 +592,7 @@ mod tests { "Error", None, None, + false, ) .await; assert!(result.is_err(), "should return Err when patch_status fails"); @@ -636,6 +640,7 @@ mod tests { "Active", None, None, + true, ) .await; assert!(result.is_err(), "should return Err on 404"); @@ -685,6 +690,7 @@ mod tests { "Active", None, None, + true, ) .await; assert!( @@ -731,6 +737,7 @@ mod tests { "Error", None, None, + false, ) .await; assert!( @@ -782,6 +789,7 @@ mod tests { "ShuttingDown", None, None, + false, ) .await .expect("grace period update should succeed"); @@ -825,6 +833,7 @@ mod tests { "ShuttingDown", None, None, + false, ) .await; assert!(result.is_err(), "409 conflict should return error"); diff --git a/src/reconcilers/scheduled_machine.rs b/src/reconcilers/scheduled_machine.rs index ce5bc84..3e62cc2 100644 --- a/src/reconcilers/scheduled_machine.rs +++ b/src/reconcilers/scheduled_machine.rs @@ -40,7 +40,7 @@ use kube::{ }, Client, Resource, ResourceExt, }; -use tracing::{debug, error, info, Instrument}; +use tracing::{debug, error, info, warn, Instrument}; use crate::constants::{ ERROR_REQUEUE_SECS, PHASE_ACTIVE, PHASE_DISABLED, PHASE_ERROR, PHASE_INACTIVE, PHASE_PENDING, @@ -422,7 +422,7 @@ async fn reconcile_inner( PHASE_INACTIVE => handle_inactive_phase(resource, ctx, should_be_active).await, PHASE_DISABLED => handle_disabled_phase(resource, ctx, should_be_active).await, PHASE_TERMINATED => handle_terminated_phase(resource, ctx).await, - PHASE_ERROR => handle_error_phase(resource, ctx).await, + PHASE_ERROR => handle_error_phase(resource, ctx), // PHASE_PENDING and unknown phases handled by default _ => handle_pending_phase(resource, ctx, should_be_active).await, } @@ -460,6 +460,7 @@ async fn handle_pending_phase( PHASE_DISABLED, Some(REASON_SCHEDULE_DISABLED), Some("Schedule is disabled"), + false, // in_schedule: disabled means not in schedule ) .await?; return Ok(Action::requeue(Duration::from_secs(TIMER_REQUEUE_SECS))); @@ -476,6 +477,7 @@ async fn handle_pending_phase( PHASE_INACTIVE, Some(REASON_SCHEDULE_INACTIVE), Some("Outside scheduled time window"), + false, // in_schedule: outside window ) .await?; return Ok(Action::requeue(Duration::from_secs(TIMER_REQUEUE_SECS))); @@ -495,6 +497,7 @@ async fn handle_pending_phase( PHASE_ERROR, Some("MachineCreationFailed"), Some(&format!("Failed to create CAPI Machine: {e}")), + true, // in_schedule: error occurred while in schedule ) .await?; return Ok(Action::requeue(Duration::from_secs(ERROR_REQUEUE_SECS))); @@ -509,6 +512,7 @@ async fn handle_pending_phase( PHASE_ACTIVE, Some(REASON_MACHINE_CREATED), Some("CAPI Machine created successfully"), + true, // in_schedule: machine created because we're in schedule ) .await?; @@ -542,6 +546,7 @@ async fn handle_active_phase( PHASE_SHUTTING_DOWN, Some(REASON_SCHEDULE_DISABLED), Some("Schedule disabled - starting graceful shutdown"), + false, // in_schedule: disabled means not in schedule ) .await?; return Ok(Action::requeue(Duration::from_secs(TIMER_REQUEUE_SECS))); @@ -558,6 +563,7 @@ async fn handle_active_phase( PHASE_SHUTTING_DOWN, Some(REASON_GRACE_PERIOD), Some("Outside schedule - starting graceful shutdown"), + false, // in_schedule: outside schedule window ) .await?; return Ok(Action::requeue(Duration::from_secs(TIMER_REQUEUE_SECS))); @@ -652,6 +658,7 @@ async fn handle_shutting_down_phase( PHASE_ERROR, Some("MachineDeletionFailed"), Some(&format!("Failed to delete CAPI Machine: {e}")), + false, // in_schedule: shutting down means outside schedule ) .await?; return Ok(Action::requeue(Duration::from_secs(ERROR_REQUEUE_SECS))); @@ -666,6 +673,7 @@ async fn handle_shutting_down_phase( PHASE_INACTIVE, Some(REASON_MACHINE_DELETED), Some("Machine removed from cluster"), + false, // in_schedule: inactive means outside schedule ) .await?; @@ -713,6 +721,7 @@ async fn handle_inactive_phase( PHASE_PENDING, Some(REASON_SCHEDULE_ACTIVE), Some("Schedule became active - initiating machine creation"), + true, // in_schedule: transitioning because schedule is now active ) .await?; @@ -749,6 +758,7 @@ async fn handle_disabled_phase( PHASE_PENDING, Some(REASON_SCHEDULE_ACTIVE), Some("Schedule re-enabled"), + false, // in_schedule: will be evaluated in pending phase ) .await?; return Ok(Action::requeue(Duration::from_secs(TIMER_REQUEUE_SECS))); @@ -775,32 +785,30 @@ async fn handle_terminated_phase( /// Handle the `Error` phase — attempt automatic recovery. /// -/// Resets the phase to `Pending`, which causes the controller to re-evaluate -/// the schedule and attempt to reconcile from a clean state. Uses -/// [`ERROR_REQUEUE_SECS`] for the requeue interval to avoid tight retry loops. -async fn handle_error_phase( +/// Simply requeues with the error backoff delay WITHOUT updating status. +/// This avoids the tight retry loop caused by status updates triggering +/// watch events. The resource stays in Error phase until the user fixes +/// the underlying issue (e.g., invalid annotation), at which point the +/// spec change triggers a new reconciliation that succeeds. +#[allow(clippy::needless_pass_by_value)] // Consistent with other phase handlers +fn handle_error_phase( resource: Arc, - ctx: Arc, + _ctx: Arc, ) -> Result { let namespace = resource.namespace().ok_or_else(|| { ReconcilerError::InvalidConfig("ScheduledMachine must be namespaced".to_string()) })?; let name = resource.name_any(); - // Log error and retry from Pending - error!(resource = %name, namespace = %namespace, "In Error phase - attempting recovery"); - - update_phase( - &ctx, - &namespace, - &name, - Some(PHASE_ERROR), - PHASE_PENDING, - Some("RetryingReconciliation"), - Some("Attempting recovery from error"), - ) - .await?; + // Log at warn level (not error) since this is expected during backoff + warn!( + resource = %name, + namespace = %namespace, + "Resource in Error phase - waiting for spec change or manual intervention" + ); + // Requeue with backoff delay but do NOT update status. + // This prevents the watch event that causes tight loops. Ok(Action::requeue(Duration::from_secs(ERROR_REQUEUE_SECS))) } diff --git a/src/reconcilers/scheduled_machine_tests.rs b/src/reconcilers/scheduled_machine_tests.rs index d866c43..9a588ec 100644 --- a/src/reconcilers/scheduled_machine_tests.rs +++ b/src/reconcilers/scheduled_machine_tests.rs @@ -23,16 +23,16 @@ mod tests { enabled: true, }, cluster_name: "test-cluster".to_string(), - bootstrap_spec: crate::crd::EmbeddedResource { - api_version: "bootstrap.cluster.x-k8s.io/v1beta1".to_string(), - kind: "K0sWorkerConfig".to_string(), - spec: json!({"args": []}), - }, - infrastructure_spec: crate::crd::EmbeddedResource { - api_version: "infrastructure.cluster.x-k8s.io/v1beta1".to_string(), - kind: "RemoteMachine".to_string(), - spec: json!({"address": "192.168.1.100", "port": 22}), - }, + bootstrap_spec: crate::crd::EmbeddedResource(json!({ + "apiVersion": "bootstrap.cluster.x-k8s.io/v1beta1", + "kind": "K0sWorkerConfig", + "spec": {"args": []} + })), + infrastructure_spec: crate::crd::EmbeddedResource(json!({ + "apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1", + "kind": "RemoteMachine", + "spec": {"address": "192.168.1.100", "port": 22} + })), machine_template: None, priority: 50, graceful_shutdown_timeout: "5m".to_string(),