diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 908ba273..99855f4c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -167,23 +167,50 @@ jobs: sed -i 's/base_domain = ""/base_domain = "infra.serverwolken.de"/g' *_cluster.tf # AKS: set resource_group - sed -i 's/resource_group = ""/resource_group = "terraform-kubestack-testing"/g' aks_zero_cluster.tf || true + sed -i 's/# resource_group = "my-resource-group"/resource_group = "terraform-kubestack-testing"/g' aks_zero_cluster.tf || true + + # AKS: set availability_zones + sed -i 's/# availability_zones = \["1", "2", "3"\]/availability_zones = ["1", "2", "3"]/g' aks_zero_cluster.tf || true + + # AKS: set vm_size (multi-cloud quickstart example) + sed -i 's/# vm_size = "Standard_B2s"/vm_size = "Standard_B2s"/g' aks_zero_cluster.tf || true + + # AKS: set vm_size (aks quickstart example) + sed -i 's/# vm_size = "Standard_D2s_v3"/vm_size = "Standard_B2s"/g' aks_zero_cluster.tf || true # EKS: set region sed -i 's/region = ""/region = "eu-west-1"/g' eks_zero_providers.tf || true - # EKS: set cluster_availability_zones - sed -i 's/cluster_availability_zones = \[\]/cluster_availability_zones = ["eu-west-1a", "eu-west-1b"]/g' eks_zero_cluster.tf || true + # EKS: set cluster_availability_zones (apps) + sed -i 's/# cluster_availability_zones = \["eu-west-1a", "eu-west-1b", "eu-west-1c"\]/cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"]/g' eks_zero_cluster.tf || true + + # EKS: set cluster_availability_zones (ops) + sed -i 's/# cluster_availability_zones = \["eu-west-1a", "eu-west-1b"\]/cluster_availability_zones = ["eu-west-1a", "eu-west-1b"]/g' eks_zero_cluster.tf || true # GKE: set project_id - sed -i 's/project_id = ""/project_id = "terraform-kubestack-testing"/g' gke_zero_cluster.tf || true + sed -i 's/# project_id = "my-gcp-project"/project_id = "terraform-kubestack-testing"/g' gke_zero_cluster.tf || true # GKE: set region - sed -i 's/region = ""/region = "europe-west1"/g' gke_zero_cluster.tf || true + sed -i 's/# region = "europe-west1"/region = "europe-west1"/g' gke_zero_cluster.tf || true # GKE: set cluster_node_locations sed -i 's/cluster_node_locations = \[\]/cluster_node_locations = ["europe-west1-b", "europe-west1-c", "europe-west1-d"]/g' gke_zero_cluster.tf || true + # GKE: set machine_type + sed -i 's/# machine_type = "e2-medium"/machine_type = "e2-medium"/g' gke_zero_cluster.tf || true + + # GKE: set min_node_count + sed -i 's/# min_node_count = 1/min_node_count = 1/g' gke_zero_cluster.tf || true + + # GKE: set max_node_count + sed -i 's/# max_node_count = 3/max_node_count = 3/g' gke_zero_cluster.tf || true + + # SCW: set region + sed -i 's/# region = "fr-par"/region = "fr-par"/g' scw_zero_cluster.tf || true + + # SCW: set zones + sed -i 's/# zones = \["fr-par-1", "fr-par-2", "fr-par-3"\]/zones = ["fr-par-1", "fr-par-2", "fr-par-3"]/g' scw_zero_cluster.tf || true + - name: "OpenTofu init" working-directory: ./kubestack-starter-${{ matrix.starter }} run: | diff --git a/AGENTS.md b/AGENTS.md index cc662fee..bea32a52 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,7 @@ Before considering any task complete, check each row of the table below and appl | Any module code or `tests/` configuration | Run `make validate`. | | Any divergence between implementation and rules discovered or resolved | Add or remove the entry in `DEVIATIONS.md`. | | Any permanent exception to the standard module contract added or removed | Add or remove the entry in `DEVIATIONS.md`. | +| New required configuration attribute added to a module (guarded by a `lifecycle precondition`) | Add a commented-out example line for it in every affected quickstart file. Add a matching `sed` line to the **"Configure Kubestack"** step in `.github/workflows/main.yml`. See the Quickstart Placeholder and CI Injection Pattern section. | | New cloud provider added | Add provider-specific CLI build and dist targets to `oci/Dockerfile`, following the pattern of existing providers. Add auth instructions to the shared quickstart `README.md`. | | New quickstart added | Symlink shared files (`README.md`, `.gitignore`, `.user/`) to `quickstart/src/configurations/_shared/` instead of duplicating them. | @@ -60,7 +61,7 @@ When a provider does not support a required behaviour (e.g. private nodes), appl - All environments inherit their configuration from the base environment. The base environment key defaults to `apps` (set via `configuration_base_key`). An environment-specific value always overrides the inherited base value. - The inheritance logic is implemented in the `common/configuration` module. All Kubestack modules MUST use it. See the Variables and Outputs section for the required usage pattern. -- Because all `configuration` object attributes must be marked `optional(...)` to support inheritance, values that have no module default and must always be user-provided (region, zones, instance type, and `min`/`max` node counts — see Multi-Zone Resilience and Node Pool Lifecycle sections) cannot be enforced by the type system alone. For each such attribute, modules MUST add a `lifecycle` `precondition` block on the primary resource (the cluster resource in a cluster module, the node-pool resource in a node-pool module). The condition MUST check that `local.cfg.` is not `null`. The error message MUST name the attribute and state that it is required, e.g.: `" must be set — Kubestack does not provide a default for this value."`. +- Because all `configuration` object attributes must be marked `optional(...)` to support inheritance, values that have no module default and must always be user-provided (region, zones, instance type, and `min`/`max` node counts — see Multi-Zone Resilience and Node Pool Lifecycle sections) cannot be enforced by the type system alone. For each such attribute, modules MUST add a `lifecycle` `precondition` block on the primary resource (the cluster resource in a cluster module, the node-pool resource in a node-pool module). The condition MUST check that `local.cfg.` is not `null`. The error message MUST name the attribute and state that it is required, e.g.: `"missing required configuration attribute: "`. ### Multi-Zone Resilience @@ -399,4 +400,34 @@ The Kubestack framework is released as three asset types. Refer to the Mandatory |---|---| | **Versioned modules** | Consumed via `module` blocks using GitHub URLs. | | **Container image** | Base image for CI/CD pipelines and manual tasks, bundling cloud provider CLIs for IAM authentication and debugging, and Kustomize, OpenTofu/Terraform binaries at the correct version (`oci/Dockerfile`). | -| **Quickstarts** | Example directory layouts for bootstrapping new user repositories. Quickstart examples MUST include `name_prefix`, `base_domain`, region, zones, instance type, and autoscaling `min`/`max` — all values that have no module default and must always be user-provided. Leave all other values absent to rely on module and provider defaults. Common files (`README.md`, `.gitignore`, `.user/`) are shared via symlinks pointing to `quickstart/src/configurations/_shared/`. | +| **Quickstarts** | Example directory layouts for bootstrapping new user repositories. Quickstart examples MUST include `name_prefix`, `base_domain`, region, zones, instance type, and autoscaling `min`/`max` — all values that have no module default and must always be user-provided. Required attributes that are guarded by a `lifecycle precondition` MUST be represented as a commented-out example line (see Quickstart Placeholder and CI Injection Pattern below) so that users who forget to set them receive the helpful precondition error message rather than a cryptic provider error. Leave all other values absent to rely on module and provider defaults. Common files (`README.md`, `.gitignore`, `.user/`) are shared via symlinks pointing to `quickstart/src/configurations/_shared/`. | + +### Quickstart Placeholder and CI Injection Pattern + +Attributes guarded by a `lifecycle precondition` — those with no module default that must always be user-provided — MUST be represented in quickstart files as a **commented-out example line**, not as an active assignment with an empty value. This ensures that a user who clones a quickstart and runs `tofu plan` without filling in required values receives the helpful precondition error message (e.g. `missing required configuration attribute: region`) rather than a cryptic provider-level error caused by an empty string or empty list being passed through. + +**Quickstart file convention for precondition-guarded attributes:** + +```hcl +# region = "eu-west-1" +``` + +Because the key is absent (commented out), OpenTofu evaluates it as `null`, which triggers the precondition. The comment serves as inline documentation showing the user exactly what format the value must take. + +**Non-precondition-guarded required values** (e.g. `name_prefix`, `base_domain`) continue to use an active assignment with an empty string placeholder (`= ""`), because they are not null-checked by a precondition and the empty value is intentional as a prompt to the user. + +**CI injection:** The CI pipeline injects real values before running `tofu validate` and `tofu plan` using `sed` substitution in the **"Configure Kubestack"** step of `.github/workflows/main.yml`. For commented-out example lines the `sed` command removes the leading `# ` to activate the line: + +```bash +# SCW: set region +sed -i 's/# region = "fr-par"/region = "fr-par"/g' scw_zero_cluster.tf || true +``` + +**Rules:** + +- Every precondition-guarded attribute MUST have a commented-out example line in every affected quickstart file, using a realistic placeholder value that matches the expected format. +- Every such commented-out line MUST have a corresponding `sed` line in the **"Configure Kubestack"** CI step that activates it by removing the `# ` prefix, following the comment convention `# : set `. +- The `|| true` suffix on `sed` commands targeting provider-specific files (e.g. `aks_zero_cluster.tf`) is required so that the step does not fail when that file is absent in a single-provider quickstart. +- When a new precondition-guarded attribute is added to a module, you MUST: + 1. Add a commented-out example line for it in every affected quickstart file. + 2. Add a matching `sed` line to the **"Configure Kubestack"** step in `.github/workflows/main.yml`. diff --git a/DEVIATIONS.md b/DEVIATIONS.md index 361fb71b..8c3df083 100644 --- a/DEVIATIONS.md +++ b/DEVIATIONS.md @@ -15,8 +15,6 @@ When a divergence is resolved, remove its entry from this file. ### Entries -> **Divergence — All cluster and node-pool modules:** Node Pool Lifecycle — instance type and `min`/`max` node counts — existing modules set hardcoded defaults for instance type and `min`/`max` node counts, violating the rule that these must always be user-provided with no module default. Planned resolution: defaults will be removed. - > **Divergence — `azurerm/cluster`:** Cluster API Authentication — the kubeconfig output uses a client certificate issued via `kube_admin_config` rather than a short-lived Azure AD / Entra ID token, violating the rule that static long-lived credentials must be disabled. Planned resolution: research current upstream default authentication - if defautl is something short lived, switch to that, otherwise keep the cert approach and document the exception. > **Divergence — `azurerm/cluster`:** Networking — the VNet and subnet are only created when `network_plugin = "azure"`; when the default `"kubenet"` plugin is used no dedicated network resources are provisioned by the module, violating the rule that every cluster module MUST create its own dedicated network resources. Planned resolution: switch the network_plugin default from deprecated kubenet to the current upstream recommendation. @@ -25,10 +23,6 @@ When a divergence is resolved, remove its entry from this file. > **Divergence — `azurerm/cluster`:** Cross-Provider Developer Experience — there is no `region` configuration attribute; the Azure region is derived from `data.azurerm_resource_group.current.location` rather than being exposed as a required configuration attribute with a `precondition`. Planned resolution: keep the exception that AKS region is taken from the resource group and document it. -> **Divergence — `azurerm/cluster`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `azurerm_kubernetes_cluster` for required attributes (`resource_group`, `availability_zones`, `vm_size`, `min_count`, `max_count`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - -> **Divergence — `azurerm/cluster/node-pool`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `azurerm_kubernetes_cluster_node_pool` for required attributes (`availability_zones`, `vm_size`, `min_count`, `max_count`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - > **Divergence — `azurerm/cluster`:** Tagging and Labelling — the cluster module exposes `additional_metadata_labels` for user-supplied tags on cloud resources; the upstream `azurerm_kubernetes_cluster` resource argument is `tags`, so the configuration attribute should be named `tags` to mirror it. Planned resolution: rename to `tags`. > **Divergence — `azurerm/cluster/node-pool`:** Tagging and Labelling — `azurerm_kubernetes_cluster_node_pool` has no `tags` block applying metadata labels, violating the rule that all cloud resources MUST be tagged using the metadata module output. Planned resolution: add `cluster_metadata` variable and a `tags` argument referencing its labels. @@ -41,10 +35,6 @@ When a divergence is resolved, remove its entry from this file. > **Divergence — `aws/cluster`:** Cross-Provider Developer Experience — there is no `region` configuration attribute; the AWS region is sourced entirely from the provider configuration rather than being exposed as a required configuration attribute with a `precondition`. Planned resolution: document the special case that `region` for EKS is configured on the provider. The eks quickstart already shows how to alias the provider per cluster. -> **Divergence — `aws/cluster`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `aws_eks_cluster` for required attributes (`cluster_availability_zones`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - -> **Divergence — `aws/cluster/node-pool`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `aws_eks_node_group` for required attributes (`availability_zones`, `instance_types`, `min_size`, `max_size`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - > **Divergence — `aws/cluster`:** Tagging and Labelling — user-supplied cloud resource tags are only configurable via `additional_node_tags` nested inside `default_node_pool`, meaning there is no top-level `tags` attribute on the cluster configuration object and extra-node-pool resources (VPC, subnets, security groups, etc.) cannot receive user-supplied tags. Planned resolution: add a top-level `tags` attribute to the cluster configuration. > **Divergence — `aws/cluster/node-pool`:** Tagging and Labelling — the node-pool `labels` configuration attribute is not applied to the Kubernetes node object's `.metadata.labels`; EKS node group labels are set on the cloud API only, violating the rule that node labels MUST be applied to both the cloud API and the Kubernetes node object. Planned resolution: apply labels to both APIs. @@ -57,13 +47,9 @@ When a divergence is resolved, remove its entry from this file. > **Divergence — `google/cluster/node-pool`:** Tagging and Labelling — the merge order in `node_config.labels` is `merge(cfg.labels, cluster_metadata.labels)`, placing metadata labels last so they override user-supplied labels, which is the inverse of the required precedence (metadata labels → user-supplied cloud resource labels → user-supplied node labels → mandatory provider labels). Planned resolution: fix the merge order. -> **Divergence — `google/cluster`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `google_container_cluster` for required attributes (`region`, `project_id`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - -> **Divergence — `google/cluster/node-pool`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `google_container_node_pool` for required attributes (`location`, `machine_type`, `min_node_count`, `max_node_count`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - > **Divergence — `google/cluster/node-pool`:** Node Pool Lifecycle — `google_container_node_pool` sets `initial_node_count` but has no `lifecycle { ignore_changes = [initial_node_count] }` block, violating the rule that the desired/initial node count must be owned by the autoscaler and not managed by OpenTofu/Terraform state. Planned resolution: add the `lifecycle` ignore block. -> **Divergence — `google/cluster/node-pool`:** Node Pool Lifecycle — `machine_type` defaults to `""` (empty string) via `try(coalesce(local.cfg.machine_type, null), "")`, which the GCP provider interprets as a provider-level default, violating the rule that instance type must always be user-provided with no module default. Planned resolution: remove the empty-string default in the v1 release (covered by the existing "All cluster and node-pool modules" divergence but noted here for the specific empty-string mechanism). + > **Divergence — `google/cluster`:** Cluster Module Outputs — the cluster module exposes a `default_ingress_ip` output beyond the four required outputs, without a compelling provider-specific justification. Planned resolution: keep for historical reasons to avoid braking change for existing users, document exception. @@ -71,10 +57,6 @@ When a divergence is resolved, remove its entry from this file. > **Divergence — `google/cluster`:** Required File Layout — `provider.tf` contains the data source `data.google_client_config.default`, which is used by both `kubeconfig.tf` and the kubernetes provider alias block in the same file; per the data source placement rule, a data source used by multiple resources belongs in `data_sources.tf`, not co-located with a provider block. Planned resolution: move the data source to `data_sources.tf`. -> **Divergence — `scaleway/cluster`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `scaleway_k8s_cluster` for required attributes (`region`, `cluster_version`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - -> **Divergence — `scaleway/cluster/node-pool`:** Configuration Inheritance — `precondition` lifecycle blocks are absent from `scaleway_k8s_pool` for required attributes (`zones`, `node_type`, `min_size`, `max_size`), violating the rule that required attributes with no module default must be guarded by a precondition on the primary resource. Planned resolution: add preconditions. - > **Divergence — `scaleway/cluster`:** Networking — nodes are assigned public IPs by default (`public_ip_disabled` defaults to `false` in `node_pool.tf`), violating the rule that nodes MUST be configured with private IPs only by default. Planned resolution: flip the default to `true` and add any additional resources that need to be created; if these additioanl resources are only required in one or the other case make them conditional. > **Divergence — `scaleway/cluster/node-pool`:** Tagging and Labelling — the node-pool module exposes no configuration attribute for Kubernetes node labels; node labels can only be approximated by encoding them as Scaleway tag strings, with no guarantee that they are applied to the Kubernetes node object's `.metadata.labels`, violating the rule that node labels MUST be applied to both the cloud API and the Kubernetes node object. Planned resolution: add a dedicated node label attribute once the Scaleway provider exposes first-class Kubernetes node label support. diff --git a/aws/cluster/main.tf b/aws/cluster/main.tf index 0713bef8..149c0f22 100644 --- a/aws/cluster/main.tf +++ b/aws/cluster/main.tf @@ -74,6 +74,13 @@ resource "aws_eks_cluster" "current" { aws_iam_role_policy_attachment.master_service_policy, ] + lifecycle { + precondition { + condition = local.cfg.cluster_availability_zones != null + error_message = "missing required configuration attribute: cluster_availability_zones" + } + } + version = local.cfg.cluster_version enabled_cluster_log_types = try(coalesce(local.cfg.enabled_cluster_log_types, null), ["api", "audit", "authenticator", "controllerManager", "scheduler"]) diff --git a/aws/cluster/node-pool/main.tf b/aws/cluster/node-pool/main.tf index 55d9d333..52555b84 100644 --- a/aws/cluster/node-pool/main.tf +++ b/aws/cluster/node-pool/main.tf @@ -15,6 +15,32 @@ resource "aws_eks_node_group" "nodes" { cluster_name = var.cluster.name node_group_name = local.cfg.name node_role_arn = data.aws_iam_role.node.arn + + lifecycle { + # when autoscaler is enabled, desired_size needs to be ignored + ignore_changes = [scaling_config[0].desired_size] + + precondition { + condition = local.cfg.availability_zones != null + error_message = "missing required configuration attribute: availability_zones" + } + + precondition { + condition = local.cfg.instance_types != null + error_message = "missing required configuration attribute: instance_types" + } + + precondition { + condition = local.cfg.min_size != null + error_message = "missing required configuration attribute: min_size" + } + + precondition { + condition = local.cfg.max_size != null + error_message = "missing required configuration attribute: max_size" + } + } + subnet_ids = coalesce( # first prio, user specified subnet ids try(coalesce(var.cluster_default_node_pool_subnet_ids, local.cfg.vpc_subnet_ids), null), @@ -62,11 +88,4 @@ resource "aws_eks_node_group" "nodes" { effect = taint.value.effect } } - - # when autoscaler is enabled, desired_size needs to be ignored - # better would be to handle this in the resource and not require - # desired_size, min_size and max_size in scaling_config - lifecycle { - ignore_changes = [scaling_config[0].desired_size] - } } diff --git a/aws/cluster/node_pool.tf b/aws/cluster/node_pool.tf index 4c47d7f3..9d5999eb 100644 --- a/aws/cluster/node_pool.tf +++ b/aws/cluster/node_pool.tf @@ -11,12 +11,13 @@ module "node_pool" { (terraform.workspace) = { name = "default" + availability_zones = try(local.cfg.cluster_availability_zones, null) instance_types = try(local.cfg.default_node_pool.instance_types, null) ami_type = try(local.cfg.default_node_pool.ami_type, null) ami_release_version = try(local.cfg.default_node_pool.ami_release_version, null) desired_capacity = try(coalesce(local.cfg.default_node_pool.desired_capacity, null), 1) - min_size = try(coalesce(local.cfg.default_node_pool.min_size, null), 1) - max_size = try(coalesce(local.cfg.default_node_pool.max_size, null), 1) + min_size = try(local.cfg.default_node_pool.min_size, null) + max_size = try(local.cfg.default_node_pool.max_size, null) disk_size = try(coalesce(local.cfg.default_node_pool.root_device_volume_size, null), 20) metadata_options = try(local.cfg.default_node_pool.metadata_options, null) @@ -32,4 +33,3 @@ module "node_pool" { depends_on = [kubernetes_config_map.current] } - diff --git a/azurerm/cluster/main.tf b/azurerm/cluster/main.tf index ac607395..75f40b06 100644 --- a/azurerm/cluster/main.tf +++ b/azurerm/cluster/main.tf @@ -26,9 +26,37 @@ data "azurerm_resource_group" "current" { } resource "azurerm_kubernetes_cluster" "current" { - name = module.cluster_metadata.name - location = data.azurerm_resource_group.current.location - resource_group_name = data.azurerm_resource_group.current.name + name = module.cluster_metadata.name + location = data.azurerm_resource_group.current.location + resource_group_name = data.azurerm_resource_group.current.name + + lifecycle { + precondition { + condition = local.cfg.resource_group != null + error_message = "missing required configuration attribute: resource_group" + } + + precondition { + condition = local.cfg.availability_zones != null + error_message = "missing required configuration attribute: availability_zones" + } + + precondition { + condition = try(local.cfg.default_node_pool.vm_size, null) != null + error_message = "missing required configuration attribute: default_node_pool.vm_size" + } + + precondition { + condition = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) == false || try(local.cfg.default_node_pool.min_count, null) != null + error_message = "missing required configuration attribute: default_node_pool.min_count" + } + + precondition { + condition = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) == false || try(local.cfg.default_node_pool.max_count, null) != null + error_message = "missing required configuration attribute: default_node_pool.max_count" + } + } + dns_prefix = try(coalesce(local.cfg.dns_prefix, null), "api") sku_tier = try(coalesce(local.cfg.sku_tier, null), "Free") kubernetes_version = local.cfg.kubernetes_version @@ -43,13 +71,13 @@ resource "azurerm_kubernetes_cluster" "current" { auto_scaling_enabled = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) # set min and max count only if autoscaling is _enabled_ - min_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? try(coalesce(local.cfg.default_node_pool.min_count, null), 1) : null - max_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? try(coalesce(local.cfg.default_node_pool.max_count, null), 1) : null + min_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? local.cfg.default_node_pool.min_count : null + max_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? local.cfg.default_node_pool.max_count : null # set node count only if auto scaling is _disabled_ node_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? null : try(coalesce(local.cfg.default_node_pool.node_count, null), 1) - vm_size = try(coalesce(local.cfg.default_node_pool.vm_size, null), "Standard_D1_v2") + vm_size = local.cfg.default_node_pool.vm_size os_disk_size_gb = try(coalesce(local.cfg.default_node_pool.os_disk_size_gb, null), 30) vnet_subnet_id = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? azurerm_subnet.current[0].id : null @@ -57,7 +85,7 @@ resource "azurerm_kubernetes_cluster" "current" { only_critical_addons_enabled = try(coalesce(local.cfg.default_node_pool.only_critical_addons, null), false) - zones = try(coalesce(local.cfg.availability_zones, null), []) + zones = local.cfg.availability_zones upgrade_settings { max_surge = try(coalesce(try(local.cfg.default_node_pool.upgrade_settings, null).max_surge, null), "10%") diff --git a/azurerm/cluster/node-pool/main.tf b/azurerm/cluster/node-pool/main.tf index 2d8ba627..6ee74ba9 100644 --- a/azurerm/cluster/node-pool/main.tf +++ b/azurerm/cluster/node-pool/main.tf @@ -31,19 +31,43 @@ resource "azurerm_kubernetes_cluster_node_pool" "current" { name = local.cfg.node_pool_name kubernetes_cluster_id = var.cluster.id auto_scaling_enabled = try(coalesce(local.cfg.enable_auto_scaling, null), true) - max_count = local.effective_max_count - min_count = local.effective_min_count - node_count = try(coalesce(local.cfg.node_count, null), 1) - vm_size = try(coalesce(local.cfg.vm_size, null), "Standard_B2s") - node_labels = local.node_labels_merged - node_taints = local.node_taints_merged - zones = local.cfg.availability_zones - max_pods = try(coalesce(local.cfg.max_pods, null), 110) - os_disk_type = try(coalesce(local.cfg.os_disk_type, null), "Managed") - os_disk_size_gb = local.cfg.os_disk_size_gb - priority = local.effective_priority - eviction_policy = local.effective_eviction_policy - spot_max_price = local.effective_max_spot_price + + lifecycle { + ignore_changes = [node_count] + + precondition { + condition = local.cfg.availability_zones != null + error_message = "missing required configuration attribute: availability_zones" + } + + precondition { + condition = local.cfg.vm_size != null + error_message = "missing required configuration attribute: vm_size" + } + + precondition { + condition = try(coalesce(local.cfg.enable_auto_scaling, null), true) == false || local.cfg.min_count != null + error_message = "missing required configuration attribute: min_count" + } + + precondition { + condition = try(coalesce(local.cfg.enable_auto_scaling, null), true) == false || local.cfg.max_count != null + error_message = "missing required configuration attribute: max_count" + } + } + max_count = local.effective_max_count + min_count = local.effective_min_count + node_count = try(coalesce(local.cfg.node_count, null), 1) + vm_size = local.cfg.vm_size + node_labels = local.node_labels_merged + node_taints = local.node_taints_merged + zones = local.cfg.availability_zones + max_pods = try(coalesce(local.cfg.max_pods, null), 110) + os_disk_type = try(coalesce(local.cfg.os_disk_type, null), "Managed") + os_disk_size_gb = local.cfg.os_disk_size_gb + priority = local.effective_priority + eviction_policy = local.effective_eviction_policy + spot_max_price = local.effective_max_spot_price upgrade_settings { max_surge = try(coalesce(try(local.cfg.upgrade_settings, null).max_surge, null), "10%") @@ -54,10 +78,4 @@ resource "azurerm_kubernetes_cluster_node_pool" "current" { # The cluster's agent_pool_profile may contain empty strings for vnet_subnet_id # in some configurations. In that case we rely on the defaults. vnet_subnet_id = length(local.vnet_subnets) == 0 ? null : coalesce(tolist(local.vnet_subnets)...) - - # When autoscaling acts, the node_count gets changed, but it should not be - # forced to match the config - lifecycle { - ignore_changes = [node_count] - } } diff --git a/google/cluster/main.tf b/google/cluster/main.tf index 8cc49879..3346bcaf 100644 --- a/google/cluster/main.tf +++ b/google/cluster/main.tf @@ -22,6 +22,18 @@ resource "google_container_cluster" "current" { project = local.cfg.project_id name = module.cluster_metadata.name + lifecycle { + precondition { + condition = local.cfg.region != null + error_message = "missing required configuration attribute: region" + } + + precondition { + condition = local.cfg.project_id != null + error_message = "missing required configuration attribute: project_id" + } + } + deletion_protection = local.cfg.deletion_protection location = local.cfg.region diff --git a/google/cluster/node-pool/main.tf b/google/cluster/node-pool/main.tf index 435ea7dc..3932d9f9 100644 --- a/google/cluster/node-pool/main.tf +++ b/google/cluster/node-pool/main.tf @@ -27,11 +27,33 @@ resource "google_container_node_pool" "current" { cluster = var.cluster.name location = local.cfg.location + lifecycle { + precondition { + condition = local.cfg.location != null + error_message = "missing required configuration attribute: location" + } + + precondition { + condition = local.cfg.machine_type != null + error_message = "missing required configuration attribute: machine_type" + } + + precondition { + condition = local.cfg.min_node_count != null + error_message = "missing required configuration attribute: min_node_count" + } + + precondition { + condition = local.cfg.max_node_count != null + error_message = "missing required configuration attribute: max_node_count" + } + } + initial_node_count = local.cfg.initial_node_count autoscaling { - min_node_count = try(coalesce(local.cfg.min_node_count, null), 1) - max_node_count = try(coalesce(local.cfg.max_node_count, null), 1) + min_node_count = local.cfg.min_node_count + max_node_count = local.cfg.max_node_count location_policy = try(coalesce(local.cfg.location_policy, null), "BALANCED") } @@ -59,7 +81,7 @@ resource "google_container_node_pool" "current" { disk_type = try(coalesce(local.cfg.disk_type, null), "pd-balanced") image_type = try(coalesce(local.cfg.image_type, null), "COS_CONTAINERD") - machine_type = try(coalesce(local.cfg.machine_type, null), "") + machine_type = local.cfg.machine_type preemptible = try(coalesce(local.cfg.preemptible, null), false) labels = merge(try(coalesce(local.cfg.labels, null), {}), var.cluster_metadata.labels) diff --git a/google/cluster/node_pool.tf b/google/cluster/node_pool.tf index f4d5751e..16713278 100644 --- a/google/cluster/node_pool.tf +++ b/google/cluster/node_pool.tf @@ -13,8 +13,8 @@ module "node_pool" { service_account_email = google_service_account.current.email initial_node_count = try(coalesce(local.cfg.default_node_pool.initial_node_count, null), 1) - min_node_count = try(coalesce(local.cfg.default_node_pool.min_node_count, null), 1) - max_node_count = try(coalesce(local.cfg.default_node_pool.max_node_count, null), 1) + min_node_count = try(local.cfg.default_node_pool.min_node_count, null) + max_node_count = try(local.cfg.default_node_pool.max_node_count, null) location_policy = try(local.cfg.default_node_pool.node_location_policy, null) extra_oauth_scopes = try(coalesce(local.cfg.default_node_pool.extra_oauth_scopes, null), []) @@ -22,7 +22,7 @@ module "node_pool" { disk_size_gb = try(coalesce(local.cfg.default_node_pool.disk_size_gb, null), 100) disk_type = try(coalesce(local.cfg.default_node_pool.disk_type, null), "pd-standard") image_type = try(coalesce(local.cfg.default_node_pool.image_type, null), "COS_containerd") - machine_type = try(coalesce(local.cfg.default_node_pool.machine_type, null), "") + machine_type = try(local.cfg.default_node_pool.machine_type, null) preemptible = try(coalesce(local.cfg.default_node_pool.preemptible, null), false) auto_repair = try(coalesce(local.cfg.default_node_pool.auto_repair, null), true) diff --git a/quickstart/src/configurations/aks/aks_zero_cluster.tf b/quickstart/src/configurations/aks/aks_zero_cluster.tf index d39b87d4..8351afc0 100644 --- a/quickstart/src/configurations/aks/aks_zero_cluster.tf +++ b/quickstart/src/configurations/aks/aks_zero_cluster.tf @@ -17,7 +17,21 @@ module "aks_zero" { base_domain = "" # The Azure resource group to use - resource_group = "" + # resource_group = "my-resource-group" + + # Availability zones to distribute the cluster and default node pool across. + # Azure zone numbers as strings. + # availability_zones = ["1", "2", "3"] + + # Default node pool configuration + default_node_pool = { + # Azure VM size for default node pool nodes + # vm_size = "Standard_D2s_v3" + + # Min and max node counts for autoscaling + min_count = 1 + max_count = 1 + } # CNI/Advanced networking configuration parameters. # Leave commented for default 'kubenet' networking diff --git a/quickstart/src/configurations/eks/eks_zero_cluster.tf b/quickstart/src/configurations/eks/eks_zero_cluster.tf index 40006d95..2360263e 100644 --- a/quickstart/src/configurations/eks/eks_zero_cluster.tf +++ b/quickstart/src/configurations/eks/eks_zero_cluster.tf @@ -30,8 +30,7 @@ module "eks_zero" { # EKS requires a min. of 2 zones # Must match region set in provider - # e.g. cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] - cluster_availability_zones = [] + # cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] } # ops environment, inherits from apps diff --git a/quickstart/src/configurations/gke/gke_zero_cluster.tf b/quickstart/src/configurations/gke/gke_zero_cluster.tf index 90a70206..816430d2 100644 --- a/quickstart/src/configurations/gke/gke_zero_cluster.tf +++ b/quickstart/src/configurations/gke/gke_zero_cluster.tf @@ -9,7 +9,7 @@ module "gke_zero" { # apps environment apps = { # The Google cloud project ID to use - project_id = "" + # project_id = "my-gcp-project" # Set name_prefix used to generate the cluster_name # [name_prefix]-[workspace]-[region] @@ -26,13 +26,18 @@ module "gke_zero" { # Initial desired K8s version, will be upgraded automatically cluster_min_master_version = "1.22" - # Initial number of desired nodes per zone + # Default node pool configuration default_node_pool = { - initial_node_count = 1 + # GCP machine type for default node pool nodes + # machine_type = "e2-medium" + + # Min and max node counts for autoscaling + # min_node_count = 1 + # max_node_count = 3 } # The Google cloud region to deploy the clusters in - region = "" + # region = "europe-west1" # Comma-separated list of zone names to deploy worker nodes in. # Must match region above. diff --git a/quickstart/src/configurations/multi-cloud/aks_zero_cluster.tf b/quickstart/src/configurations/multi-cloud/aks_zero_cluster.tf index eacca54a..0a7184b7 100644 --- a/quickstart/src/configurations/multi-cloud/aks_zero_cluster.tf +++ b/quickstart/src/configurations/multi-cloud/aks_zero_cluster.tf @@ -17,7 +17,21 @@ module "aks_zero" { base_domain = "" # The Azure resource group to use - resource_group = "" + # resource_group = "my-resource-group" + + # Availability zones to distribute the cluster across. + # Must be a list of zone numbers as strings. + # availability_zones = ["1", "2", "3"] + + # Default node pool configuration + default_node_pool = { + # Azure VM size for the default node pool nodes. + # vm_size = "Standard_B2s" + + # Minimum and maximum node count for autoscaling + min_count = 1 + max_count = 1 + } } # Settings for Ops-cluster diff --git a/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf b/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf index 3f1ea76d..974a59d8 100644 --- a/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf +++ b/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf @@ -30,8 +30,7 @@ module "eks_zero" { # EKS requires a min. of 2 zones # Must match region set in provider - # e.g. cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] - cluster_availability_zones = [] + # cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] } # Settings for Ops-cluster @@ -39,8 +38,7 @@ module "eks_zero" { # Overwrite apps["cluster_availability_zones"] to have a smaller # ops cluster # EKS requires a min. of 2 zones - # e.g. cluster_availability_zones = ["eu-west-1a", "eu-west-1b"] - cluster_availability_zones = [] + # cluster_availability_zones = ["eu-west-1a", "eu-west-1b"] } } } diff --git a/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf b/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf index a1e24bd5..29ee485d 100644 --- a/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf +++ b/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf @@ -1,6 +1,6 @@ module "gke_zero" { providers = { - kubernetes = kubernetes.eks_zero + kubernetes = kubernetes.gke_zero } source = "github.com/kbst/terraform-kubestack//google/cluster?ref={{version}}" @@ -9,7 +9,7 @@ module "gke_zero" { # Settings for Apps-cluster apps = { # The Google cloud project ID to use - project_id = "" + # project_id = "my-gcp-project" # Set name_prefix used to generate the cluster_name # [name_prefix]-[workspace]-[region] @@ -26,13 +26,18 @@ module "gke_zero" { # Initial desired K8s version, will be upgraded automatically cluster_min_master_version = "1.22" - # Initial number of desired nodes per zone + # Default node pool configuration default_node_pool = { - initial_node_count = 1 + # GCP machine type for default node pool nodes + # machine_type = "e2-medium" + + # Min and max node counts for autoscaling + # min_node_count = 1 + # max_node_count = 3 } # The Google cloud region to deploy the clusters in - region = "" + # region = "europe-west1" # Comma-separated list of zone names to deploy worker nodes in. # Must match region above. diff --git a/quickstart/src/configurations/multi-cloud/scw_zero_cluster.tf b/quickstart/src/configurations/multi-cloud/scw_zero_cluster.tf index 4c48e42d..2b65d41e 100644 --- a/quickstart/src/configurations/multi-cloud/scw_zero_cluster.tf +++ b/quickstart/src/configurations/multi-cloud/scw_zero_cluster.tf @@ -19,7 +19,7 @@ module "scw_zero" { # The Scaleway region to deploy the clusters in # e.g. "fr-par", "nl-ams", "pl-waw" - region = "" + # region = "fr-par" # Kubernetes version for the cluster # Use a minor version string to allow patch upgrades, e.g. "1.32" diff --git a/quickstart/src/configurations/scw/scw_zero_cluster.tf b/quickstart/src/configurations/scw/scw_zero_cluster.tf index 4eea6d2a..136e4ce4 100644 --- a/quickstart/src/configurations/scw/scw_zero_cluster.tf +++ b/quickstart/src/configurations/scw/scw_zero_cluster.tf @@ -18,8 +18,8 @@ module "scw_zero" { base_domain = "" # The Scaleway region to deploy the cluster in - # e.g. "fr-par", "nl-ams", "pl-waw" - region = "" + # Uncomment and set to your target region, e.g: + # region = "fr-par" default_node_pool = { # Commercial node type — must be available in every zone listed below. @@ -28,10 +28,10 @@ module "scw_zero" { # Availability zones to distribute the default node pool across. # One pool is created per zone. Must be explicitly specified. - # e.g. fr-par: ["fr-par-1", "fr-par-2", "fr-par-3"] - # e.g. nl-ams: ["nl-ams-1", "nl-ams-2", "nl-ams-3"] - # e.g. pl-waw: ["pl-waw-1", "pl-waw-2", "pl-waw-3"] - zones = [] + # fr-par: zones = ["fr-par-1", "fr-par-2", "fr-par-3"] + # nl-ams: zones = ["nl-ams-1", "nl-ams-2", "nl-ams-3"] + # pl-waw: zones = ["pl-waw-1", "pl-waw-2", "pl-waw-3"] + # zones = ["fr-par-1", "fr-par-2", "fr-par-3"] min_size = 1 max_size = 2 diff --git a/scaleway/cluster/main.tf b/scaleway/cluster/main.tf index 3fa73866..1d57df2d 100644 --- a/scaleway/cluster/main.tf +++ b/scaleway/cluster/main.tf @@ -22,7 +22,19 @@ resource "scaleway_k8s_cluster" "current" { name = module.cluster_metadata.name project_id = local.cfg.project_id region = local.cfg.region - tags = concat(module.cluster_metadata.tags, try(coalesce(local.cfg.extra_tags, null), [])) + + lifecycle { + precondition { + condition = local.cfg.region != null + error_message = "missing required configuration attribute: region" + } + + precondition { + condition = local.cfg.cluster_version != null + error_message = "missing required configuration attribute: cluster_version" + } + } + tags = concat(module.cluster_metadata.tags, try(coalesce(local.cfg.extra_tags, null), [])) type = "kapsule" version = local.cfg.cluster_version diff --git a/scaleway/cluster/node-pool/main.tf b/scaleway/cluster/node-pool/main.tf index 604958b7..d1cc6046 100644 --- a/scaleway/cluster/node-pool/main.tf +++ b/scaleway/cluster/node-pool/main.tf @@ -20,17 +20,17 @@ locals { } resource "scaleway_k8s_pool" "current" { - for_each = toset(local.cfg.zones) + for_each = toset(coalesce(local.cfg.zones, [])) cluster_id = var.cluster.id name = "${local.cfg.name}-${each.value}" region = var.cluster.region zone = each.value - node_type = try(coalesce(local.cfg.node_type, null), "GP1-XS") + node_type = local.cfg.node_type size = try(coalesce(local.cfg.size, null), 1) - min_size = try(coalesce(local.cfg.min_size, null), 1) - max_size = try(coalesce(local.cfg.max_size, null), 2) + min_size = local.cfg.min_size + max_size = local.cfg.max_size autoscaling = try(coalesce(local.cfg.autoscaling, null), true) autohealing = try(coalesce(local.cfg.autohealing, null), true) @@ -60,5 +60,25 @@ resource "scaleway_k8s_pool" "current" { # be forced back to match the configuration on the next plan. lifecycle { ignore_changes = [size] + + precondition { + condition = local.cfg.zones != null && length(local.cfg.zones) > 0 + error_message = "missing required configuration attribute: zones" + } + + precondition { + condition = local.cfg.node_type != null + error_message = "missing required configuration attribute: node_type" + } + + precondition { + condition = local.cfg.min_size != null + error_message = "missing required configuration attribute: min_size" + } + + precondition { + condition = local.cfg.max_size != null + error_message = "missing required configuration attribute: max_size" + } } } diff --git a/scaleway/cluster/node_pool.tf b/scaleway/cluster/node_pool.tf index 81eefff5..28f483c2 100644 --- a/scaleway/cluster/node_pool.tf +++ b/scaleway/cluster/node_pool.tf @@ -9,10 +9,10 @@ module "node_pool" { name = "default" zones = local.cfg.default_node_pool.zones - node_type = try(coalesce(local.cfg.default_node_pool.node_type, null), "GP1-XS") + node_type = try(local.cfg.default_node_pool.node_type, null) size = try(coalesce(local.cfg.default_node_pool.size, null), 1) - min_size = try(coalesce(local.cfg.default_node_pool.min_size, null), 1) - max_size = try(coalesce(local.cfg.default_node_pool.max_size, null), 2) + min_size = try(local.cfg.default_node_pool.min_size, null) + max_size = try(local.cfg.default_node_pool.max_size, null) autoscaling = try(coalesce(local.cfg.default_node_pool.autoscaling, null), true) autohealing = try(coalesce(local.cfg.default_node_pool.autohealing, null), true) diff --git a/tests/aks_zero_cluster.tf b/tests/aks_zero_cluster.tf index a4241521..061394ea 100644 --- a/tests/aks_zero_cluster.tf +++ b/tests/aks_zero_cluster.tf @@ -8,6 +8,8 @@ module "aks_zero" { name_prefix = "kbstacctest" base_domain = "infra.serverwolken.de" + availability_zones = ["1", "2", "3"] + default_node_pool = { vm_size = "Standard_B2s" min_count = 1 diff --git a/tests/aks_zero_node_pools.tf b/tests/aks_zero_node_pools.tf index e1db62e7..e4236904 100644 --- a/tests/aks_zero_node_pools.tf +++ b/tests/aks_zero_node_pools.tf @@ -8,6 +8,8 @@ module "aks_zero_node_pool" { apps = { node_pool_name = "test1" + availability_zones = ["1", "2", "3"] + vm_size = "Standard_B2s" min_count = 1 max_count = 1