From 409189c8445518b4e5a8caa3e1f84488d0095c36 Mon Sep 17 00:00:00 2001 From: Philipp Strube Date: Thu, 12 Feb 2026 09:01:52 +0100 Subject: [PATCH 1/2] Abandon cluster-local experiment --- .github/workflows/main.yml | 8 +-- aws/cluster-local/configuration.tf | 40 ------------- aws/cluster-local/elb-dns/variables.tf | 14 ----- aws/cluster-local/elb-dns/versions.tf | 13 ----- aws/cluster-local/main.tf | 26 --------- aws/cluster-local/outputs.tf | 11 ---- aws/cluster-local/variables.tf | 10 ---- aws/cluster-local/versions.tf | 9 --- azurerm/cluster-local/configuration.tf | 41 ------------- azurerm/cluster-local/main.tf | 29 ---------- azurerm/cluster-local/outputs.tf | 19 ------ azurerm/cluster-local/variables.tf | 10 ---- azurerm/cluster-local/versions.tf | 4 -- google/cluster-local/configuration.tf | 42 -------------- google/cluster-local/main.tf | 26 --------- google/cluster-local/outputs.tf | 18 ------ google/cluster-local/variables.tf | 10 ---- google/cluster-local/versions.tf | 4 -- kind/_modules/kind/main.tf | 58 ------------------- kind/_modules/kind/outputs.tf | 4 -- kind/_modules/kind/variables.tf | 44 -------------- kind/_modules/kind/versions.tf | 11 ---- kind/cluster/configuration.tf | 26 --------- kind/cluster/main.tf | 26 --------- kind/cluster/outputs.tf | 12 ---- kind/cluster/variables.tf | 16 ----- kind/cluster/versions.tf | 3 - oci/Dockerfile | 30 ---------- .../configurations/aks/aks_zero_cluster.tf | 3 - .../configurations/eks/eks_zero_cluster.tf | 3 - .../configurations/gke/gke_zero_cluster.tf | 3 - quickstart/src/configurations/kind/.gitignore | 1 - quickstart/src/configurations/kind/.user | 1 - quickstart/src/configurations/kind/Dockerfile | 1 - quickstart/src/configurations/kind/README.md | 1 - .../configurations/kind/kind_zero_cluster.tf | 22 ------- .../kind/kind_zero_providers.tf | 4 -- .../src/configurations/kind/versions.tf | 9 --- .../multi-cloud/aks_zero_cluster.tf | 2 - .../multi-cloud/eks_zero_cluster.tf | 2 - .../multi-cloud/gke_zero_cluster.tf | 2 - 41 files changed, 4 insertions(+), 614 deletions(-) delete mode 100644 aws/cluster-local/configuration.tf delete mode 100644 aws/cluster-local/elb-dns/variables.tf delete mode 100644 aws/cluster-local/elb-dns/versions.tf delete mode 100644 aws/cluster-local/main.tf delete mode 100644 aws/cluster-local/outputs.tf delete mode 100644 aws/cluster-local/variables.tf delete mode 100644 aws/cluster-local/versions.tf delete mode 100644 azurerm/cluster-local/configuration.tf delete mode 100644 azurerm/cluster-local/main.tf delete mode 100644 azurerm/cluster-local/outputs.tf delete mode 100644 azurerm/cluster-local/variables.tf delete mode 100644 azurerm/cluster-local/versions.tf delete mode 100644 google/cluster-local/configuration.tf delete mode 100644 google/cluster-local/main.tf delete mode 100644 google/cluster-local/outputs.tf delete mode 100644 google/cluster-local/variables.tf delete mode 100644 google/cluster-local/versions.tf delete mode 100644 kind/_modules/kind/main.tf delete mode 100644 kind/_modules/kind/outputs.tf delete mode 100644 kind/_modules/kind/variables.tf delete mode 100644 kind/_modules/kind/versions.tf delete mode 100644 kind/cluster/configuration.tf delete mode 100644 kind/cluster/main.tf delete mode 100644 kind/cluster/outputs.tf delete mode 100644 kind/cluster/variables.tf delete mode 100644 kind/cluster/versions.tf delete mode 120000 quickstart/src/configurations/kind/.gitignore delete mode 120000 quickstart/src/configurations/kind/.user delete mode 100644 quickstart/src/configurations/kind/Dockerfile delete mode 120000 quickstart/src/configurations/kind/README.md delete mode 100644 quickstart/src/configurations/kind/kind_zero_cluster.tf delete mode 100644 quickstart/src/configurations/kind/kind_zero_providers.tf delete mode 100644 quickstart/src/configurations/kind/versions.tf diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bd70e252..13215e34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,7 +54,7 @@ jobs: needs: [build-test-artifacts] strategy: matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] permissions: id-token: write # required for keyless signing @@ -117,7 +117,7 @@ jobs: strategy: fail-fast: false matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] steps: - name: "Download test-artifacts" @@ -230,7 +230,7 @@ jobs: strategy: matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] steps: - name: "Download test-artifacts" @@ -277,7 +277,7 @@ jobs: strategy: matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] steps: - name: "Download test-artifacts" diff --git a/aws/cluster-local/configuration.tf b/aws/cluster-local/configuration.tf deleted file mode 100644 index de1015a8..00000000 --- a/aws/cluster-local/configuration.tf +++ /dev/null @@ -1,40 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - http_port_default = terraform.workspace == "apps" ? 80 : 8080 - http_port = lookup(local.cfg, "http_port", local.http_port_default) - - https_port_default = terraform.workspace == "apps" ? 443 : 8443 - https_port = lookup(local.cfg, "https_port", local.https_port_default) - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - node_image = lookup(local.cfg, "node_image", null) - - node_count = lookup(local.cfg, "cluster_min_size", 1) - nodes = [ - for node, _ in range(local.node_count) : - "worker" - ] - extra_nodes = join(",", local.nodes) - - # on AWS the region is determined by the provider configuration - # in the local implementation we don't have access to that - # to still support multi-region setups locally, we hash the availability zones - # and use that as the region part of the cluster name prefixed with eks- - cluster_availability_zones_lookup = lookup(local.cfg, "cluster_availability_zones", "") - fake_region_hash = substr(sha256(local.cluster_availability_zones_lookup), 0, 7) - fake_region = "eks-${local.fake_region_hash}" -} diff --git a/aws/cluster-local/elb-dns/variables.tf b/aws/cluster-local/elb-dns/variables.tf deleted file mode 100644 index e54b5fa5..00000000 --- a/aws/cluster-local/elb-dns/variables.tf +++ /dev/null @@ -1,14 +0,0 @@ -variable "ingress_service_name" { - type = string - description = "Metadata name of the ingress service." -} - -variable "ingress_service_namespace" { - type = string - description = "Metadata namespace of the ingress service." -} - -variable "metadata_fqdn" { - type = string - description = "Cluster module FQDN." -} diff --git a/aws/cluster-local/elb-dns/versions.tf b/aws/cluster-local/elb-dns/versions.tf deleted file mode 100644 index b5087a9c..00000000 --- a/aws/cluster-local/elb-dns/versions.tf +++ /dev/null @@ -1,13 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - - kubernetes = { - source = "hashicorp/kubernetes" - } - } - - required_version = ">= 0.13" -} diff --git a/aws/cluster-local/main.tf b/aws/cluster-local/main.tf deleted file mode 100644 index f1ca8694..00000000 --- a/aws/cluster-local/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -module "cluster_metadata" { - source = "../../common/metadata" - - name_prefix = local.name_prefix - base_domain = local.base_domain - - provider_name = "aws" - provider_region = local.fake_region -} - -module "cluster" { - source = "../../kind/_modules/kind" - - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_tags = module.cluster_metadata.tags - metadata_labels = module.cluster_metadata.labels - - node_image = local.node_image - extra_nodes = local.extra_nodes - - http_port = local.http_port - https_port = local.https_port - - disable_default_ingress = local.disable_default_ingress -} diff --git a/aws/cluster-local/outputs.tf b/aws/cluster-local/outputs.tf deleted file mode 100644 index 424f37e9..00000000 --- a/aws/cluster-local/outputs.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "current_config" { - value = module.configuration.merged[terraform.workspace] -} - -output "current_metadata" { - value = module.cluster_metadata -} - -output "kubeconfig" { - value = module.cluster.kubeconfig -} diff --git a/aws/cluster-local/variables.tf b/aws/cluster-local/variables.tf deleted file mode 100644 index 8bf95161..00000000 --- a/aws/cluster-local/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "configuration" { - type = map(map(string)) - description = "Map with per workspace cluster configuration." -} - -variable "configuration_base_key" { - type = string - description = "The key in the configuration map all other keys inherit from." - default = "apps" -} diff --git a/aws/cluster-local/versions.tf b/aws/cluster-local/versions.tf deleted file mode 100644 index 8771a99b..00000000 --- a/aws/cluster-local/versions.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - } - - required_version = ">= 0.13" -} diff --git a/azurerm/cluster-local/configuration.tf b/azurerm/cluster-local/configuration.tf deleted file mode 100644 index 5aafcf57..00000000 --- a/azurerm/cluster-local/configuration.tf +++ /dev/null @@ -1,41 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - # on Azure the region is determined by resource group - # in the local implementation we don't have access to that - # to still support multi-region setups locally, we hash the resource group name - # and use that as the region part of the cluster name prefixed with aks- - resource_group = local.cfg["resource_group"] - fake_region_hash = substr(sha256(local.resource_group), 0, 7) - fake_region = "aks-${local.fake_region_hash}" - - http_port_default = terraform.workspace == "apps" ? 80 : 8080 - http_port = lookup(local.cfg, "http_port", local.http_port_default) - - https_port_default = terraform.workspace == "apps" ? 443 : 8443 - https_port = lookup(local.cfg, "https_port", local.https_port_default) - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - node_image = lookup(local.cfg, "node_image", null) - - node_count = lookup(local.cfg, "default_node_pool_min_count", "1") - nodes = [ - for node, _ in range(local.node_count) : - "worker" - ] - extra_nodes = join(",", local.nodes) - -} diff --git a/azurerm/cluster-local/main.tf b/azurerm/cluster-local/main.tf deleted file mode 100644 index 2f78176c..00000000 --- a/azurerm/cluster-local/main.tf +++ /dev/null @@ -1,29 +0,0 @@ -module "cluster_metadata" { - source = "../../common/metadata" - - name_prefix = local.name_prefix - base_domain = local.base_domain - - provider_name = "azure" - provider_region = local.fake_region - - # Azure does not allow / character in labels - label_namespace = "kubestack.com-" -} - -module "cluster" { - source = "../../kind/_modules/kind" - - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_tags = module.cluster_metadata.tags - metadata_labels = module.cluster_metadata.labels - - node_image = local.node_image - extra_nodes = local.extra_nodes - - http_port = local.http_port - https_port = local.https_port - - disable_default_ingress = local.disable_default_ingress -} diff --git a/azurerm/cluster-local/outputs.tf b/azurerm/cluster-local/outputs.tf deleted file mode 100644 index 4b0efb9c..00000000 --- a/azurerm/cluster-local/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "current_config" { - value = module.configuration.merged[terraform.workspace] -} - -output "current_metadata" { - value = module.cluster_metadata -} - -output "kubeconfig" { - sensitive = true - value = module.cluster.kubeconfig -} - -output "default_ingress_ip" { - # the cluster module returns an IP as a string - # we YAML encode null for cluster-local to provide - # a unified output to consumers - value = yamlencode(null) -} diff --git a/azurerm/cluster-local/variables.tf b/azurerm/cluster-local/variables.tf deleted file mode 100644 index 8bf95161..00000000 --- a/azurerm/cluster-local/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "configuration" { - type = map(map(string)) - description = "Map with per workspace cluster configuration." -} - -variable "configuration_base_key" { - type = string - description = "The key in the configuration map all other keys inherit from." - default = "apps" -} diff --git a/azurerm/cluster-local/versions.tf b/azurerm/cluster-local/versions.tf deleted file mode 100644 index 2606a5ac..00000000 --- a/azurerm/cluster-local/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.13" -} diff --git a/google/cluster-local/configuration.tf b/google/cluster-local/configuration.tf deleted file mode 100644 index c469498d..00000000 --- a/google/cluster-local/configuration.tf +++ /dev/null @@ -1,42 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - # while we have the real region for GKE - # we still hash and prefix it with gke- - # to align with the local implementations - # for AKS end EKS - fake_region_hash = substr(sha256(local.cfg["region"]), 0, 7) - fake_region = "gke-${local.fake_region_hash}" - - http_port_default = terraform.workspace == "apps" ? 80 : 8080 - http_port = lookup(local.cfg, "http_port", local.http_port_default) - - https_port_default = terraform.workspace == "apps" ? 443 : 8443 - https_port = lookup(local.cfg, "https_port", local.https_port_default) - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - node_image = lookup(local.cfg, "node_image", null) - - # technically it should be min_node_count times number of AZs - # but it seems better to keep node count low in the dev env - node_count = lookup(local.cfg, "cluster_min_node_count", 1) - nodes = [ - for node, _ in range(local.node_count) : - "worker" - ] - extra_nodes = join(",", local.nodes) - -} diff --git a/google/cluster-local/main.tf b/google/cluster-local/main.tf deleted file mode 100644 index 1dfa3cf0..00000000 --- a/google/cluster-local/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -module "cluster_metadata" { - source = "../../common/metadata" - - name_prefix = local.name_prefix - base_domain = local.base_domain - - provider_name = "gcp" - provider_region = local.fake_region -} - -module "cluster" { - source = "../../kind/_modules/kind" - - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_tags = module.cluster_metadata.tags - metadata_labels = module.cluster_metadata.labels - - node_image = local.node_image - extra_nodes = local.extra_nodes - - http_port = local.http_port - https_port = local.https_port - - disable_default_ingress = local.disable_default_ingress -} diff --git a/google/cluster-local/outputs.tf b/google/cluster-local/outputs.tf deleted file mode 100644 index cba3bd30..00000000 --- a/google/cluster-local/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -output "current_config" { - value = module.configuration.merged[terraform.workspace] -} - -output "current_metadata" { - value = module.cluster_metadata -} - -output "kubeconfig" { - value = module.cluster.kubeconfig -} - -output "default_ingress_ip" { - # the cluster module returns an IP as a string - # we YAML encode null for cluster-local to provide - # a unified output to consumers - value = yamlencode(null) -} diff --git a/google/cluster-local/variables.tf b/google/cluster-local/variables.tf deleted file mode 100644 index 8bf95161..00000000 --- a/google/cluster-local/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "configuration" { - type = map(map(string)) - description = "Map with per workspace cluster configuration." -} - -variable "configuration_base_key" { - type = string - description = "The key in the configuration map all other keys inherit from." - default = "apps" -} diff --git a/google/cluster-local/versions.tf b/google/cluster-local/versions.tf deleted file mode 100644 index 2606a5ac..00000000 --- a/google/cluster-local/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.13" -} diff --git a/kind/_modules/kind/main.tf b/kind/_modules/kind/main.tf deleted file mode 100644 index 97067805..00000000 --- a/kind/_modules/kind/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -locals { - kubeadm_config_patches = [ - < Date: Tue, 24 Feb 2026 17:53:29 +0100 Subject: [PATCH 2/2] Migrate cluster modules from map(map(string)) to map(object) * refactor code to not rely on _modules anymore removing one level of module nesting for easier maintainability * provide move blocks for all affected modules and resources * introduce mandatory cluster output on cluster modules and corresponding input on node pool modulesto reduce dependence on data sources causing issues when using the node pool module for the clsuter provisioned default node pool * unify node pool module inputs between providers using new cluster module output * reduce use of locals to improve DX for contributions and maintenance work * added support for merging of nested maps and objects to inheritance module and updated previous string vars to more specific types --- .github/workflows/main.yml | 4 +- aws/_modules/eks/ingress.tf | 5 - aws/_modules/eks/main.tf | 17 -- aws/_modules/eks/master.tf | 39 --- aws/_modules/eks/node_pool.tf | 37 --- aws/_modules/eks/node_pool/launch_template.tf | 63 ----- aws/_modules/eks/node_pool/main.tf | 60 ---- aws/_modules/eks/node_pool/variables.tf | 108 -------- aws/_modules/eks/node_pool/versions.tf | 4 - aws/_modules/eks/outputs.tf | 3 - aws/_modules/eks/roles_master.tf | 30 -- aws/_modules/eks/variables.tf | 196 ------------- aws/_modules/eks/versions.tf | 24 -- aws/_modules/eks/vpc.tf | 103 ------- .../eks => cluster}/aws_iam_authenticator.tf | 6 +- aws/cluster/configuration.tf | 91 ------ aws/cluster/default_node_pool.tf | 35 +++ aws/cluster/elb-dns/ingress.tf | 8 +- aws/cluster/elb-dns/variables.tf | 13 +- aws/cluster/ingress.tf | 5 + aws/{_modules/eks => cluster}/kubeconfig.tf | 0 aws/cluster/main.tf | 117 ++++---- aws/cluster/moved.tf | 134 +++++++++ aws/cluster/node-pool/configuration.tf | 40 --- aws/cluster/node-pool/data_sources.tf | 43 +-- aws/cluster/node-pool/launch_template.tf | 62 +++++ aws/cluster/node-pool/main.tf | 88 ++++-- aws/cluster/node-pool/moved.tf | 9 + aws/cluster/node-pool/outputs.tf | 3 + aws/cluster/node-pool/variables.tf | 41 ++- aws/cluster/node-pool/versions.tf | 4 + aws/cluster/node-pool/vpc.tf | 38 +-- .../eks => cluster}/openid_connect.tf | 4 +- aws/cluster/outputs.tf | 8 +- .../eks/roles_worker.tf => cluster/roles.tf} | 37 ++- aws/{_modules/eks => cluster}/sg_masters.tf | 2 +- .../eks => cluster}/templates/kubeconfig.tpl | 0 aws/cluster/variables.tf | 60 +++- aws/cluster/versions.tf | 4 + aws/cluster/vpc.tf | 103 +++++++ azurerm/_modules/aks/main.tf | 94 ------- azurerm/_modules/aks/node_pool/main.tf | 38 --- azurerm/_modules/aks/node_pool/variables.tf | 84 ------ azurerm/_modules/aks/node_pool/versions.tf | 3 - azurerm/_modules/aks/outputs.tf | 12 - azurerm/_modules/aks/service_principal.tf | 17 -- azurerm/_modules/aks/variables.tf | 217 --------------- azurerm/_modules/aks/versions.tf | 18 -- azurerm/_modules/aks/vnet.tf | 20 -- azurerm/cluster/configuration.tf | 76 ------ azurerm/{_modules/aks => cluster}/ingress.tf | 22 +- .../{_modules/aks => cluster}/kubeconfig.tf | 0 .../aks => cluster}/log_analytics.tf | 9 +- azurerm/cluster/main.tf | 135 +++++---- azurerm/cluster/moved.tf | 59 ++++ azurerm/cluster/node-pool/configuration.tf | 43 --- azurerm/cluster/node-pool/main.tf | 83 ++++-- azurerm/cluster/node-pool/moved.tf | 4 + azurerm/cluster/node-pool/outputs.tf | 3 + azurerm/cluster/node-pool/variables.tf | 32 ++- azurerm/cluster/node-pool/versions.tf | 2 +- azurerm/cluster/outputs.tf | 12 +- azurerm/{_modules/aks => cluster}/provider.tf | 0 azurerm/cluster/service_principal.tf | 17 ++ .../aks => cluster}/templates/kubeconfig.tpl | 0 azurerm/cluster/variables.tf | 63 ++++- azurerm/cluster/versions.tf | 11 +- azurerm/cluster/vnet.tf | 19 ++ common/configuration/outputs.tf | 32 ++- .../tests/non_string_values/test_hash.tf | 18 +- .../tests/non_string_values/test_object.tf | 29 +- .../tests/non_string_values/wrapper/main.tf | 11 + common/metadata/variables.tf | 1 - google/_modules/gke/cluster.tf | 131 --------- google/_modules/gke/network.tf | 78 ------ google/_modules/gke/node_pool.tf | 38 --- google/_modules/gke/node_pool/main.tf | 92 ------- google/_modules/gke/node_pool/variables.tf | 170 ------------ google/_modules/gke/node_pool/versions.tf | 4 - google/_modules/gke/outputs.tf | 11 - google/_modules/gke/variables.tf | 258 ------------------ google/_modules/gke/versions.tf | 16 -- .../gke => cluster}/cluster_role_binding.tf | 0 google/cluster/configuration.tf | 104 ------- google/{_modules/gke => cluster}/ingress.tf | 22 +- .../{_modules/gke => cluster}/kubeconfig.tf | 0 google/cluster/main.tf | 176 ++++++++---- google/cluster/moved.tf | 69 +++++ google/cluster/network.tf | 78 ++++++ google/cluster/node-pool/configuration.tf | 49 ---- .../node_pool => cluster/node-pool}/locals.tf | 3 +- google/cluster/node-pool/main.tf | 118 ++++++-- google/cluster/node-pool/moved.tf | 19 ++ .../node-pool}/outputs.tf | 4 + .../node-pool}/service_account.tf | 14 +- google/cluster/node-pool/variables.tf | 17 +- google/cluster/node-pool/versions.tf | 2 +- google/cluster/node_pool.tf | 45 +++ google/cluster/outputs.tf | 14 +- google/{_modules/gke => cluster}/provider.tf | 0 .../gke => cluster}/service_account.tf | 9 +- .../gke => cluster}/templates/kubeconfig.tpl | 0 google/cluster/variables.tf | 100 ++++++- .../configurations/aks/aks_zero_cluster.tf | 6 +- .../configurations/eks/eks_zero_cluster.tf | 16 +- .../configurations/gke/gke_zero_cluster.tf | 9 +- .../multi-cloud/eks_zero_cluster.tf | 20 +- .../multi-cloud/gke_zero_cluster.tf | 13 +- tests/aks_zero_cluster.tf | 8 +- tests/aks_zero_node_pools.tf | 3 +- tests/eks_zero_cluster.tf | 22 +- tests/eks_zero_node_pools.tf | 40 +-- tests/gke_zero_cluster.tf | 10 +- tests/gke_zero_node_pools.tf | 1 + 114 files changed, 1761 insertions(+), 2860 deletions(-) delete mode 100644 aws/_modules/eks/ingress.tf delete mode 100644 aws/_modules/eks/main.tf delete mode 100644 aws/_modules/eks/master.tf delete mode 100644 aws/_modules/eks/node_pool.tf delete mode 100644 aws/_modules/eks/node_pool/launch_template.tf delete mode 100644 aws/_modules/eks/node_pool/main.tf delete mode 100644 aws/_modules/eks/node_pool/variables.tf delete mode 100644 aws/_modules/eks/node_pool/versions.tf delete mode 100644 aws/_modules/eks/outputs.tf delete mode 100644 aws/_modules/eks/roles_master.tf delete mode 100644 aws/_modules/eks/variables.tf delete mode 100644 aws/_modules/eks/versions.tf delete mode 100644 aws/_modules/eks/vpc.tf rename aws/{_modules/eks => cluster}/aws_iam_authenticator.tf (61%) delete mode 100644 aws/cluster/configuration.tf create mode 100644 aws/cluster/default_node_pool.tf create mode 100644 aws/cluster/ingress.tf rename aws/{_modules/eks => cluster}/kubeconfig.tf (100%) create mode 100644 aws/cluster/moved.tf delete mode 100644 aws/cluster/node-pool/configuration.tf create mode 100644 aws/cluster/node-pool/launch_template.tf create mode 100644 aws/cluster/node-pool/moved.tf create mode 100644 aws/cluster/node-pool/outputs.tf rename aws/{_modules/eks => cluster}/openid_connect.tf (66%) rename aws/{_modules/eks/roles_worker.tf => cluster/roles.tf} (50%) rename aws/{_modules/eks => cluster}/sg_masters.tf (93%) rename aws/{_modules/eks => cluster}/templates/kubeconfig.tpl (100%) create mode 100644 aws/cluster/vpc.tf delete mode 100644 azurerm/_modules/aks/main.tf delete mode 100644 azurerm/_modules/aks/node_pool/main.tf delete mode 100644 azurerm/_modules/aks/node_pool/variables.tf delete mode 100644 azurerm/_modules/aks/node_pool/versions.tf delete mode 100644 azurerm/_modules/aks/outputs.tf delete mode 100644 azurerm/_modules/aks/service_principal.tf delete mode 100644 azurerm/_modules/aks/variables.tf delete mode 100644 azurerm/_modules/aks/versions.tf delete mode 100644 azurerm/_modules/aks/vnet.tf delete mode 100644 azurerm/cluster/configuration.tf rename azurerm/{_modules/aks => cluster}/ingress.tf (53%) rename azurerm/{_modules/aks => cluster}/kubeconfig.tf (100%) rename azurerm/{_modules/aks => cluster}/log_analytics.tf (67%) create mode 100644 azurerm/cluster/moved.tf delete mode 100644 azurerm/cluster/node-pool/configuration.tf create mode 100644 azurerm/cluster/node-pool/moved.tf create mode 100644 azurerm/cluster/node-pool/outputs.tf rename azurerm/{_modules/aks => cluster}/provider.tf (100%) create mode 100644 azurerm/cluster/service_principal.tf rename azurerm/{_modules/aks => cluster}/templates/kubeconfig.tpl (100%) create mode 100644 azurerm/cluster/vnet.tf delete mode 100644 google/_modules/gke/cluster.tf delete mode 100644 google/_modules/gke/network.tf delete mode 100644 google/_modules/gke/node_pool.tf delete mode 100644 google/_modules/gke/node_pool/main.tf delete mode 100644 google/_modules/gke/node_pool/variables.tf delete mode 100644 google/_modules/gke/node_pool/versions.tf delete mode 100644 google/_modules/gke/outputs.tf delete mode 100644 google/_modules/gke/variables.tf delete mode 100644 google/_modules/gke/versions.tf rename google/{_modules/gke => cluster}/cluster_role_binding.tf (100%) delete mode 100644 google/cluster/configuration.tf rename google/{_modules/gke => cluster}/ingress.tf (52%) rename google/{_modules/gke => cluster}/kubeconfig.tf (100%) create mode 100644 google/cluster/moved.tf create mode 100644 google/cluster/network.tf delete mode 100644 google/cluster/node-pool/configuration.tf rename google/{_modules/gke/node_pool => cluster/node-pool}/locals.tf (76%) create mode 100644 google/cluster/node-pool/moved.tf rename google/{_modules/gke/node_pool => cluster/node-pool}/outputs.tf (69%) rename google/{_modules/gke/node_pool => cluster/node-pool}/service_account.tf (61%) create mode 100644 google/cluster/node_pool.tf rename google/{_modules/gke => cluster}/provider.tf (100%) rename google/{_modules/gke => cluster}/service_account.tf (70%) rename google/{_modules/gke => cluster}/templates/kubeconfig.tpl (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 13215e34..2053546a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,7 +173,7 @@ jobs: sed -i 's/region = ""/region = "eu-west-1"/g' eks_zero_providers.tf || true # EKS: set cluster_availability_zones - sed -i 's/cluster_availability_zones = ""/cluster_availability_zones = "eu-west-1a,eu-west-1b"/g' eks_zero_cluster.tf || true + sed -i 's/cluster_availability_zones = \[\]/cluster_availability_zones = ["eu-west-1a", "eu-west-1b"]/g' eks_zero_cluster.tf || true # GKE: set project_id sed -i 's/project_id = ""/project_id = "terraform-kubestack-testing"/g' gke_zero_cluster.tf || true @@ -182,7 +182,7 @@ jobs: sed -i 's/region = ""/region = "europe-west1"/g' gke_zero_cluster.tf || true # GKE: set cluster_node_locations - sed -i 's/cluster_node_locations = ""/cluster_node_locations = "europe-west1-b,europe-west1-c,europe-west1-d"/g' gke_zero_cluster.tf || true + sed -i 's/cluster_node_locations = \[\]/cluster_node_locations = ["europe-west1-b", "europe-west1-c", "europe-west1-d"]/g' gke_zero_cluster.tf || true - name: "Terraform init" working-directory: ./kubestack-starter-${{ matrix.starter }} diff --git a/aws/_modules/eks/ingress.tf b/aws/_modules/eks/ingress.tf deleted file mode 100644 index 917a9aa5..00000000 --- a/aws/_modules/eks/ingress.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "aws_route53_zone" "current" { - count = var.disable_default_ingress ? 0 : 1 - - name = "${var.metadata_fqdn}." -} diff --git a/aws/_modules/eks/main.tf b/aws/_modules/eks/main.tf deleted file mode 100644 index a2f87d9f..00000000 --- a/aws/_modules/eks/main.tf +++ /dev/null @@ -1,17 +0,0 @@ -# AWS refers to labels (key-value pairs) as tags. -# EKS also requires below specifc tag set. This is why we merge this one -# and our default labels from metadata here into eks_tags to use throughout -# the eks module. -locals { - eks_tags = { - "kubernetes.io/cluster/${var.metadata_name}" = "shared" - } - - eks_metadata_tags = merge(var.metadata_labels, local.eks_tags) -} - -data "aws_partition" "current" {} - -data "aws_eks_cluster_auth" "current" { - name = aws_eks_cluster.current.name -} diff --git a/aws/_modules/eks/master.tf b/aws/_modules/eks/master.tf deleted file mode 100644 index ed8f6b0d..00000000 --- a/aws/_modules/eks/master.tf +++ /dev/null @@ -1,39 +0,0 @@ -resource "aws_eks_cluster" "current" { - name = var.metadata_name - role_arn = aws_iam_role.master.arn - - vpc_config { - security_group_ids = [aws_security_group.masters.id] - subnet_ids = aws_subnet.current.*.id - endpoint_private_access = var.cluster_endpoint_private_access - endpoint_public_access = var.cluster_endpoint_public_access - public_access_cidrs = var.cluster_public_access_cidrs - } - - dynamic "kubernetes_network_config" { - for_each = var.cluster_service_cidr != null ? toset([1]) : toset([]) - content { - service_ipv4_cidr = var.cluster_service_cidr - } - } - - dynamic "encryption_config" { - for_each = var.cluster_encryption_key_arn != null ? toset([1]) : toset([]) - content { - resources = ["secrets"] - - provider { - key_arn = var.cluster_encryption_key_arn - } - } - } - - depends_on = [ - aws_iam_role_policy_attachment.master_cluster_policy, - aws_iam_role_policy_attachment.master_service_policy, - ] - - version = var.cluster_version - - enabled_cluster_log_types = var.enabled_cluster_log_types -} diff --git a/aws/_modules/eks/node_pool.tf b/aws/_modules/eks/node_pool.tf deleted file mode 100644 index 9d22f1ed..00000000 --- a/aws/_modules/eks/node_pool.tf +++ /dev/null @@ -1,37 +0,0 @@ -module "node_pool" { - source = "./node_pool" - - metadata_labels = var.metadata_labels - eks_metadata_tags = local.eks_metadata_tags - - cluster_name = aws_eks_cluster.current.name - node_group_name = "default" - - role_arn = aws_iam_role.node.arn - - subnet_ids = var.vpc_legacy_node_subnets ? aws_subnet.current.*.id : aws_subnet.node_pool.*.id - - instance_types = var.instance_types - desired_size = var.desired_capacity - max_size = var.max_size - min_size = var.min_size - - disk_size = var.root_device_volume_size - - metadata_options = var.metadata_options - - taints = var.taints - - ami_type = var.worker_ami_type - ami_release_version = var.worker_ami_release_version - - tags = var.additional_node_tags - - kubernetes_version = aws_eks_cluster.current.version - - # force node_pool to depend on aws-auth configmap - depends-on-aws-auth = { - name = kubernetes_config_map.current.metadata[0].name - namespace = kubernetes_config_map.current.metadata[0].namespace - } -} diff --git a/aws/_modules/eks/node_pool/launch_template.tf b/aws/_modules/eks/node_pool/launch_template.tf deleted file mode 100644 index 0fbd5ffd..00000000 --- a/aws/_modules/eks/node_pool/launch_template.tf +++ /dev/null @@ -1,63 +0,0 @@ -locals { - create_launch_template = var.metadata_options != null ? true : false - - launch_template_name_parts = join("-", [var.cluster_name, var.node_group_name]) - launch_template_name_hash = sha256(local.launch_template_name_parts) - launch_template_name = "${substr(local.launch_template_name_parts, 0, 120)}-${substr(local.launch_template_name_hash, 0, 7)}" - - cpu_ami_name = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "amazon-linux-2023/arm64/standard" : "amazon-linux-2023/x86_64/standard" - is_gpu = length(data.aws_ec2_instance_type.current.gpus) > 0 - ami_name = local.is_gpu ? "amazon-linux-2023/x86_64/nvidia" : local.cpu_ami_name - ami_release_prefix = local.is_gpu ? "amazon-eks-gpu-node" : "amazon-eks-node" - ami_release_date = var.ami_release_version == null ? "" : split("-", var.ami_release_version)[1] - ami_release_name = var.ami_release_version == null ? "recommended" : "${local.ami_release_prefix}-${var.kubernetes_version}-v${local.ami_release_date}" -} - -data "aws_ssm_parameter" "eks_ami_release_version" { - count = var.disk_size != null ? 1 : 0 - - name = "/aws/service/eks/optimized-ami/${var.kubernetes_version}/${local.ami_name}/${local.ami_release_name}/image_id" -} - -data "aws_ami" "eks_optimized" { - count = var.disk_size != null ? 1 : 0 - - owners = ["amazon"] - - filter { - name = "image-id" - values = [nonsensitive(data.aws_ssm_parameter.eks_ami_release_version[0].value)] - } -} - -resource "aws_launch_template" "current" { - count = local.create_launch_template ? 1 : 0 - - name = local.launch_template_name - - tags = merge(var.tags, var.eks_metadata_tags) - - dynamic "block_device_mappings" { - for_each = var.disk_size != null ? toset([1]) : toset([]) - - content { - device_name = data.aws_ami.eks_optimized[0].root_device_name - - ebs { - volume_size = var.disk_size - } - } - } - - dynamic "metadata_options" { - for_each = var.metadata_options != null ? toset([1]) : toset([]) - - content { - http_endpoint = var.metadata_options.http_endpoint - http_tokens = var.metadata_options.http_tokens - http_put_response_hop_limit = var.metadata_options.http_put_response_hop_limit - http_protocol_ipv6 = var.metadata_options.http_protocol_ipv6 - instance_metadata_tags = var.metadata_options.instance_metadata_tags - } - } -} diff --git a/aws/_modules/eks/node_pool/main.tf b/aws/_modules/eks/node_pool/main.tf deleted file mode 100644 index 4b534c4a..00000000 --- a/aws/_modules/eks/node_pool/main.tf +++ /dev/null @@ -1,60 +0,0 @@ -data "aws_ec2_instance_type" "current" { - # ami_type is always determined by the first instance_type in the list - instance_type = element(tolist(var.instance_types), 0) -} - -locals { - cpu_ami_type = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "AL2023_ARM_64_STANDARD" : "AL2023_x86_64_STANDARD" - ami_type = length(data.aws_ec2_instance_type.current.gpus) > 0 ? "AL2023_x86_64_NVIDIA" : local.cpu_ami_type -} - -resource "aws_eks_node_group" "nodes" { - cluster_name = var.cluster_name - node_group_name = var.node_group_name - node_role_arn = var.role_arn - subnet_ids = var.subnet_ids - - scaling_config { - desired_size = var.desired_size - max_size = var.max_size - min_size = var.min_size - } - - version = var.kubernetes_version - ami_type = var.ami_type == null ? local.ami_type : var.ami_type - release_version = var.ami_release_version - instance_types = var.instance_types - - disk_size = local.create_launch_template ? null : var.disk_size - - dynamic "launch_template" { - for_each = local.create_launch_template ? toset([1]) : toset([]) - - content { - id = aws_launch_template.current[0].id - version = aws_launch_template.current[0].latest_version - } - } - - tags = merge(var.tags, var.eks_metadata_tags) - labels = merge(var.labels, var.metadata_labels) - - dynamic "taint" { - for_each = var.taints - - content { - key = taint.value["key"] - value = taint.value["value"] - effect = taint.value["effect"] - } - } - - depends_on = [var.depends-on-aws-auth] - - # when autoscaler is enabled, desired_size needs to be ignored - # better would be to handle this in the resource and not require - # desired_size, min_size and max_size in scaling_config - lifecycle { - ignore_changes = [scaling_config[0].desired_size] - } -} diff --git a/aws/_modules/eks/node_pool/variables.tf b/aws/_modules/eks/node_pool/variables.tf deleted file mode 100644 index 265d6c01..00000000 --- a/aws/_modules/eks/node_pool/variables.tf +++ /dev/null @@ -1,108 +0,0 @@ -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "eks_metadata_tags" { - type = map(any) - description = "EKS metadata tags to use." -} - -variable "cluster_name" { - type = string - description = "Cluster name of the EKS cluster." -} - -variable "node_group_name" { - type = string - description = "Name for this node pool." -} - -variable "role_arn" { - type = string - description = "ARN of the IAM role for worker nodes." -} - -variable "instance_types" { - type = set(string) - description = "Set of AWS instance types to use for nodes." -} - -variable "desired_size" { - type = string - description = "Desired number of worker nodes." -} - -variable "max_size" { - type = string - description = "Maximum number of worker nodes." -} - -variable "min_size" { - type = string - description = "Minimum number of worker nodes." -} - -variable "disk_size" { - type = string - default = "20" - description = "Will set the volume size of the root device" -} - -variable "subnet_ids" { - type = list(string) - description = "List of VPC subnet IDs to use for nodes." -} - -variable "depends-on-aws-auth" { - type = map(string) - description = "Used as a depends_on shim to first create the aws-auth configmap before creating the node_pool." -} - -variable "taints" { - type = set(object({ - key = string - value = string - effect = string - })) - description = "Kubernetes taints to set for node pool." -} - -variable "tags" { - type = map(string) - description = "AWS tags to set on the node pool. Merged with Kubestack default tags." - default = {} -} - -variable "labels" { - type = map(string) - description = "Kubernetes labels to set on the nodes created by the node pool. Merged with Kubestack default labels." - default = {} -} - -variable "ami_type" { - type = string - description = "AMI type to use for nodes of the node pool." -} - -variable "ami_release_version" { - type = string - description = "AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version" - default = null -} - -variable "kubernetes_version" { - type = string - description = "Kubernetes version to use for node pool." -} - -variable "metadata_options" { - description = "EC2 metadata service options." - type = object({ - http_endpoint = optional(string) - http_tokens = optional(string) - http_put_response_hop_limit = optional(number) - http_protocol_ipv6 = optional(string) - instance_metadata_tags = optional(string) - }) -} diff --git a/aws/_modules/eks/node_pool/versions.tf b/aws/_modules/eks/node_pool/versions.tf deleted file mode 100644 index ac97c6ac..00000000 --- a/aws/_modules/eks/node_pool/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/aws/_modules/eks/outputs.tf b/aws/_modules/eks/outputs.tf deleted file mode 100644 index 55b14881..00000000 --- a/aws/_modules/eks/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "kubeconfig" { - value = local.kubeconfig -} diff --git a/aws/_modules/eks/roles_master.tf b/aws/_modules/eks/roles_master.tf deleted file mode 100644 index dd76f32f..00000000 --- a/aws/_modules/eks/roles_master.tf +++ /dev/null @@ -1,30 +0,0 @@ -resource "aws_iam_role" "master" { - name = "${var.metadata_name}-master" - - assume_role_policy = < 0 ? 1 : 0 + count = var.cluster_default_node_pool_subnet_ids == null ? 1 : 0 filter { name = "vpc-id" - values = [data.aws_vpc.current.id] + values = [var.cluster.vpc_config[0].vpc_id] } # if the node pool is in one or more specific AZs # only link subnet_ids belonging to these AZs filter { name = "availability-zone" - values = local.availability_zones + values = coalesce(local.cfg.availability_zones, []) } # exclude control plane subnets filter { name = "subnet-id" - values = tolist(data.aws_eks_node_group.default.subnet_ids) + values = tolist(data.aws_eks_node_group.default[0].subnet_ids) } } + data "aws_internet_gateway" "current" { - count = local.vpc_subnet_newbits == null ? 0 : local.vpc_subnet_map_public_ip == false ? 0 : 1 + count = local.cfg.vpc_subnet_newbits == null ? 0 : local.cfg.vpc_subnet_map_public_ip == false ? 0 : 1 filter { name = "attachment.vpc-id" - values = [data.aws_vpc.current.id] + values = [var.cluster.vpc_config[0].vpc_id] } } data "aws_nat_gateway" "current" { - count = local.vpc_subnet_newbits == null ? 0 : local.vpc_subnet_map_public_ip == false ? length(local.availability_zones) : 0 + count = local.cfg.vpc_subnet_newbits == null ? 0 : local.cfg.vpc_subnet_map_public_ip == false ? length(lookup(local.cfg, "availability_zones", [])) : 0 - vpc_id = data.aws_vpc.current.id + vpc_id = var.cluster.vpc_config[0].vpc_id tags = { - "kubestack.com/cluster_name" = data.aws_eks_cluster.current.name - "kubestack.com/cluster_provider_zone" = local.availability_zones[count.index] + "kubestack.com/cluster_name" = var.cluster.name + "kubestack.com/cluster_provider_zone" = local.cfg.availability_zones[count.index] } } + +data "aws_ec2_instance_type" "current" { + # ami_type is always determined by the first instance_type in the list + # fall back to t3.medium when no instance_types are configured + instance_type = element(coalesce(local.cfg.instance_types, ["t3.medium"]), 0) +} diff --git a/aws/cluster/node-pool/launch_template.tf b/aws/cluster/node-pool/launch_template.tf new file mode 100644 index 00000000..db5a188f --- /dev/null +++ b/aws/cluster/node-pool/launch_template.tf @@ -0,0 +1,62 @@ +locals { + create_launch_template = local.cfg.metadata_options != null + launch_template_name_parts = join("-", [var.cluster.name, local.cfg.name]) + launch_template_name_hash = sha256(local.launch_template_name_parts) + launch_template_name = "${substr(local.launch_template_name_parts, 0, 120)}-${substr(local.launch_template_name_hash, 0, 7)}" + + cpu_ami_name = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "amazon-linux-2023/arm64/standard" : "amazon-linux-2023/x86_64/standard" + is_gpu = length(data.aws_ec2_instance_type.current.gpus) > 0 + ami_name = local.is_gpu ? "amazon-linux-2023/x86_64/nvidia" : local.cpu_ami_name + ami_release_prefix = local.is_gpu ? "amazon-eks-gpu-node" : "amazon-eks-node" + ami_release_date = local.cfg.ami_release_version == null ? "" : split("-", local.cfg.ami_release_version)[1] + ami_release_name = local.cfg.ami_release_version == null ? "recommended" : "${local.ami_release_prefix}-${var.cluster.version}-v${local.ami_release_date}" +} + +data "aws_ssm_parameter" "eks_ami_release_version" { + count = try(coalesce(local.cfg.disk_size, null), 20) != null ? 1 : 0 + + name = "/aws/service/eks/optimized-ami/${var.cluster.version}/${local.ami_name}/${local.ami_release_name}/image_id" +} + +data "aws_ami" "eks_optimized" { + count = try(coalesce(local.cfg.disk_size, null), 20) != null ? 1 : 0 + + owners = ["amazon"] + + filter { + name = "image-id" + values = [nonsensitive(data.aws_ssm_parameter.eks_ami_release_version[0].value)] + } +} + +resource "aws_launch_template" "current" { + count = local.create_launch_template ? 1 : 0 + + name = local.launch_template_name + + tags = coalesce(local.cfg.tags, {}) + + dynamic "block_device_mappings" { + for_each = try(coalesce(local.cfg.disk_size, null), 20) != null ? toset([1]) : toset([]) + + content { + device_name = data.aws_ami.eks_optimized[0].root_device_name + + ebs { + volume_size = try(coalesce(local.cfg.disk_size, null), 20) + } + } + } + + dynamic "metadata_options" { + for_each = local.cfg.metadata_options != null ? toset([1]) : toset([]) + + content { + http_endpoint = local.cfg.metadata_options.http_endpoint + http_tokens = local.cfg.metadata_options.http_tokens + http_put_response_hop_limit = local.cfg.metadata_options.http_put_response_hop_limit + http_protocol_ipv6 = local.cfg.metadata_options.http_protocol_ipv6 + instance_metadata_tags = local.cfg.metadata_options.instance_metadata_tags + } + } +} diff --git a/aws/cluster/node-pool/main.tf b/aws/cluster/node-pool/main.tf index 077fa1b3..55d9d333 100644 --- a/aws/cluster/node-pool/main.tf +++ b/aws/cluster/node-pool/main.tf @@ -1,44 +1,72 @@ -locals { - // if provider level tags are set, the node_group data source tags attr - // includes the resource level and provider level tags - // we have to exclude the provider level tags when setting them for node pools below - node_group_tag_keys = toset(keys(data.aws_eks_node_group.default.tags)) - provider_level_tag_keys = toset(keys(data.aws_default_tags.current.tags)) - tags_without_all_tags_keys = setsubtract(local.node_group_tag_keys, local.provider_level_tag_keys) - tags_without_all_tags = { for k in local.tags_without_all_tags_keys : k => data.aws_eks_node_group.default.tags[k] } +module "configuration" { + source = "../../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key } -module "node_pool" { - source = "../../_modules/eks/node_pool" - - cluster_name = data.aws_eks_cluster.current.name - metadata_labels = data.aws_eks_node_group.default.labels - eks_metadata_tags = local.tags_without_all_tags - role_arn = data.aws_eks_node_group.default.node_role_arn +locals { + cfg = module.configuration.merged[terraform.workspace] - node_group_name = local.name + cpu_ami_type = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "AL2023_ARM_64_STANDARD" : "AL2023_x86_64_STANDARD" + computed_ami_type = length(data.aws_ec2_instance_type.current.gpus) > 0 ? "AL2023_x86_64_NVIDIA" : local.cpu_ami_type +} - subnet_ids = local.vpc_subnet_newbits == null ? local.vpc_subnet_ids : aws_subnet.current.*.id +resource "aws_eks_node_group" "nodes" { + cluster_name = var.cluster.name + node_group_name = local.cfg.name + node_role_arn = data.aws_iam_role.node.arn + subnet_ids = coalesce( + # first prio, user specified subnet ids + try(coalesce(var.cluster_default_node_pool_subnet_ids, local.cfg.vpc_subnet_ids), null), + # second prio, created subnet ids + length(aws_subnet.current.*.id) > 0 ? aws_subnet.current.*.id : null, + # fall back, existing subnet ids + length(data.aws_subnets.current) == 1 ? data.aws_subnets.current[0].ids : null, + ) - instance_types = local.instance_types - ami_release_version = local.ami_release_version - desired_size = local.desired_capacity - max_size = local.max_size - min_size = local.min_size + scaling_config { + desired_size = local.cfg.desired_capacity + max_size = local.cfg.max_size + min_size = local.cfg.min_size + } - ami_type = local.ami_type + version = var.cluster.version + ami_type = local.cfg.ami_type == null ? local.computed_ami_type : local.cfg.ami_type + release_version = local.cfg.ami_release_version + instance_types = try(toset(local.cfg.instance_types), toset([])) - kubernetes_version = data.aws_eks_cluster.current.version + disk_size = local.create_launch_template ? null : try(coalesce(local.cfg.disk_size, null), 20) - disk_size = local.disk_size + dynamic "launch_template" { + for_each = local.create_launch_template ? toset([1]) : toset([]) - metadata_options = local.metadata_options + content { + id = aws_launch_template.current[0].id + version = aws_launch_template.current[0].latest_version + } + } - taints = local.taints + tags = merge( + { "kubernetes.io/cluster/${var.cluster.name}" = "shared" }, + var.cluster_metadata.labels, + coalesce(local.cfg.tags, {}) + ) + labels = merge(var.cluster_metadata.labels, coalesce(local.cfg.labels, {})) - tags = local.tags + dynamic "taint" { + for_each = coalesce(local.cfg.taints, []) - labels = local.labels + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } - depends-on-aws-auth = null + # when autoscaler is enabled, desired_size needs to be ignored + # better would be to handle this in the resource and not require + # desired_size, min_size and max_size in scaling_config + lifecycle { + ignore_changes = [scaling_config[0].desired_size] + } } diff --git a/aws/cluster/node-pool/moved.tf b/aws/cluster/node-pool/moved.tf new file mode 100644 index 00000000..44e99b61 --- /dev/null +++ b/aws/cluster/node-pool/moved.tf @@ -0,0 +1,9 @@ +moved { + from = module.node_pool.aws_eks_node_group.nodes + to = aws_eks_node_group.nodes +} + +moved { + from = module.node_pool.aws_launch_template.current + to = aws_launch_template.current +} diff --git a/aws/cluster/node-pool/outputs.tf b/aws/cluster/node-pool/outputs.tf new file mode 100644 index 00000000..871c9f88 --- /dev/null +++ b/aws/cluster/node-pool/outputs.tf @@ -0,0 +1,3 @@ +output "current_config" { + value = local.cfg +} diff --git a/aws/cluster/node-pool/variables.tf b/aws/cluster/node-pool/variables.tf index 82b0367d..5b0fcac6 100644 --- a/aws/cluster/node-pool/variables.tf +++ b/aws/cluster/node-pool/variables.tf @@ -3,12 +3,12 @@ variable "configuration" { name = optional(string) - instance_types = optional(string) + instance_types = optional(list(string)) ami_release_version = optional(string) - desired_capacity = optional(string) - min_size = optional(string) - max_size = optional(string) - disk_size = optional(string) + desired_capacity = optional(number) + min_size = optional(number) + max_size = optional(number) + disk_size = optional(number) ami_type = optional(string) @@ -20,14 +20,14 @@ variable "configuration" { instance_metadata_tags = optional(string) })) - availability_zones = optional(string) + availability_zones = optional(list(string)) - vpc_subnet_ids = optional(string) + vpc_subnet_ids = optional(list(string)) vpc_secondary_cidr = optional(string) - vpc_subnet_newbits = optional(string) - vpc_subnet_number_offset = optional(string) + vpc_subnet_newbits = optional(number) + vpc_subnet_number_offset = optional(number) vpc_subnet_map_public_ip = optional(bool) taints = optional(set(object({ @@ -41,20 +41,37 @@ variable "configuration" { labels = optional(map(string)) })) description = "Map with per workspace module configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } -variable "cluster_name" { - type = string - description = "Name of the cluster to attach the node pool to." +variable "cluster" { + type = any + description = "The cluster output from the cluster module." + nullable = false +} + +variable "cluster_metadata" { + type = any + description = "Metadata of the cluster to attach the node pool to." + nullable = false } + variable "cluster_default_node_pool_name" { type = string description = "Name of the cluster's default node pool to inherit IAM role and subnets from." default = "default" + nullable = false +} + +variable "cluster_default_node_pool_subnet_ids" { + type = list(string) + description = "Subnet IDs for the default node pool subnet." + default = null } diff --git a/aws/cluster/node-pool/versions.tf b/aws/cluster/node-pool/versions.tf index 07ac0e26..d64d1d92 100644 --- a/aws/cluster/node-pool/versions.tf +++ b/aws/cluster/node-pool/versions.tf @@ -3,6 +3,10 @@ terraform { aws = { source = "hashicorp/aws" } + + tls = { + source = "hashicorp/tls" + } } required_version = ">= 1.3.0" diff --git a/aws/cluster/node-pool/vpc.tf b/aws/cluster/node-pool/vpc.tf index 2219684a..f758288c 100644 --- a/aws/cluster/node-pool/vpc.tf +++ b/aws/cluster/node-pool/vpc.tf @@ -1,47 +1,51 @@ resource "aws_vpc_ipv4_cidr_block_association" "current" { - count = local.vpc_secondary_cidr == null ? 0 : 1 + count = local.cfg.vpc_secondary_cidr == null ? 0 : 1 - vpc_id = data.aws_vpc.current.id - cidr_block = local.vpc_secondary_cidr + vpc_id = var.cluster.vpc_config[0].vpc_id + cidr_block = local.cfg.vpc_secondary_cidr } locals { - subnet_cidr = length(aws_vpc_ipv4_cidr_block_association.current) == 1 ? aws_vpc_ipv4_cidr_block_association.current[0].cidr_block : data.aws_vpc.current.cidr_block + subnet_cidr = length(aws_vpc_ipv4_cidr_block_association.current) == 1 ? aws_vpc_ipv4_cidr_block_association.current[0].cidr_block : data.aws_vpc.cluster.cidr_block } resource "aws_subnet" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 - availability_zone = local.availability_zones[count.index] + availability_zone = local.cfg.availability_zones[count.index] cidr_block = cidrsubnet( local.subnet_cidr, - local.vpc_subnet_newbits, - local.vpc_subnet_number_offset + count.index + local.cfg.vpc_subnet_newbits, + (local.cfg.vpc_subnet_number_offset == null ? 1 : local.cfg.vpc_subnet_number_offset) + count.index ) - vpc_id = data.aws_vpc.current.id - map_public_ip_on_launch = local.vpc_subnet_map_public_ip == null ? true : local.vpc_subnet_map_public_ip + vpc_id = var.cluster.vpc_config[0].vpc_id + map_public_ip_on_launch = local.cfg.vpc_subnet_map_public_ip == null ? true : local.cfg.vpc_subnet_map_public_ip - tags = data.aws_eks_node_group.default.tags + tags = merge( + { "kubernetes.io/cluster/${var.cluster.name}" = "shared" }, + var.cluster_metadata.labels, + coalesce(local.cfg.tags, {}) + ) } resource "aws_route_table" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 - vpc_id = data.aws_vpc.current.id + vpc_id = var.cluster.vpc_config[0].vpc_id } resource "aws_route" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 route_table_id = aws_route_table.current[count.index].id - gateway_id = local.vpc_subnet_map_public_ip == false ? null : data.aws_internet_gateway.current[0].id - nat_gateway_id = local.vpc_subnet_map_public_ip == false ? data.aws_nat_gateway.current[count.index].id : null + gateway_id = local.cfg.vpc_subnet_map_public_ip == false ? null : data.aws_internet_gateway.current[0].id + nat_gateway_id = local.cfg.vpc_subnet_map_public_ip == false ? data.aws_nat_gateway.current[count.index].id : null destination_cidr_block = "0.0.0.0/0" } resource "aws_route_table_association" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 subnet_id = aws_subnet.current[count.index].id route_table_id = aws_route_table.current[count.index].id diff --git a/aws/_modules/eks/openid_connect.tf b/aws/cluster/openid_connect.tf similarity index 66% rename from aws/_modules/eks/openid_connect.tf rename to aws/cluster/openid_connect.tf index 06745c60..1dae6189 100644 --- a/aws/_modules/eks/openid_connect.tf +++ b/aws/cluster/openid_connect.tf @@ -1,11 +1,11 @@ data "tls_certificate" "current" { - count = var.disable_openid_connect_provider == false ? 1 : 0 + count = try(coalesce(local.cfg.disable_openid_connect_provider, null), false) == false ? 1 : 0 url = aws_eks_cluster.current.identity[0].oidc[0].issuer } resource "aws_iam_openid_connect_provider" "current" { - count = var.disable_openid_connect_provider == false ? 1 : 0 + count = try(coalesce(local.cfg.disable_openid_connect_provider, null), false) == false ? 1 : 0 client_id_list = ["sts.${data.aws_partition.current.dns_suffix}"] thumbprint_list = [data.tls_certificate.current[0].certificates[0].sha1_fingerprint] diff --git a/aws/cluster/outputs.tf b/aws/cluster/outputs.tf index 424f37e9..53125dc3 100644 --- a/aws/cluster/outputs.tf +++ b/aws/cluster/outputs.tf @@ -1,5 +1,9 @@ +output "cluster" { + value = aws_eks_cluster.current +} + output "current_config" { - value = module.configuration.merged[terraform.workspace] + value = local.cfg } output "current_metadata" { @@ -7,5 +11,5 @@ output "current_metadata" { } output "kubeconfig" { - value = module.cluster.kubeconfig + value = local.kubeconfig } diff --git a/aws/_modules/eks/roles_worker.tf b/aws/cluster/roles.tf similarity index 50% rename from aws/_modules/eks/roles_worker.tf rename to aws/cluster/roles.tf index f81fd16e..26d9ff41 100644 --- a/aws/_modules/eks/roles_worker.tf +++ b/aws/cluster/roles.tf @@ -1,5 +1,37 @@ +resource "aws_iam_role" "master" { + name = "${module.cluster_metadata.name}-master" + + assume_role_policy = < 0 ? azurerm_virtual_network.current[0] : null -} - -output "kubeconfig" { - sensitive = true - value = local.kubeconfig -} - -output "default_ingress_ip" { - value = length(azurerm_public_ip.current) > 0 ? azurerm_public_ip.current[0].ip_address : null -} diff --git a/azurerm/_modules/aks/service_principal.tf b/azurerm/_modules/aks/service_principal.tf deleted file mode 100644 index 4dec3756..00000000 --- a/azurerm/_modules/aks/service_principal.tf +++ /dev/null @@ -1,17 +0,0 @@ -resource "azuread_application" "current" { - count = var.disable_managed_identities == true ? 1 : 0 - - display_name = var.metadata_name -} - -resource "azuread_service_principal" "current" { - count = var.disable_managed_identities == true ? 1 : 0 - - client_id = azuread_application.current[0].client_id -} - -resource "azuread_service_principal_password" "current" { - count = var.disable_managed_identities == true ? 1 : 0 - - service_principal_id = azuread_service_principal.current[0].id -} diff --git a/azurerm/_modules/aks/variables.tf b/azurerm/_modules/aks/variables.tf deleted file mode 100644 index 2f9246ad..00000000 --- a/azurerm/_modules/aks/variables.tf +++ /dev/null @@ -1,217 +0,0 @@ -variable "metadata_name" { - type = string - description = "Metadata name to use." -} - -variable "metadata_fqdn" { - type = string - description = "DNS name of the zone. E.g. `infra.example.com`" -} - -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "metadata_label_namespace" { - type = string - description = "Prefix labels are namespaced with." -} - -variable "resource_group" { - type = string - description = "Resource group to use for the cluster." -} - -variable "legacy_vnet_name" { - type = bool - description = "Wheter to use the legacy vnet and subnet names." - default = false -} - -variable "vnet_address_space" { - type = list(string) - description = "Address space for the virtual network." -} - -variable "subnet_address_prefixes" { - type = list(string) - description = "Address space for the cluster subnet." -} - -variable "subnet_service_endpoints" { - type = list(string) - description = "List of service endpoints to expose on the subnet." -} - -variable "network_plugin" { - type = string - description = "Network plugin to use. Set to 'azure' for CNI; 'kubenet' for Basic" -} - -variable "network_policy" { - type = string - description = "Network policy to use. Set to 'azure' for CNI; 'calico' for Basic" -} - -variable "dns_prefix" { - type = string - description = "DNS prefix of AKS cluster API server." - default = "api" -} - -variable "service_cidr" { - type = string - description = "CIDR range for kubernetes service." -} - -variable "dns_service_ip" { - type = string - description = "IP address for the kube-dns service. Must be within service_cidr." -} - -variable "pod_cidr" { - type = string - description = "CIDR range for pods." -} - -variable "max_pods" { - type = number - description = "Maximum number of pods to run per node, if using CNI for networking." -} - -variable "default_node_pool_name" { - type = string - description = "Name of default node pool." - default = "default" -} - -variable "default_node_pool_type" { - type = string - description = "Type of default node pool. Defaults to VirtualMachineScaleSets, use AvailabilitySet for backwards compatibility with older clusters." - default = "VirtualMachineScaleSets" -} - -variable "default_node_pool_enable_auto_scaling" { - type = bool - description = "Wether to enable auto scaling for the default node pool. Defaults to true." - default = true -} - -variable "default_node_pool_min_count" { - type = string - description = "Min number of worker nodes" - default = "1" -} - -variable "default_node_pool_max_count" { - type = string - description = "Max number of worker nodes" - default = "1" -} - -variable "default_node_pool_node_count" { - type = string - description = "Static number of worker nodes" - default = "1" -} - -variable "default_node_pool_vm_size" { - type = string - description = "VM size of worker nodes" - default = "Standard_D1_v2" -} - -variable "default_node_pool_os_disk_size_gb" { - type = string - description = "Disk size of worker nodes (in GB)" - default = "30" -} - -variable "default_node_pool_only_critical_addons" { - type = bool - description = "Enabling will taint default node pool with CriticalAddonsOnly=true:NoSchedule taint. Changing this forces a new resource to be created." - default = false -} - -variable "disable_default_ingress" { - type = bool - description = "Whether to disable the default ingress." -} - -variable "default_ingress_ip_zones" { - type = list(string) - description = "Explicit list of zones for the ingress public IP." -} - -variable "enable_azure_policy_agent" { - type = bool - description = "whether to deploy the Azure policy agent to the cluster" - default = false -} - -variable "disable_managed_identities" { - type = bool - description = "Keep using legacy service principal instead of new managed identities." - default = false -} - -variable "user_assigned_identity_id" { - type = string - description = "ID of the UserAssigned identity to use." - default = null -} - -variable "sku_tier" { - type = string - description = "The SKU tier to use for the cluster. Possible values are 'Free' and 'Paid'. Defaults to 'Free'." - default = "Free" -} - -variable "kubernetes_version" { - type = string - description = "Version of Kubernetes to use when creating the cluster. If not specified, latest version will be used." -} - -variable "automatic_channel_upgrade" { - type = string - description = "The upgrade channel for this Kubernetes Cluster. Possible values are 'patch', 'rapid', 'node-image' and 'stable'." -} - -variable "enable_log_analytics" { - type = bool - description = "whether to deploy the Azure Log analytics with the cluster" - default = true -} - -variable "availability_zones" { - type = list(string) - description = "The list of availability zones to create the node pool in" - default = [] -} - - -variable "keda_enabled" { - type = bool - description = "Whether KEDA Autoscaler should be enabled for the cluster." -} - -variable "vertical_pod_autoscaler_enabled" { - type = bool - description = "Whether Vertical Pod Autoscaler should be enabled for the cluster." -} - -variable "upgade_settings_drain_timeout_in_minutes" { - type = number - description = "The amount of time in minutes to wait on eviction of pods and graceful termination per node." -} - -variable "upgade_settings_max_surge" { - type = string - description = "The maximum number or percentage of nodes which will be added to the Node Pool size during an upgrade." -} - -variable "upgade_settings_node_soak_duration_in_minutes" { - type = number - description = "The amount of time in minutes to wait after draining a node and before reimaging and moving on to next node." -} diff --git a/azurerm/_modules/aks/versions.tf b/azurerm/_modules/aks/versions.tf deleted file mode 100644 index 4bc72444..00000000 --- a/azurerm/_modules/aks/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ - -terraform { - required_providers { - azurerm = { - # https://registry.terraform.io/providers/hashicorp/azurerm/latest - source = "hashicorp/azurerm" - version = ">= 3.4.0" - } - - azuread = { - # https://registry.terraform.io/providers/hashicorp/azuread/latest - source = "hashicorp/azuread" - version = ">= 1.3.0" - } - } - - required_version = ">= 0.13" -} diff --git a/azurerm/_modules/aks/vnet.tf b/azurerm/_modules/aks/vnet.tf deleted file mode 100644 index 2cf18dcb..00000000 --- a/azurerm/_modules/aks/vnet.tf +++ /dev/null @@ -1,20 +0,0 @@ - -resource "azurerm_virtual_network" "current" { - count = var.network_plugin == "azure" ? 1 : 0 - - name = var.legacy_vnet_name ? "vnet-aks-${terraform.workspace}-cluster" : var.metadata_name - address_space = var.vnet_address_space - resource_group_name = data.azurerm_resource_group.current.name - location = data.azurerm_resource_group.current.location -} - -resource "azurerm_subnet" "current" { - count = var.network_plugin == "azure" ? 1 : 0 - - name = var.legacy_vnet_name ? "aks-node-subnet" : "${var.metadata_name}-${var.default_node_pool_name}-node-pool" - address_prefixes = var.subnet_address_prefixes - resource_group_name = data.azurerm_resource_group.current.name - virtual_network_name = azurerm_virtual_network.current[0].name - - service_endpoints = length(var.subnet_service_endpoints) > 0 ? var.subnet_service_endpoints : null -} diff --git a/azurerm/cluster/configuration.tf b/azurerm/cluster/configuration.tf deleted file mode 100644 index a6cdfb75..00000000 --- a/azurerm/cluster/configuration.tf +++ /dev/null @@ -1,76 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - resource_group = local.cfg["resource_group"] - - dns_prefix = lookup(local.cfg, "dns_prefix", "api") - - sku_tier = lookup(local.cfg, "sku_tier", "Free") - - legacy_vnet_name = lookup(local.cfg, "legacy_vnet_name", false) - vnet_address_space = split(",", lookup(local.cfg, "vnet_address_space", "10.0.0.0/8")) - subnet_address_prefixes = split(",", lookup(local.cfg, "subnet_address_prefixes", "10.1.0.0/16")) - - subnet_service_endpoints_lookup = lookup(local.cfg, "subnet_service_endpoints", "") - subnet_service_endpoints = local.subnet_service_endpoints_lookup != "" ? split(",", local.subnet_service_endpoints_lookup) : [] - - network_plugin = lookup(local.cfg, "network_plugin", "kubenet") - network_policy = lookup(local.cfg, "network_policy", "calico") - service_cidr = lookup(local.cfg, "service_cidr", "10.0.0.0/16") - dns_service_ip = lookup(local.cfg, "dns_service_ip", "10.0.0.10") - pod_cidr = lookup(local.cfg, "pod_cidr", "10.244.0.0/16") - max_pods = lookup(local.cfg, "max_pods", null) - - default_node_pool_name = lookup(local.cfg, "default_node_pool_name", "default") - default_node_pool_type = lookup(local.cfg, "default_node_pool_type", "VirtualMachineScaleSets") - - default_node_pool_enable_auto_scaling = lookup(local.cfg, "default_node_pool_enable_auto_scaling", true) - default_node_pool_min_count = lookup(local.cfg, "default_node_pool_min_count", "1") - default_node_pool_max_count = lookup(local.cfg, "default_node_pool_max_count", "1") - default_node_pool_node_count = lookup(local.cfg, "default_node_pool_node_count", "1") - - default_node_pool_vm_size = lookup(local.cfg, "default_node_pool_vm_size", "Standard_B2s") - default_node_pool_only_critical_addons = lookup(local.cfg, "default_node_pool_only_critical_addons", false) - default_node_pool_os_disk_size_gb = lookup(local.cfg, "default_node_pool_os_disk_size_gb", "30") - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - default_ingress_ip_zones_lookup = lookup(local.cfg, "default_ingress_ip_zones", "") - default_ingress_ip_zones = local.default_ingress_ip_zones_lookup != "" ? split(",", local.default_ingress_ip_zones_lookup) : [] - - enable_azure_policy_agent = lookup(local.cfg, "enable_azure_policy_agent", false) - - disable_managed_identities = lookup(local.cfg, "disable_managed_identities", false) - user_assigned_identity_id = lookup(local.cfg, "user_assigned_identity_id", null) - - enable_log_analytics = lookup(local.cfg, "enable_log_analytics", true) - - kubernetes_version = lookup(local.cfg, "kubernetes_version", null) - automatic_channel_upgrade = lookup(local.cfg, "automatic_channel_upgrade", null) - - availability_zones_lookup = lookup(local.cfg, "availability_zones", "") - availability_zones = local.availability_zones_lookup != "" ? split(",", local.availability_zones_lookup) : [] - - additional_metadata_labels_lookup = lookup(local.cfg, "additional_metadata_labels", "") - additional_metadata_labels_tuples = [for t in split(",", local.additional_metadata_labels_lookup) : split("=", t)] - additional_metadata_labels = { for t in local.additional_metadata_labels_tuples : t[0] => t[1] if length(t) == 2 } - - keda_enabled = lookup(local.cfg, "keda_enabled", false) - vertical_pod_autoscaler_enabled = lookup(local.cfg, "vertical_pod_autoscaler_enabled", false) - - upgade_settings_max_surge = lookup(local.cfg, "upgade_settings_max_surge", "10%") - upgade_settings_drain_timeout_in_minutes = lookup(local.cfg, "upgade_settings_drain_timeout_in_minutes", 0) - upgade_settings_node_soak_duration_in_minutes = lookup(local.cfg, "upgade_settings_node_soak_duration_in_minutes", 0) -} diff --git a/azurerm/_modules/aks/ingress.tf b/azurerm/cluster/ingress.tf similarity index 53% rename from azurerm/_modules/aks/ingress.tf rename to azurerm/cluster/ingress.tf index 4730b392..8c615ff2 100644 --- a/azurerm/_modules/aks/ingress.tf +++ b/azurerm/cluster/ingress.tf @@ -1,30 +1,30 @@ resource "azurerm_public_ip" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - name = var.metadata_name + name = module.cluster_metadata.name location = azurerm_kubernetes_cluster.current.location resource_group_name = azurerm_kubernetes_cluster.current.node_resource_group allocation_method = "Static" sku = "Standard" - zones = var.default_ingress_ip_zones + zones = try(coalesce(local.cfg.default_ingress_ip_zones, null), []) - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) depends_on = [azurerm_kubernetes_cluster.current] } resource "azurerm_dns_zone" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - name = var.metadata_fqdn + name = module.cluster_metadata.fqdn resource_group_name = data.azurerm_resource_group.current.name - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } resource "azurerm_dns_a_record" "host" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 name = "@" zone_name = azurerm_dns_zone.current[0].name @@ -32,11 +32,11 @@ resource "azurerm_dns_a_record" "host" { ttl = 300 records = [azurerm_public_ip.current[0].ip_address] - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } resource "azurerm_dns_a_record" "wildcard" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 name = "*" zone_name = azurerm_dns_zone.current[0].name @@ -44,5 +44,5 @@ resource "azurerm_dns_a_record" "wildcard" { ttl = 300 records = [azurerm_public_ip.current[0].ip_address] - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } diff --git a/azurerm/_modules/aks/kubeconfig.tf b/azurerm/cluster/kubeconfig.tf similarity index 100% rename from azurerm/_modules/aks/kubeconfig.tf rename to azurerm/cluster/kubeconfig.tf diff --git a/azurerm/_modules/aks/log_analytics.tf b/azurerm/cluster/log_analytics.tf similarity index 67% rename from azurerm/_modules/aks/log_analytics.tf rename to azurerm/cluster/log_analytics.tf index 879219b2..d63e2ff8 100644 --- a/azurerm/_modules/aks/log_analytics.tf +++ b/azurerm/cluster/log_analytics.tf @@ -1,15 +1,15 @@ resource "azurerm_log_analytics_workspace" "current" { - count = var.enable_log_analytics == true ? 1 : 0 - name = var.metadata_name + count = try(coalesce(local.cfg.enable_log_analytics, null), true) == true ? 1 : 0 + name = module.cluster_metadata.name location = data.azurerm_resource_group.current.location resource_group_name = data.azurerm_resource_group.current.name sku = "PerGB2018" - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } resource "azurerm_log_analytics_solution" "current" { - count = var.enable_log_analytics == true ? 1 : 0 + count = try(coalesce(local.cfg.enable_log_analytics, null), true) == true ? 1 : 0 solution_name = "ContainerInsights" location = data.azurerm_resource_group.current.location resource_group_name = data.azurerm_resource_group.current.name @@ -21,4 +21,3 @@ resource "azurerm_log_analytics_solution" "current" { product = "OMSGallery/ContainerInsights" } } - diff --git a/azurerm/cluster/main.tf b/azurerm/cluster/main.tf index 67516f9a..ac607395 100644 --- a/azurerm/cluster/main.tf +++ b/azurerm/cluster/main.tf @@ -1,76 +1,117 @@ -data "azurerm_resource_group" "current" { - name = local.resource_group +module "configuration" { + source = "../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] } module "cluster_metadata" { source = "../../common/metadata" - name_prefix = local.name_prefix - base_domain = local.base_domain + name_prefix = local.cfg.name_prefix + base_domain = local.cfg.base_domain provider_name = "azure" - provider_region = data.azurerm_resource_group.current.location + provider_region = data.azurerm_resource_group.current.location # references cluster.tf data source # Azure does not allow / character in labels label_namespace = "kubestack.com-" } -module "cluster" { - source = "../_modules/aks" +data "azurerm_resource_group" "current" { + name = local.cfg.resource_group +} + +resource "azurerm_kubernetes_cluster" "current" { + name = module.cluster_metadata.name + location = data.azurerm_resource_group.current.location + resource_group_name = data.azurerm_resource_group.current.name + dns_prefix = try(coalesce(local.cfg.dns_prefix, null), "api") + sku_tier = try(coalesce(local.cfg.sku_tier, null), "Free") + kubernetes_version = local.cfg.kubernetes_version + automatic_upgrade_channel = local.cfg.automatic_channel_upgrade + + role_based_access_control_enabled = true + + default_node_pool { + name = try(coalesce(local.cfg.default_node_pool.name, null), "default") + type = try(coalesce(local.cfg.default_node_pool.type, null), "VirtualMachineScaleSets") + + auto_scaling_enabled = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) + + # set min and max count only if autoscaling is _enabled_ + min_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? try(coalesce(local.cfg.default_node_pool.min_count, null), 1) : null + max_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? try(coalesce(local.cfg.default_node_pool.max_count, null), 1) : null + + # set node count only if auto scaling is _disabled_ + node_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? null : try(coalesce(local.cfg.default_node_pool.node_count, null), 1) + + vm_size = try(coalesce(local.cfg.default_node_pool.vm_size, null), "Standard_D1_v2") + os_disk_size_gb = try(coalesce(local.cfg.default_node_pool.os_disk_size_gb, null), 30) + + vnet_subnet_id = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? azurerm_subnet.current[0].id : null + max_pods = local.cfg.max_pods - resource_group = local.resource_group + only_critical_addons_enabled = try(coalesce(local.cfg.default_node_pool.only_critical_addons, null), false) - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_labels = merge(module.cluster_metadata.labels, local.additional_metadata_labels) - metadata_label_namespace = module.cluster_metadata.label_namespace + zones = try(coalesce(local.cfg.availability_zones, null), []) - dns_prefix = local.dns_prefix + upgrade_settings { + max_surge = try(coalesce(try(local.cfg.default_node_pool.upgrade_settings, null).max_surge, null), "10%") + drain_timeout_in_minutes = try(coalesce(try(local.cfg.default_node_pool.upgrade_settings, null).drain_timeout_in_minutes, null), 0) + node_soak_duration_in_minutes = try(coalesce(try(local.cfg.default_node_pool.upgrade_settings, null).node_soak_duration_in_minutes, null), 0) + } + } - sku_tier = local.sku_tier + network_profile { + network_plugin = try(coalesce(local.cfg.network_plugin, null), "kubenet") + network_policy = try(coalesce(local.cfg.network_policy, null), "calico") - legacy_vnet_name = local.legacy_vnet_name - vnet_address_space = local.vnet_address_space - subnet_address_prefixes = local.subnet_address_prefixes - subnet_service_endpoints = local.subnet_service_endpoints + service_cidr = try(coalesce(local.cfg.service_cidr, null), "10.0.0.0/16") + dns_service_ip = try(coalesce(local.cfg.dns_service_ip, null), "10.0.0.10") + pod_cidr = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? null : try(coalesce(local.cfg.pod_cidr, null), "10.244.0.0/16") + } - network_plugin = local.network_plugin - network_policy = local.network_policy - service_cidr = local.service_cidr - dns_service_ip = local.dns_service_ip - pod_cidr = local.pod_cidr - max_pods = local.max_pods + dynamic "identity" { + for_each = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? toset([]) : toset([1]) - default_node_pool_name = local.default_node_pool_name - default_node_pool_type = local.default_node_pool_type + content { + type = local.cfg.user_assigned_identity_id == null ? "SystemAssigned" : "UserAssigned" - default_node_pool_enable_auto_scaling = local.default_node_pool_enable_auto_scaling - default_node_pool_min_count = local.default_node_pool_min_count - default_node_pool_max_count = local.default_node_pool_max_count - default_node_pool_node_count = local.default_node_pool_node_count + identity_ids = local.cfg.user_assigned_identity_id == null ? null : [local.cfg.user_assigned_identity_id] + } + } - default_node_pool_only_critical_addons = local.default_node_pool_only_critical_addons - default_node_pool_vm_size = local.default_node_pool_vm_size - default_node_pool_os_disk_size_gb = local.default_node_pool_os_disk_size_gb + dynamic "service_principal" { + for_each = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? toset([1]) : toset([]) - disable_default_ingress = local.disable_default_ingress - default_ingress_ip_zones = local.default_ingress_ip_zones + content { + client_id = azuread_application.current[0].application_id + client_secret = azuread_service_principal_password.current[0].value + } + } - enable_azure_policy_agent = local.enable_azure_policy_agent + azure_policy_enabled = try(coalesce(local.cfg.enable_azure_policy_agent, null), false) - disable_managed_identities = local.disable_managed_identities - user_assigned_identity_id = local.user_assigned_identity_id + dynamic "oms_agent" { + for_each = try(coalesce(local.cfg.enable_log_analytics, null), true) ? toset([1]) : toset([]) - kubernetes_version = local.kubernetes_version - automatic_channel_upgrade = local.automatic_channel_upgrade - enable_log_analytics = local.enable_log_analytics + content { + log_analytics_workspace_id = azurerm_log_analytics_workspace.current[0].id + } + } - availability_zones = local.availability_zones + dynamic "workload_autoscaler_profile" { + for_each = try(coalesce(local.cfg.keda_enabled, null), false) || try(coalesce(local.cfg.vertical_pod_autoscaler_enabled, null), false) ? toset([1]) : toset([]) - keda_enabled = local.keda_enabled - vertical_pod_autoscaler_enabled = local.vertical_pod_autoscaler_enabled + content { + keda_enabled = try(coalesce(local.cfg.keda_enabled, null), false) + vertical_pod_autoscaler_enabled = try(coalesce(local.cfg.vertical_pod_autoscaler_enabled, null), false) + } + } - upgade_settings_max_surge = local.upgade_settings_max_surge - upgade_settings_drain_timeout_in_minutes = local.upgade_settings_drain_timeout_in_minutes - upgade_settings_node_soak_duration_in_minutes = local.upgade_settings_node_soak_duration_in_minutes + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } diff --git a/azurerm/cluster/moved.tf b/azurerm/cluster/moved.tf new file mode 100644 index 00000000..097a9f03 --- /dev/null +++ b/azurerm/cluster/moved.tf @@ -0,0 +1,59 @@ +moved { + from = module.cluster.azurerm_kubernetes_cluster.current + to = azurerm_kubernetes_cluster.current +} + +moved { + from = module.cluster.azurerm_virtual_network.current + to = azurerm_virtual_network.current +} + +moved { + from = module.cluster.azurerm_subnet.current + to = azurerm_subnet.current +} + +moved { + from = module.cluster.azurerm_log_analytics_workspace.current + to = azurerm_log_analytics_workspace.current +} + +moved { + from = module.cluster.azurerm_log_analytics_solution.current + to = azurerm_log_analytics_solution.current +} + +moved { + from = module.cluster.azurerm_public_ip.current + to = azurerm_public_ip.current +} + +moved { + from = module.cluster.azurerm_dns_zone.current + to = azurerm_dns_zone.current +} + +moved { + from = module.cluster.azurerm_dns_a_record.host + to = azurerm_dns_a_record.host +} + +moved { + from = module.cluster.azurerm_dns_a_record.wildcard + to = azurerm_dns_a_record.wildcard +} + +moved { + from = module.cluster.azuread_application.current + to = azuread_application.current +} + +moved { + from = module.cluster.azuread_service_principal.current + to = azuread_service_principal.current +} + +moved { + from = module.cluster.azuread_service_principal_password.current + to = azuread_service_principal_password.current +} diff --git a/azurerm/cluster/node-pool/configuration.tf b/azurerm/cluster/node-pool/configuration.tf deleted file mode 100644 index c7c4a307..00000000 --- a/azurerm/cluster/node-pool/configuration.tf +++ /dev/null @@ -1,43 +0,0 @@ -module "configuration" { - source = "../../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - cfg = module.configuration.merged[terraform.workspace] - - node_pool_name = local.cfg["node_pool_name"] - vm_size = local.cfg["vm_size"] != null ? local.cfg["vm_size"] : "Standard_B2s" - max_pods = local.cfg["max_pods"] != null ? local.cfg["max_pods"] : "110" - os_disk_type = local.cfg["os_disk_type"] != null ? local.cfg["os_disk_type"] : "Managed" - os_disk_size_gb = local.cfg["os_disk_size_gb"] - availability_zones_list = local.cfg["availability_zones"] != null ? local.cfg["availability_zones"] : [] - availability_zones = length(local.availability_zones_list) == 0 ? null : local.availability_zones_list - - enable_auto_scaling = local.cfg["enable_auto_scaling"] != null ? local.cfg["enable_auto_scaling"] : true - max_count_string = local.cfg["max_count"] != null ? local.cfg["max_count"] : "1" - min_count_string = local.cfg["min_count"] != null ? local.cfg["min_count"] : "1" - max_count = local.enable_auto_scaling ? local.max_count_string : null - min_count = local.enable_auto_scaling ? local.min_count_string : null - node_count = local.cfg["node_count"] != null ? local.cfg["node_count"] : "1" - - use_spot = local.cfg["use_spot"] != null ? local.cfg["use_spot"] : false - priority = local.use_spot ? "Spot" : "Regular" - eviction_policy = local.use_spot ? local.cfg["eviction_policy"] : null - max_spot_price_value = local.cfg["max_spot_price"] != null ? local.cfg["max_spot_price"] : "-1" - max_spot_price = local.use_spot ? local.max_spot_price_value : null - - user_node_labels = local.cfg["node_labels"] != null ? local.cfg["node_labels"] : {} - user_node_taints = local.cfg["node_taints"] != null ? local.cfg["node_taints"] : [] - spot_taints = [ - "kubernetes.azure.com/scalesetpriority=spot:NoSchedule", - ] - spot_labels = { "kubernetes.azure.com/scalesetpriority" = "spot" } - - taints = compact(concat(local.use_spot ? local.spot_taints : [], local.user_node_taints)) - labels = merge(local.use_spot ? local.spot_labels : {}, local.user_node_labels) - node_labels = length(local.labels) == 0 ? null : local.labels - node_taints = length(local.taints) == 0 ? null : local.taints -} diff --git a/azurerm/cluster/node-pool/main.tf b/azurerm/cluster/node-pool/main.tf index 65d5a3e8..2d8ba627 100644 --- a/azurerm/cluster/node-pool/main.tf +++ b/azurerm/cluster/node-pool/main.tf @@ -1,22 +1,63 @@ -module "node_pool" { - source = "../../_modules/aks/node_pool" - - cluster_name = var.cluster_name - resource_group = var.resource_group - - node_pool_name = local.node_pool_name - enable_auto_scaling = local.enable_auto_scaling - max_count = local.max_count - min_count = local.min_count - node_count = local.node_count - vm_size = local.vm_size - eviction_policy = local.eviction_policy - priority = local.priority - max_spot_price = local.max_spot_price - node_labels = local.node_labels - node_taints = local.node_taints - availability_zones = local.availability_zones - os_disk_type = local.os_disk_type - os_disk_size_gb = local.os_disk_size_gb - max_pods = local.max_pods +module "configuration" { + source = "../../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] + + vnet_subnets = compact(var.cluster.default_node_pool.*.vnet_subnet_id) + + # Conditional logic for auto_scaling + effective_max_count = try(coalesce(local.cfg.enable_auto_scaling, null), true) == true ? local.cfg.max_count : null + effective_min_count = try(coalesce(local.cfg.enable_auto_scaling, null), true) == true ? local.cfg.min_count : null + + # Conditional logic for spot instances + effective_priority = try(coalesce(local.cfg.use_spot, null), false) == true ? "Spot" : "Regular" + effective_eviction_policy = try(coalesce(local.cfg.use_spot, null), false) == true ? local.cfg.eviction_policy : null + effective_max_spot_price = try(coalesce(local.cfg.use_spot, null), false) == true ? try(coalesce(local.cfg.max_spot_price, null), -1) : null + + # Spot instance labels and taints + spot_label = try(coalesce(local.cfg.use_spot, null), false) == true ? { "kubernetes.azure.com/scalesetpriority" = "spot" } : {} + spot_taints = try(coalesce(local.cfg.use_spot, null), false) == true ? ["kubernetes.azure.com/scalesetpriority=spot:NoSchedule"] : [] + + # Merge labels and taints + node_labels_merged = merge(local.spot_label, try(coalesce(local.cfg.node_labels, null), {})) + node_taints_merged = compact(concat(local.spot_taints, try(coalesce(local.cfg.node_taints, null), []))) +} + +resource "azurerm_kubernetes_cluster_node_pool" "current" { + name = local.cfg.node_pool_name + kubernetes_cluster_id = var.cluster.id + auto_scaling_enabled = try(coalesce(local.cfg.enable_auto_scaling, null), true) + max_count = local.effective_max_count + min_count = local.effective_min_count + node_count = try(coalesce(local.cfg.node_count, null), 1) + vm_size = try(coalesce(local.cfg.vm_size, null), "Standard_B2s") + node_labels = local.node_labels_merged + node_taints = local.node_taints_merged + zones = local.cfg.availability_zones + max_pods = try(coalesce(local.cfg.max_pods, null), 110) + os_disk_type = try(coalesce(local.cfg.os_disk_type, null), "Managed") + os_disk_size_gb = local.cfg.os_disk_size_gb + priority = local.effective_priority + eviction_policy = local.effective_eviction_policy + spot_max_price = local.effective_max_spot_price + + upgrade_settings { + max_surge = try(coalesce(try(local.cfg.upgrade_settings, null).max_surge, null), "10%") + drain_timeout_in_minutes = try(coalesce(try(local.cfg.upgrade_settings, null).drain_timeout_in_minutes, null), 0) + node_soak_duration_in_minutes = try(coalesce(try(local.cfg.upgrade_settings, null).node_soak_duration_in_minutes, null), 0) + } + + # The cluster's agent_pool_profile may contain empty strings for vnet_subnet_id + # in some configurations. In that case we rely on the defaults. + vnet_subnet_id = length(local.vnet_subnets) == 0 ? null : coalesce(tolist(local.vnet_subnets)...) + + # When autoscaling acts, the node_count gets changed, but it should not be + # forced to match the config + lifecycle { + ignore_changes = [node_count] + } } diff --git a/azurerm/cluster/node-pool/moved.tf b/azurerm/cluster/node-pool/moved.tf new file mode 100644 index 00000000..20b14c3f --- /dev/null +++ b/azurerm/cluster/node-pool/moved.tf @@ -0,0 +1,4 @@ +moved { + from = module.node_pool.azurerm_kubernetes_cluster_node_pool.current + to = azurerm_kubernetes_cluster_node_pool.current +} diff --git a/azurerm/cluster/node-pool/outputs.tf b/azurerm/cluster/node-pool/outputs.tf new file mode 100644 index 00000000..871c9f88 --- /dev/null +++ b/azurerm/cluster/node-pool/outputs.tf @@ -0,0 +1,3 @@ +output "current_config" { + value = local.cfg +} diff --git a/azurerm/cluster/node-pool/variables.tf b/azurerm/cluster/node-pool/variables.tf index e62c97b2..a2cc1c8f 100644 --- a/azurerm/cluster/node-pool/variables.tf +++ b/azurerm/cluster/node-pool/variables.tf @@ -2,39 +2,43 @@ variable "configuration" { type = map(object({ node_pool_name = optional(string) vm_size = optional(string) - node_count = optional(string) + node_count = optional(number) enable_auto_scaling = optional(bool) - max_count = optional(string) - min_count = optional(string) + max_count = optional(number) + min_count = optional(number) eviction_policy = optional(string) - max_pods = optional(string) + max_pods = optional(number) os_disk_type = optional(string) - os_disk_size_gb = optional(string) + os_disk_size_gb = optional(number) use_spot = optional(bool) - max_spot_price = optional(string) + max_spot_price = optional(number) node_labels = optional(map(string)) node_taints = optional(list(string)) availability_zones = optional(list(string)) + + upgrade_settings = optional(object({ + max_surge = optional(string) + drain_timeout_in_minutes = optional(number) + node_soak_duration_in_minutes = optional(number) + })) })) description = "Map with per workspace node pool configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } -variable "cluster_name" { - type = string - description = "The name of the cluster to attach the node pool to" -} - -variable "resource_group" { - type = string - description = "The resource group of the cluster to attach the node pool to" +variable "cluster" { + type = any + description = "The cluster output from the cluster module." + nullable = false } diff --git a/azurerm/cluster/node-pool/versions.tf b/azurerm/cluster/node-pool/versions.tf index e4967d8e..cf41232e 100644 --- a/azurerm/cluster/node-pool/versions.tf +++ b/azurerm/cluster/node-pool/versions.tf @@ -1,6 +1,6 @@ terraform { required_providers { - azure = { + azurerm = { source = "hashicorp/azurerm" } } diff --git a/azurerm/cluster/outputs.tf b/azurerm/cluster/outputs.tf index d0d3deda..7a27b44d 100644 --- a/azurerm/cluster/outputs.tf +++ b/azurerm/cluster/outputs.tf @@ -1,9 +1,13 @@ +output "cluster" { + value = azurerm_kubernetes_cluster.current +} + output "aks_vnet" { - value = module.cluster.aks_vnet + value = length(azurerm_virtual_network.current) > 0 ? azurerm_virtual_network.current[0] : null } output "current_config" { - value = module.configuration.merged[terraform.workspace] + value = local.cfg } output "current_metadata" { @@ -12,9 +16,9 @@ output "current_metadata" { output "kubeconfig" { sensitive = true - value = module.cluster.kubeconfig + value = local.kubeconfig } output "default_ingress_ip" { - value = module.cluster.default_ingress_ip + value = length(azurerm_public_ip.current) > 0 ? azurerm_public_ip.current[0].ip_address : null } diff --git a/azurerm/_modules/aks/provider.tf b/azurerm/cluster/provider.tf similarity index 100% rename from azurerm/_modules/aks/provider.tf rename to azurerm/cluster/provider.tf diff --git a/azurerm/cluster/service_principal.tf b/azurerm/cluster/service_principal.tf new file mode 100644 index 00000000..376158de --- /dev/null +++ b/azurerm/cluster/service_principal.tf @@ -0,0 +1,17 @@ +resource "azuread_application" "current" { + count = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? 1 : 0 + + display_name = module.cluster_metadata.name +} + +resource "azuread_service_principal" "current" { + count = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? 1 : 0 + + client_id = azuread_application.current[0].client_id +} + +resource "azuread_service_principal_password" "current" { + count = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? 1 : 0 + + service_principal_id = azuread_service_principal.current[0].id +} diff --git a/azurerm/_modules/aks/templates/kubeconfig.tpl b/azurerm/cluster/templates/kubeconfig.tpl similarity index 100% rename from azurerm/_modules/aks/templates/kubeconfig.tpl rename to azurerm/cluster/templates/kubeconfig.tpl diff --git a/azurerm/cluster/variables.tf b/azurerm/cluster/variables.tf index 8bf95161..2fd729d4 100644 --- a/azurerm/cluster/variables.tf +++ b/azurerm/cluster/variables.tf @@ -1,10 +1,71 @@ variable "configuration" { - type = map(map(string)) + type = map(object({ + # Required attributes + name_prefix = optional(string) + base_domain = optional(string) + resource_group = optional(string) + + # Optional attributes without defaults + dns_prefix = optional(string) + sku_tier = optional(string) + + legacy_vnet_name = optional(bool) + vnet_address_space = optional(list(string)) + subnet_address_prefixes = optional(list(string)) + subnet_service_endpoints = optional(list(string)) + + network_plugin = optional(string) + network_policy = optional(string) + service_cidr = optional(string) + dns_service_ip = optional(string) + pod_cidr = optional(string) + max_pods = optional(number) + + default_node_pool = optional(object({ + name = optional(string) + type = optional(string) + enable_auto_scaling = optional(bool) + min_count = optional(number) + max_count = optional(number) + node_count = optional(number) + vm_size = optional(string) + only_critical_addons = optional(bool) + os_disk_size_gb = optional(number) + upgrade_settings = optional(object({ + max_surge = optional(string) + drain_timeout_in_minutes = optional(number) + node_soak_duration_in_minutes = optional(number) + })) + })) + + disable_default_ingress = optional(bool) + + default_ingress_ip_zones = optional(list(string)) + + enable_azure_policy_agent = optional(bool) + + disable_managed_identities = optional(bool) + user_assigned_identity_id = optional(string) + + enable_log_analytics = optional(bool) + + kubernetes_version = optional(string) + automatic_channel_upgrade = optional(string) + + availability_zones = optional(list(string)) + + additional_metadata_labels = optional(map(string)) + + keda_enabled = optional(bool) + vertical_pod_autoscaler_enabled = optional(bool) + })) description = "Map with per workspace cluster configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } diff --git a/azurerm/cluster/versions.tf b/azurerm/cluster/versions.tf index 2606a5ac..f1424a25 100644 --- a/azurerm/cluster/versions.tf +++ b/azurerm/cluster/versions.tf @@ -1,4 +1,13 @@ - terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + + azuread = { + source = "hashicorp/azuread" + } + } + required_version = ">= 0.13" } diff --git a/azurerm/cluster/vnet.tf b/azurerm/cluster/vnet.tf new file mode 100644 index 00000000..c59f4776 --- /dev/null +++ b/azurerm/cluster/vnet.tf @@ -0,0 +1,19 @@ +resource "azurerm_virtual_network" "current" { + count = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? 1 : 0 + + name = try(coalesce(local.cfg.legacy_vnet_name, null), false) ? "vnet-aks-${terraform.workspace}-cluster" : module.cluster_metadata.name + address_space = try(coalesce(local.cfg.vnet_address_space, null), ["10.0.0.0/8"]) + resource_group_name = data.azurerm_resource_group.current.name + location = data.azurerm_resource_group.current.location +} + +resource "azurerm_subnet" "current" { + count = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? 1 : 0 + + name = try(coalesce(local.cfg.legacy_vnet_name, null), false) ? "aks-node-subnet" : "${module.cluster_metadata.name}-${try(coalesce(local.cfg.default_node_pool.name, null), "default")}-node-pool" + address_prefixes = try(coalesce(local.cfg.subnet_address_prefixes, null), ["10.1.0.0/16"]) + resource_group_name = data.azurerm_resource_group.current.name + virtual_network_name = azurerm_virtual_network.current[0].name + + service_endpoints = length(try(coalesce(local.cfg.subnet_service_endpoints, null), [])) > 0 ? try(coalesce(local.cfg.subnet_service_endpoints, null), []) : null +} diff --git a/common/configuration/outputs.tf b/common/configuration/outputs.tf index 37695c95..010c38a3 100644 --- a/common/configuration/outputs.tf +++ b/common/configuration/outputs.tf @@ -1,14 +1,32 @@ locals { - base_config = var.configuration[var.base_key] + base_cfg = var.configuration[var.base_key] merged = { - for env_key, env in var.configuration : + # loop through all environments + for env_key, env_cfg in var.configuration : env_key => { - # loop through all config keys in base_key environment and current env - # if current env has that key, use the value from current env - # if not, use the value from the base_key environment - for key in setunion(keys(env), keys(local.base_config)) : - key => lookup(env, key, null) != null ? env[key] : local.base_config[key] + # loop through all keys in current and base_key environment config + for cfg_key in setunion(keys(env_cfg), keys(local.base_cfg)) : + cfg_key => try( + # try one level of nesting + { + for nested_key in setunion(keys(env_cfg[cfg_key]), keys(local.base_cfg[cfg_key])) : + nested_key => try( + # use current environment's nested value or base env's value + coalesce( + lookup(env_cfg[cfg_key], nested_key, null), + lookup(local.base_cfg[cfg_key], nested_key, null), + ), + local.base_cfg[cfg_key][nested_key] + ) + }, + # fall back to use current environment's value or base env's value + coalesce( + lookup(env_cfg, cfg_key, null), + lookup(local.base_cfg, cfg_key, null), + ), + local.base_cfg[cfg_key] + ) } } } diff --git a/common/configuration/tests/non_string_values/test_hash.tf b/common/configuration/tests/non_string_values/test_hash.tf index 10e47496..b0d40b19 100644 --- a/common/configuration/tests/non_string_values/test_hash.tf +++ b/common/configuration/tests/non_string_values/test_hash.tf @@ -22,18 +22,18 @@ module "mut_hash" { base_key = "apps" } -resource "test_assertions" "overwrite_hash" { - component = "overwrite_hash" +resource "test_assertions" "merge_hash" { + component = "merge_hash" equal "scheme" { - description = "can overwrite hashes" + description = "can merge hashes" got = module.mut_hash.merged want = { "apps" = { - "test_hash" = tomap({ + "test_hash" = { "from_apps_1" = "from_apps_1" "from_apps_2" = "from_apps_2" - }) + } } "loc" = { "test_hash" = tomap({ @@ -42,9 +42,11 @@ resource "test_assertions" "overwrite_hash" { }) } "ops" = { - "test_hash" = tomap({ - "from_ops_1" = "from_ops_1" - }) + "test_hash" = { + "from_apps_1" = "from_apps_1" + "from_apps_2" = "from_apps_2" + "from_ops_1" = "from_ops_1" + } } } } diff --git a/common/configuration/tests/non_string_values/test_object.tf b/common/configuration/tests/non_string_values/test_object.tf index c45e4239..99638891 100644 --- a/common/configuration/tests/non_string_values/test_object.tf +++ b/common/configuration/tests/non_string_values/test_object.tf @@ -2,11 +2,11 @@ module "mut_object" { source = "./wrapper" } -resource "test_assertions" "overwrite_object" { - component = "overwrite_object" +resource "test_assertions" "merge_object" { + component = "merge_object" equal "scheme" { - description = "can overwrite objects" + description = "can merge objects" got = module.mut_object.merged want = { "apps" = { @@ -18,9 +18,14 @@ resource "test_assertions" "overwrite_object" { "test_list_string" = tolist([ "from_apps", ]) - "test_map_string" = tomap({ + "test_map_object" = { + "env" = { + "key" = "from_apps" + } + } + "test_map_string" = { "key" = "from_apps" - }) + } "test_object" = { "key" = "from_apps" } @@ -35,6 +40,11 @@ resource "test_assertions" "overwrite_object" { "test_list_string" = tolist([ "from_apps", ]) + "test_map_object" = tomap({ + "env" = { + "key" = "from_apps" + } + }) "test_map_string" = tomap({ "key" = "from_apps" }) @@ -52,9 +62,14 @@ resource "test_assertions" "overwrite_object" { "test_list_string" = tolist([ "from_ops", ]) - "test_map_string" = tomap({ + "test_map_object" = { + "env" = { + "key" = "from_ops" + } + } + "test_map_string" = { "key" = "from_ops" - }) + } "test_object" = { "key" = "from_ops" } diff --git a/common/configuration/tests/non_string_values/wrapper/main.tf b/common/configuration/tests/non_string_values/wrapper/main.tf index ed2a2e7b..3a16f872 100644 --- a/common/configuration/tests/non_string_values/wrapper/main.tf +++ b/common/configuration/tests/non_string_values/wrapper/main.tf @@ -13,6 +13,9 @@ variable "test_configuration" { test_object = optional(object({ key = string })) + test_map_object = optional(map(object({ + key = string + }))) })) description = "Map with per workspace module configuration." default = { @@ -34,6 +37,10 @@ variable "test_configuration" { test_object = { key = "from_apps" } + + test_map_object = { + env = { key = "from_apps" } + } } ops = { @@ -54,6 +61,10 @@ variable "test_configuration" { test_object = { key = "from_ops" } + + test_map_object = { + env = { key = "from_ops" } + } } loc = { diff --git a/common/metadata/variables.tf b/common/metadata/variables.tf index fface973..6044891e 100644 --- a/common/metadata/variables.tf +++ b/common/metadata/variables.tf @@ -35,4 +35,3 @@ variable "label_namespace" { description = "Prefix labels are namespaced with." default = "kubestack.com/" } - diff --git a/google/_modules/gke/cluster.tf b/google/_modules/gke/cluster.tf deleted file mode 100644 index b0f445dd..00000000 --- a/google/_modules/gke/cluster.tf +++ /dev/null @@ -1,131 +0,0 @@ -resource "google_container_cluster" "current" { - project = var.project - name = var.metadata_name - - deletion_protection = var.deletion_protection - - location = var.location - node_locations = var.node_locations - - min_master_version = var.min_master_version - - release_channel { - channel = var.release_channel - } - - remove_default_node_pool = var.remove_default_node_pool - initial_node_count = var.initial_node_count - - master_auth { - client_certificate_config { - issue_client_certificate = false - } - } - - network = google_compute_network.current.self_link - - dynamic "workload_identity_config" { - for_each = var.disable_workload_identity == false ? toset([1]) : toset([]) - content { - workload_pool = "${var.project}.svc.id.goog" - } - } - - dynamic "database_encryption" { - for_each = var.cluster_database_encryption_key_name != null ? toset([1]) : toset([]) - content { - state = "ENCRYPTED" - key_name = var.cluster_database_encryption_key_name - } - } - - # - # - # Addon config - addons_config { - http_load_balancing { - disabled = true - } - - horizontal_pod_autoscaling { - disabled = false - } - - network_policy_config { - disabled = false - } - - dynamic "gcs_fuse_csi_driver_config" { - for_each = var.enable_gcs_fuse_csi_driver != null ? [1] : [] - - content { - enabled = var.enable_gcs_fuse_csi_driver - } - } - } - - network_policy { - enabled = true - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.daily_maintenance_window_start_time - } - - dynamic "maintenance_exclusion" { - for_each = var.maintenance_exclusion_start_time != "" ? [1] : [] - - content { - start_time = var.maintenance_exclusion_start_time - end_time = var.maintenance_exclusion_end_time - exclusion_name = var.maintenance_exclusion_name - - exclusion_options { - scope = var.maintenance_exclusion_scope - } - } - } - } - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config_cidr_blocks == null ? toset([]) : toset([1]) - - content { - dynamic "cidr_blocks" { - for_each = var.master_authorized_networks_config_cidr_blocks - - content { - cidr_block = cidr_blocks.value - display_name = "terraform-kubestack_${cidr_blocks.value}" - } - } - } - } - - logging_config { - enable_components = var.logging_config_enable_components - } - - monitoring_config { - enable_components = var.monitoring_config_enable_components - } - - private_cluster_config { - enable_private_nodes = var.enable_private_nodes - enable_private_endpoint = false - master_ipv4_cidr_block = var.master_cidr_block - } - - dynamic "ip_allocation_policy" { - for_each = var.enable_private_nodes ? toset([1]) : [] - - content { - cluster_ipv4_cidr_block = var.cluster_ipv4_cidr_block - services_ipv4_cidr_block = var.services_ipv4_cidr_block - } - } - - enable_intranode_visibility = var.enable_intranode_visibility - enable_tpu = var.enable_tpu -} diff --git a/google/_modules/gke/network.tf b/google/_modules/gke/network.tf deleted file mode 100644 index c398a04b..00000000 --- a/google/_modules/gke/network.tf +++ /dev/null @@ -1,78 +0,0 @@ -resource "google_compute_network" "current" { - name = var.metadata_name - project = var.project - auto_create_subnetworks = "true" -} - -resource "google_compute_address" "nat" { - count = var.enable_cloud_nat ? var.cloud_nat_ip_count : 0 - - region = google_container_cluster.current.location - project = var.project - - name = "nat-${var.metadata_name}-${count.index}" -} - -resource "google_compute_router" "current" { - count = var.enable_cloud_nat ? 1 : 0 - - project = var.project - name = var.metadata_name - region = google_container_cluster.current.location - - network = google_compute_network.current.name - - bgp { - advertise_mode = ( - var.router_advertise_config == null - ? null - : var.router_advertise_config.mode - ) - advertised_groups = ( - var.router_advertise_config == null ? null : ( - var.router_advertise_config.mode != "CUSTOM" - ? null - : var.router_advertise_config.groups - ) - ) - dynamic "advertised_ip_ranges" { - for_each = ( - var.router_advertise_config == null ? {} : ( - var.router_advertise_config.mode != "CUSTOM" - ? {} - : var.router_advertise_config.ip_ranges - ) - ) - iterator = range - content { - range = range.key - description = range.value - } - } - - # expected "bgp.0.asn" to be a RFC6996-compliant Local ASN: - # must be either in the private ASN ranges: [64512..65534], [4200000000..4294967294]; - # or be the value of [16550] - asn = var.router_asn != null ? var.router_asn : 16550 - } -} - -resource "google_compute_router_nat" "nat" { - count = var.enable_cloud_nat ? 1 : 0 - - project = var.project - name = var.metadata_name - region = google_compute_router.current[0].region - router = google_compute_router.current[0].name - - enable_endpoint_independent_mapping = var.cloud_nat_endpoint_independent_mapping - min_ports_per_vm = var.cloud_nat_min_ports_per_vm - nat_ip_allocate_option = var.cloud_nat_ip_count > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" - source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" - nat_ips = var.cloud_nat_ip_count > 0 ? google_compute_address.nat.*.self_link : null - - log_config { - enable = true - filter = "ERRORS_ONLY" - } -} diff --git a/google/_modules/gke/node_pool.tf b/google/_modules/gke/node_pool.tf deleted file mode 100644 index 05d938fa..00000000 --- a/google/_modules/gke/node_pool.tf +++ /dev/null @@ -1,38 +0,0 @@ -module "node_pool" { - source = "./node_pool" - - project = var.project - location = google_container_cluster.current.location - - cluster_name = google_container_cluster.current.name - pool_name = "default" - - service_account_email = google_service_account.current.email - disable_per_node_pool_service_account = true - - metadata_tags = var.metadata_tags - metadata_labels = var.metadata_labels - - initial_node_count = var.initial_node_count - min_node_count = var.min_node_count - max_node_count = var.max_node_count - location_policy = var.location_policy - - extra_oauth_scopes = var.extra_oauth_scopes - - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - image_type = var.image_type - machine_type = var.machine_type - - # Whether to use preemptible nodes for this node pool - preemptible = var.preemptible - # Whether the nodes will be automatically repaired - auto_repair = var.auto_repair - # Whether the nodes will be automatically upgraded - auto_upgrade = var.auto_upgrade - - node_workload_metadata_config = var.node_workload_metadata_config - - guest_accelerator = var.guest_accelerator -} diff --git a/google/_modules/gke/node_pool/main.tf b/google/_modules/gke/node_pool/main.tf deleted file mode 100644 index 9504cfaa..00000000 --- a/google/_modules/gke/node_pool/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -resource "google_container_node_pool" "current" { - name = var.pool_name - project = var.project - cluster = var.cluster_name - location = var.location - - initial_node_count = var.initial_node_count - - autoscaling { - min_node_count = var.min_node_count - max_node_count = var.max_node_count - location_policy = var.location_policy - } - - node_locations = var.node_locations - - dynamic "network_config" { - for_each = var.network_config == null ? [] : [1] - - content { - enable_private_nodes = var.network_config["enable_private_nodes"] - create_pod_range = var.network_config["create_pod_range"] - pod_ipv4_cidr_block = var.network_config["pod_ipv4_cidr_block"] - } - } - - # - # - # Node config - node_config { - service_account = var.disable_per_node_pool_service_account ? var.service_account_email : google_service_account.current[0].email - - oauth_scopes = local.oauth_scopes - - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - - image_type = var.image_type - machine_type = var.machine_type - preemptible = var.preemptible - - labels = merge(var.labels, var.metadata_labels) - - tags = concat(var.metadata_tags, var.instance_tags) - - workload_metadata_config { - mode = var.node_workload_metadata_config - } - - dynamic "guest_accelerator" { - # Make sure to generate this only once - for_each = var.guest_accelerator == null ? [] : [1] - - content { - type = var.guest_accelerator.type - count = var.guest_accelerator.count - - dynamic "gpu_sharing_config" { - for_each = var.guest_accelerator.gpu_sharing_config == null ? [] : [1] - - content { - gpu_sharing_strategy = var.guest_accelerator.gpu_sharing_config.gpu_sharing_strategy - max_shared_clients_per_gpu = var.guest_accelerator.gpu_sharing_config.max_shared_clients_per_gpu - } - } - } - } - - dynamic "taint" { - for_each = var.taints == null ? [] : var.taints - - content { - key = taint.value["key"] - value = taint.value["value"] - effect = taint.value["effect"] - } - } - - dynamic "ephemeral_storage_local_ssd_config" { - for_each = var.ephemeral_storage_local_ssd_config == null ? [] : [1] - - content { - local_ssd_count = var.ephemeral_storage_local_ssd_config.local_ssd_count - } - } - } - - management { - auto_repair = var.auto_repair - auto_upgrade = var.auto_upgrade - } -} diff --git a/google/_modules/gke/node_pool/variables.tf b/google/_modules/gke/node_pool/variables.tf deleted file mode 100644 index 7eb53a4d..00000000 --- a/google/_modules/gke/node_pool/variables.tf +++ /dev/null @@ -1,170 +0,0 @@ -variable "project" { - type = string - description = "Project the cluster belongs to." -} - -variable "cluster_name" { - type = string - description = "Name of the cluster for this node pool." -} - -variable "metadata_tags" { - type = list(string) - description = "Metadata tags to use." -} - -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "pool_name" { - description = "Name of the node pool." - type = string -} - -variable "location" { - type = string - description = "location of the cluster this node pool belongs to." -} - -variable "initial_node_count" { - description = "Initial number of nodes for this node pool." - type = string -} - -variable "min_node_count" { - description = "Min number of nodes for this node pool." - type = string -} - -variable "max_node_count" { - description = "Max number of nodes for this node pool." - type = string -} - -variable "location_policy" { - type = string - description = "Location policy specifies the algorithm used when scaling-up the node pool." -} - -variable "service_account_email" { - description = "The service account email to use for this node pool." - type = string - default = null -} - -variable "disable_per_node_pool_service_account" { - description = "Skip creating a dedicated service account to use for this node pool." - type = string - default = false -} - -variable "extra_oauth_scopes" { - description = "List of additional oauth scopes for workers." - type = list(string) -} - -variable "disk_size_gb" { - description = "The disk size of nodes in this pool." - type = string -} - -variable "disk_type" { - description = "The disk type of nodes in this pool." - type = string -} - -variable "image_type" { - description = "The image type for nodes in this pool." - type = string - default = "COS" -} - -variable "machine_type" { - description = "The machine type for nodes in this pool." - type = string -} - -variable "preemptible" { - description = "Whether to use preemptible nodes for this node pool." - type = string - default = false -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = string - default = true -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = string - default = true -} - -variable "node_workload_metadata_config" { - description = "How to expose the node metadata to the workload running on the node." - type = string -} - -variable "taints" { - type = set(object({ - key = string - value = string - effect = string - })) - description = "Taints to configure for the node pool." - default = null -} - -variable "instance_tags" { - type = list(string) - description = "List of instance tags to apply to nodes." - default = [] -} - -variable "node_locations" { - type = list(string) - description = "List of zones in the cluster's region to start worker nodes in. Defaults to cluster's node locations." - default = null -} - -variable "guest_accelerator" { - type = object({ - type = string - count = number - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = optional(string) - max_shared_clients_per_gpu = optional(number) - })) - }) - description = "`guest_accelerator` block supports during node_group creation, useful to provision GPU-capable nodes. Default to `null` or `{}` which will disable GPUs." - default = null -} - -variable "ephemeral_storage_local_ssd_config" { - type = object({ - local_ssd_count = number - }) - description = "`ephemeral_storage_local_ssd_config` block, useful for node groups with local SSD. Defaults to `null`" - default = null -} - -variable "labels" { - type = map(string) - description = "Kubernetes labels to set on the nodes created by the node pool. Merged with Kubestack default labels." - default = {} -} - -variable "network_config" { - type = object({ - enable_private_nodes = bool - create_pod_range = bool - pod_ipv4_cidr_block = string - }) - description = "Configure additional pod IP address range for the node pool. Defaults to `null`" - default = null -} diff --git a/google/_modules/gke/node_pool/versions.tf b/google/_modules/gke/node_pool/versions.tf deleted file mode 100644 index ac97c6ac..00000000 --- a/google/_modules/gke/node_pool/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/google/_modules/gke/outputs.tf b/google/_modules/gke/outputs.tf deleted file mode 100644 index 4ef9ff49..00000000 --- a/google/_modules/gke/outputs.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "kubeconfig" { - value = local.kubeconfig - - # when the node pool is destroyed before the k8s namespaces - # the namespaces get stuck in terminating - depends_on = [module.node_pool] -} - -output "default_ingress_ip" { - value = length(google_compute_address.current) > 0 ? google_compute_address.current[0].address : null -} diff --git a/google/_modules/gke/variables.tf b/google/_modules/gke/variables.tf deleted file mode 100644 index a06143ee..00000000 --- a/google/_modules/gke/variables.tf +++ /dev/null @@ -1,258 +0,0 @@ -variable "project" { - type = string - description = "Project the cluster belongs to." -} - -variable "deletion_protection" { - type = bool - description = "Must be set to false to destroy clusters." -} - -variable "metadata_name" { - type = string - description = "Metadata name to use." -} - -variable "metadata_fqdn" { - type = string - description = "DNS name of the zone. E.g. `infra.example.com`" -} - -variable "metadata_tags" { - type = list(string) - description = "Metadata tags to use." -} - -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "location" { - type = string - description = "Location accepts a region or zone and starts a regional or zonal cluster respectively. Kubestack is only supported for regional clusters." -} - -variable "location_policy" { - type = string - description = "Location policy specifies the algorithm used when scaling-up the node pool." -} - -variable "min_master_version" { - type = string - description = "Minimum GKE master version." -} - -variable "release_channel" { - type = string - description = "The selected release channel. Accepted values are: UNSPECIFIED, RAPID, REGULAR, STABLE" -} - -variable "daily_maintenance_window_start_time" { - type = string - description = "Start time of the daily maintenance window." -} - -variable "maintenance_exclusion_start_time" { - type = string - description = "Maintenance exclusion start time" -} - -variable "maintenance_exclusion_end_time" { - type = string - description = "Maintenance exclusion end time" -} - -variable "maintenance_exclusion_name" { - type = string - description = "Maintenance exclusion name" -} - -variable "maintenance_exclusion_scope" { - type = string - description = "Maintenance exclusion scope" -} - -variable "remove_default_node_pool" { - type = string - description = "Whether to remove the default node pool. Leave true, except for upgrading from Kubestack v0.2.0-beta.0." -} - -variable "initial_node_count" { - description = "Initial number of nodes for this node pool." - type = string -} - -variable "min_node_count" { - description = "Min number of nodes for this node pool." - type = string -} - -variable "max_node_count" { - description = "Max number of nodes for this node pool." - type = string -} - -variable "node_locations" { - type = list(string) - description = "List of zones in the cluster's region to start worker nodes in." -} - -variable "extra_oauth_scopes" { - description = "List of additional oauth scopes for workers." - type = list(string) -} - -variable "disk_size_gb" { - description = "The disk size of nodes in this pool." - type = string -} - -variable "disk_type" { - description = "The disk type of nodes in this pool." - type = string -} - -variable "image_type" { - description = "The image type for nodes in this pool." - type = string - default = "COS_containerd" -} - -variable "machine_type" { - description = "The machine type for nodes in this pool." - type = string -} - -variable "preemptible" { - description = "Whether to use preemptible nodes for this node pool." - type = string -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = string -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = string -} - -variable "disable_default_ingress" { - type = bool - description = "Whether to disable the default ingress." -} - -variable "enable_private_nodes" { - type = bool - description = "Whether to enable private nodes" -} - -variable "master_cidr_block" { - type = string - description = "The IP range for the master network" -} - -variable "cluster_ipv4_cidr_block" { - type = string - description = "The CIDR block used for Kubernetes pods." -} - -variable "services_ipv4_cidr_block" { - type = string - description = "The CIDR block used for Kubernetes services." -} - -variable "enable_cloud_nat" { - type = bool - description = "Whether to enable cloud nat and allow internet access for private nodes." -} - -variable "cloud_nat_endpoint_independent_mapping" { - type = bool - description = "Whether to enable cloud nat endpoint independent mapping" -} - -variable "cloud_nat_min_ports_per_vm" { - type = number - description = "The min amount of ports per VM allowed for NAT mapping" -} - -variable "cloud_nat_ip_count" { - type = number - description = "The amount of IP addresses to attach to the NAT gateway, changes NAT to use MANUAL_ONLY" -} - -variable "disable_workload_identity" { - description = "Wheter to disable workload identity support." - type = bool -} - -variable "node_workload_metadata_config" { - description = "How to expose the node metadata to the workload running on the node." - type = string -} - -variable "master_authorized_networks_config_cidr_blocks" { - description = "The list of CIDR blocks of external networks that can access the Kubernetes cluster master through HTTPS. Setting cidr_blocks to an empty list disallows public access." - type = list(string) -} - -variable "enable_intranode_visibility" { - description = "Whether to enable GKE intranode visibility." - type = bool -} - -variable "enable_tpu" { - description = "Whether to enable GKE cloud TPU support." - type = bool -} - -variable "cluster_database_encryption_key_name" { - type = string - description = "Cloud KMS key name for enabling cluster database encryption." -} - -variable "router_advertise_config" { - description = "Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions." - type = object({ - groups = list(string) - ip_ranges = map(string) - mode = string - }) -} - -variable "router_asn" { - description = "Router ASN used for auto-created router." - type = number -} - -variable "logging_config_enable_components" { - description = "Logging config components to enable." - type = list(string) -} - -variable "monitoring_config_enable_components" { - description = "Monitoring config components to enable." - type = list(string) -} - -variable "enable_gcs_fuse_csi_driver" { - description = "Whether to enable GCSFuse CSI driver addon." - type = bool -} - -variable "guest_accelerator" { - type = object({ - type = string - count = number - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = optional(string) - max_shared_clients_per_gpu = optional(number) - })) - }) - description = "`guest_accelerator` block supports during node_group creation, useful to provision GPU-capable nodes. Default to `null` or `{}` which will disable GPUs." - default = null -} \ No newline at end of file diff --git a/google/_modules/gke/versions.tf b/google/_modules/gke/versions.tf deleted file mode 100644 index 3496e5a4..00000000 --- a/google/_modules/gke/versions.tf +++ /dev/null @@ -1,16 +0,0 @@ - -terraform { - required_providers { - google = { - # https://registry.terraform.io/providers/hashicorp/google/latest - source = "hashicorp/google" - version = ">= 4.76.0" - } - - kubernetes = { - source = "hashicorp/kubernetes" - } - } - - required_version = ">= 0.13" -} diff --git a/google/_modules/gke/cluster_role_binding.tf b/google/cluster/cluster_role_binding.tf similarity index 100% rename from google/_modules/gke/cluster_role_binding.tf rename to google/cluster/cluster_role_binding.tf diff --git a/google/cluster/configuration.tf b/google/cluster/configuration.tf deleted file mode 100644 index f33be750..00000000 --- a/google/cluster/configuration.tf +++ /dev/null @@ -1,104 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - project_id = local.cfg["project_id"] - - region = local.cfg["region"] - - deletion_protection = lookup(local.cfg, "deletion_protection", null) - - cluster_node_locations_lookup = lookup(local.cfg, "cluster_node_locations", "") - cluster_node_locations = split(",", local.cluster_node_locations_lookup) - - cluster_min_master_version = local.cfg["cluster_min_master_version"] - cluster_release_channel = lookup(local.cfg, "cluster_release_channel", "STABLE") - - cluster_daily_maintenance_window_start_time = lookup( - local.cfg, - "cluster_daily_maintenance_window_start_time", - "03:00", - ) - - cluster_maintenance_exclusion_start_time = lookup(local.cfg, "cluster_maintenance_exclusion_start_time", "") - cluster_maintenance_exclusion_end_time = lookup(local.cfg, "cluster_maintenance_exclusion_end_time", "") - cluster_maintenance_exclusion_name = lookup(local.cfg, "cluster_maintenance_exclusion_name", "") - cluster_maintenance_exclusion_scope = lookup(local.cfg, "cluster_maintenance_exclusion_scope", "") - - remove_default_node_pool = lookup(local.cfg, "remove_default_node_pool", true) - - cluster_initial_node_count = lookup(local.cfg, "cluster_initial_node_count", 1) - - cluster_min_node_count = lookup(local.cfg, "cluster_min_node_count", 1) - cluster_max_node_count = lookup(local.cfg, "cluster_max_node_count", 1) - cluster_node_location_policy = lookup(local.cfg, "cluster_node_location_policy", null) - - cluster_extra_oauth_scopes_lookup = lookup(local.cfg, "cluster_extra_oauth_scopes", "") - cluster_extra_oauth_scopes = split(",", local.cluster_extra_oauth_scopes_lookup) - - cluster_disk_size_gb = lookup(local.cfg, "cluster_disk_size_gb", 100) - - cluster_disk_type = lookup(local.cfg, "cluster_disk_type", "pd-standard") - - cluster_image_type = lookup(local.cfg, "cluster_image_type", null) - - cluster_machine_type = lookup(local.cfg, "cluster_machine_type", "") - - cluster_preemptible = lookup(local.cfg, "cluster_preemptible", false) - - cluster_auto_repair = lookup(local.cfg, "cluster_auto_repair", true) - - cluster_auto_upgrade = lookup(local.cfg, "cluster_auto_upgrade", true) - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - enable_private_nodes = lookup(local.cfg, "enable_private_nodes", true) - master_cidr_block = lookup(local.cfg, "master_cidr_block", "172.16.0.32/28") - - cluster_ipv4_cidr_block = lookup(local.cfg, "cluster_ipv4_cidr_block", null) - services_ipv4_cidr_block = lookup(local.cfg, "services_ipv4_cidr_block", null) - - cluster_database_encryption_key_name = lookup(local.cfg, "cluster_database_encryption_key_name", null) - - # by default include cloud_nat when private nodes are enabled - enable_cloud_nat = lookup(local.cfg, "enable_cloud_nat", local.enable_private_nodes) - cloud_nat_endpoint_independent_mapping = lookup(local.cfg, "cloud_nat_enable_endpoint_independent_mapping", null) - cloud_nat_min_ports_per_vm = lookup(local.cfg, "cloud_nat_min_ports_per_vm", null) - cloud_nat_ip_count = lookup(local.cfg, "cloud_nat_ip_count", 0) - - disable_workload_identity = lookup(local.cfg, "disable_workload_identity", false) - default_node_workload_metadata_config = tobool(local.disable_workload_identity) == false ? "GKE_METADATA" : "MODE_UNSPECIFIED" - node_workload_metadata_config = lookup(local.cfg, "node_workload_metadata_config", local.default_node_workload_metadata_config) - - master_authorized_networks_config_cidr_blocks_lookup = lookup(local.cfg, "master_authorized_networks_config_cidr_blocks", null) - master_authorized_networks_config_cidr_blocks = local.master_authorized_networks_config_cidr_blocks_lookup == null ? null : split(",", local.master_authorized_networks_config_cidr_blocks_lookup) - - enable_intranode_visibility = lookup(local.cfg, "enable_intranode_visibility", false) - enable_tpu = lookup(local.cfg, "enable_tpu", false) - - router_advertise_config_groups_lookup = lookup(local.cfg, "router_advertise_config_groups", "") - router_advertise_config_groups = compact(split(",", local.router_advertise_config_groups_lookup)) - router_advertise_config_ip_ranges_lookup = lookup(local.cfg, "router_advertise_config_ip_ranges", "") - router_advertise_config_ip_ranges = compact(split(",", local.router_advertise_config_ip_ranges_lookup)) - router_advertise_config_mode = lookup(local.cfg, "router_advertise_config_mode", null) - router_asn = lookup(local.cfg, "router_asn", null) - - logging_config_enable_components_lookup = lookup(local.cfg, "logging_config_enable_components", "SYSTEM_COMPONENTS,WORKLOADS") - logging_config_enable_components = compact(split(",", local.logging_config_enable_components_lookup)) - - monitoring_config_enable_components_lookup = lookup(local.cfg, "monitoring_config_enable_components", "SYSTEM_COMPONENTS") - monitoring_config_enable_components = compact(split(",", local.monitoring_config_enable_components_lookup)) - - enable_gcs_fuse_csi_driver = lookup(local.cfg, "enable_gcs_fuse_csi_driver", null) -} diff --git a/google/_modules/gke/ingress.tf b/google/cluster/ingress.tf similarity index 52% rename from google/_modules/gke/ingress.tf rename to google/cluster/ingress.tf index f0c6f1fd..27432489 100644 --- a/google/_modules/gke/ingress.tf +++ b/google/cluster/ingress.tf @@ -1,25 +1,25 @@ resource "google_compute_address" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 region = google_container_cluster.current.location - project = var.project + project = local.cfg.project_id - name = var.metadata_name + name = module.cluster_metadata.name } resource "google_dns_managed_zone" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - project = var.project + project = local.cfg.project_id - name = var.metadata_name - dns_name = "${var.metadata_fqdn}." + name = module.cluster_metadata.name + dns_name = "${module.cluster_metadata.fqdn}." } resource "google_dns_record_set" "host" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - project = var.project + project = local.cfg.project_id name = google_dns_managed_zone.current[0].dns_name type = "A" @@ -31,9 +31,9 @@ resource "google_dns_record_set" "host" { } resource "google_dns_record_set" "wildcard" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - project = var.project + project = local.cfg.project_id name = "*.${google_dns_managed_zone.current[0].dns_name}" type = "A" diff --git a/google/_modules/gke/kubeconfig.tf b/google/cluster/kubeconfig.tf similarity index 100% rename from google/_modules/gke/kubeconfig.tf rename to google/cluster/kubeconfig.tf diff --git a/google/cluster/main.tf b/google/cluster/main.tf index 8c39748b..8cc49879 100644 --- a/google/cluster/main.tf +++ b/google/cluster/main.tf @@ -1,91 +1,151 @@ +module "configuration" { + source = "../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] +} + module "cluster_metadata" { source = "../../common/metadata" - name_prefix = local.name_prefix - base_domain = local.base_domain + name_prefix = local.cfg.name_prefix + base_domain = local.cfg.base_domain provider_name = "gcp" - provider_region = local.region + provider_region = local.cfg.region } -module "cluster" { - source = "../_modules/gke" - - project = local.project_id +resource "google_container_cluster" "current" { + project = local.cfg.project_id + name = module.cluster_metadata.name - deletion_protection = local.deletion_protection + deletion_protection = local.cfg.deletion_protection - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_tags = module.cluster_metadata.tags - metadata_labels = module.cluster_metadata.labels + location = local.cfg.region + node_locations = try(coalesce(local.cfg.cluster_node_locations, null), []) - location = local.region - node_locations = local.cluster_node_locations + min_master_version = local.cfg.cluster_min_master_version - min_master_version = local.cluster_min_master_version - release_channel = local.cluster_release_channel + release_channel { + channel = try(coalesce(local.cfg.cluster_release_channel, null), "STABLE") + } - daily_maintenance_window_start_time = local.cluster_daily_maintenance_window_start_time + remove_default_node_pool = true + initial_node_count = 1 - maintenance_exclusion_start_time = local.cluster_maintenance_exclusion_start_time - maintenance_exclusion_end_time = local.cluster_maintenance_exclusion_end_time - maintenance_exclusion_name = local.cluster_maintenance_exclusion_name - maintenance_exclusion_scope = local.cluster_maintenance_exclusion_scope + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } - remove_default_node_pool = local.remove_default_node_pool + network = google_compute_network.current.self_link - initial_node_count = local.cluster_initial_node_count - min_node_count = local.cluster_min_node_count - max_node_count = local.cluster_max_node_count - location_policy = local.cluster_node_location_policy + dynamic "workload_identity_config" { + for_each = try(coalesce(local.cfg.disable_workload_identity, null), false) == false ? toset([1]) : toset([]) + content { + workload_pool = "${local.cfg.project_id}.svc.id.goog" + } + } - extra_oauth_scopes = local.cluster_extra_oauth_scopes + dynamic "database_encryption" { + for_each = local.cfg.cluster_database_encryption_key_name != null ? [1] : [] + content { + state = "ENCRYPTED" + key_name = local.cfg.cluster_database_encryption_key_name + } + } - disk_size_gb = local.cluster_disk_size_gb - disk_type = local.cluster_disk_type - image_type = local.cluster_image_type - machine_type = local.cluster_machine_type + # + # + # Addon config + addons_config { + http_load_balancing { + disabled = true + } + + horizontal_pod_autoscaling { + disabled = false + } + + network_policy_config { + disabled = false + } + + dynamic "gcs_fuse_csi_driver_config" { + for_each = local.cfg.enable_gcs_fuse_csi_driver != null ? [1] : [] + + content { + enabled = local.cfg.enable_gcs_fuse_csi_driver + } + } + } - preemptible = local.cluster_preemptible + network_policy { + enabled = true + } - auto_repair = local.cluster_auto_repair + maintenance_policy { + daily_maintenance_window { + start_time = try(coalesce(local.cfg.cluster_daily_maintenance_window_start_time, null), "03:00") + } - auto_upgrade = local.cluster_auto_upgrade + dynamic "maintenance_exclusion" { + for_each = try(local.cfg.cluster_maintenance_exclusion.start_time, "") != "" ? [1] : [] - disable_default_ingress = local.disable_default_ingress + content { + start_time = try(local.cfg.cluster_maintenance_exclusion.start_time, "") + end_time = try(local.cfg.cluster_maintenance_exclusion.end_time, "") + exclusion_name = try(local.cfg.cluster_maintenance_exclusion.name, "") - enable_private_nodes = local.enable_private_nodes - master_cidr_block = local.master_cidr_block + exclusion_options { + scope = try(local.cfg.cluster_maintenance_exclusion.scope, "") + } + } + } + } - cluster_ipv4_cidr_block = local.cluster_ipv4_cidr_block - services_ipv4_cidr_block = local.services_ipv4_cidr_block + dynamic "master_authorized_networks_config" { + for_each = local.cfg.master_authorized_networks_config_cidr_blocks == null ? [] : [1] - enable_cloud_nat = local.enable_cloud_nat - cloud_nat_endpoint_independent_mapping = local.cloud_nat_endpoint_independent_mapping - cloud_nat_ip_count = local.cloud_nat_ip_count + content { + dynamic "cidr_blocks" { + for_each = local.cfg.master_authorized_networks_config_cidr_blocks - master_authorized_networks_config_cidr_blocks = local.master_authorized_networks_config_cidr_blocks + content { + cidr_block = cidr_blocks.value + display_name = "terraform-kubestack_${cidr_blocks.value}" + } + } + } + } - cloud_nat_min_ports_per_vm = local.cloud_nat_min_ports_per_vm + logging_config { + enable_components = try(coalesce(local.cfg.logging_config.enable_components, null), ["SYSTEM_COMPONENTS", "WORKLOADS"]) + } - disable_workload_identity = local.disable_workload_identity - node_workload_metadata_config = local.node_workload_metadata_config + monitoring_config { + enable_components = try(coalesce(local.cfg.monitoring_config.enable_components, null), ["SYSTEM_COMPONENTS"]) + } - cluster_database_encryption_key_name = local.cluster_database_encryption_key_name + private_cluster_config { + enable_private_nodes = try(coalesce(local.cfg.enable_private_nodes, null), true) + enable_private_endpoint = try(coalesce(local.cfg.enable_private_endpoint, null), false) + master_ipv4_cidr_block = try(coalesce(local.cfg.master_cidr_block, null), "172.16.0.32/28") + } - enable_intranode_visibility = local.enable_intranode_visibility - enable_tpu = local.enable_tpu + dynamic "ip_allocation_policy" { + for_each = try(coalesce(local.cfg.enable_private_nodes, null), true) ? [1] : [] - router_advertise_config = { - groups = local.router_advertise_config_groups - ip_ranges = { for ip in local.router_advertise_config_ip_ranges : ip => null } - mode = local.router_advertise_config_mode + content { + cluster_ipv4_cidr_block = local.cfg.cluster_ipv4_cidr_block + services_ipv4_cidr_block = local.cfg.services_ipv4_cidr_block + } } - router_asn = local.router_asn - - logging_config_enable_components = local.logging_config_enable_components - monitoring_config_enable_components = local.monitoring_config_enable_components - enable_gcs_fuse_csi_driver = local.enable_gcs_fuse_csi_driver + enable_intranode_visibility = try(coalesce(local.cfg.enable_intranode_visibility, null), false) + enable_tpu = try(coalesce(local.cfg.enable_tpu, null), false) } diff --git a/google/cluster/moved.tf b/google/cluster/moved.tf new file mode 100644 index 00000000..4ec91436 --- /dev/null +++ b/google/cluster/moved.tf @@ -0,0 +1,69 @@ +moved { + from = module.cluster.google_container_cluster.current + to = google_container_cluster.current +} + +moved { + from = module.cluster.google_compute_network.current + to = google_compute_network.current +} + +moved { + from = module.cluster.google_compute_address.nat + to = google_compute_address.nat +} + +moved { + from = module.cluster.google_compute_router.current + to = google_compute_router.current +} + +moved { + from = module.cluster.google_compute_router_nat.nat + to = google_compute_router_nat.nat +} + +moved { + from = module.cluster.google_service_account.current + to = google_service_account.current +} + +moved { + from = module.cluster.google_project_iam_member.log_writer + to = google_project_iam_member.log_writer +} + +moved { + from = module.cluster.google_project_iam_member.metric_writer + to = google_project_iam_member.metric_writer +} + +moved { + from = module.cluster.google_compute_address.current + to = google_compute_address.current +} + +moved { + from = module.cluster.google_dns_managed_zone.current + to = google_dns_managed_zone.current +} + +moved { + from = module.cluster.google_dns_record_set.host + to = google_dns_record_set.host +} + +moved { + from = module.cluster.google_dns_record_set.wildcard + to = google_dns_record_set.wildcard +} + +moved { + from = module.cluster.kubernetes_cluster_role_binding.current + to = kubernetes_cluster_role_binding.current +} + +moved { + from = module.cluster.module.node_pool + to = module.node_pool +} diff --git a/google/cluster/network.tf b/google/cluster/network.tf new file mode 100644 index 00000000..cb64d802 --- /dev/null +++ b/google/cluster/network.tf @@ -0,0 +1,78 @@ +resource "google_compute_network" "current" { + name = module.cluster_metadata.name + project = local.cfg.project_id + auto_create_subnetworks = true +} + +resource "google_compute_address" "nat" { + count = try(coalesce(local.cfg.enable_cloud_nat, null), true) ? try(coalesce(local.cfg.cloud_nat_ip_count, null), 0) : 0 + + region = google_container_cluster.current.location + project = local.cfg.project_id + + name = "nat-${module.cluster_metadata.name}-${count.index}" +} + +resource "google_compute_router" "current" { + count = try(coalesce(local.cfg.enable_cloud_nat, null), true) ? 1 : 0 + + project = local.cfg.project_id + name = module.cluster_metadata.name + region = google_container_cluster.current.location + + network = google_compute_network.current.name + + bgp { + advertise_mode = ( + local.cfg.router_advertise_config == null + ? null + : local.cfg.router_advertise_config.mode + ) + advertised_groups = ( + local.cfg.router_advertise_config == null ? null : ( + local.cfg.router_advertise_config.mode != "CUSTOM" + ? null + : try(coalesce(local.cfg.router_advertise_config.groups, null), toset([])) + ) + ) + dynamic "advertised_ip_ranges" { + for_each = ( + local.cfg.router_advertise_config == null ? [] : ( + local.cfg.router_advertise_config.mode != "CUSTOM" + ? [] + : try(coalesce(local.cfg.router_advertise_config.ip_ranges, null), tomap({})) + ) + ) + iterator = range + content { + range = range.key + description = range.value + } + } + + # expected "bgp.0.asn" to be a RFC6996-compliant Local ASN: + # must be either in the private ASN ranges: [64512..65534], [4200000000..4294967294]; + # or be the value of [16550] + asn = local.cfg.router_asn != null ? local.cfg.router_asn : 16550 + } +} + +resource "google_compute_router_nat" "nat" { + count = try(coalesce(local.cfg.enable_cloud_nat, null), true) ? 1 : 0 + + project = local.cfg.project_id + name = module.cluster_metadata.name + region = google_compute_router.current[0].region + router = google_compute_router.current[0].name + + enable_endpoint_independent_mapping = local.cfg.cloud_nat_enable_endpoint_independent_mapping + min_ports_per_vm = local.cfg.cloud_nat_min_ports_per_vm + nat_ip_allocate_option = try(coalesce(local.cfg.cloud_nat_ip_count, null), 0) > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + nat_ips = try(coalesce(local.cfg.cloud_nat_ip_count, null), 0) > 0 ? google_compute_address.nat[*].self_link : null + + log_config { + enable = true + filter = "ERRORS_ONLY" + } +} diff --git a/google/cluster/node-pool/configuration.tf b/google/cluster/node-pool/configuration.tf deleted file mode 100644 index 2b4f843a..00000000 --- a/google/cluster/node-pool/configuration.tf +++ /dev/null @@ -1,49 +0,0 @@ -module "configuration" { - source = "../../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - project_id = local.cfg["project_id"] - - name = lookup(local.cfg, "name") - - location = local.cfg["location"] - node_locations = local.cfg["node_locations"] - - initial_node_count = local.cfg["initial_node_count"] - min_node_count = local.cfg["min_node_count"] - max_node_count = local.cfg["max_node_count"] - location_policy = local.cfg["location_policy"] != null ? local.cfg["location_policy"] : "BALANCED" - - disk_size_gb = local.cfg["disk_size_gb"] - disk_type = local.cfg["disk_type"] - image_type = local.cfg["image_type"] - machine_type = local.cfg["machine_type"] - - preemptible = local.cfg["preemptible"] != null ? local.cfg["preemptible"] : false - auto_repair = local.cfg["auto_repair"] != null ? local.cfg["auto_repair"] : true - auto_upgrade = local.cfg["auto_upgrade"] != null ? local.cfg["auto_upgrade"] : true - - taints = local.cfg["taints"] - - labels = local.cfg["labels"] - - extra_oauth_scopes = local.cfg["extra_oauth_scopes"] != null ? local.cfg["extra_oauth_scopes"] : [] - - node_workload_metadata_config = local.cfg["node_workload_metadata_config"] != null ? local.cfg["node_workload_metadata_config"] : "GKE_METADATA" - - service_account_email = local.cfg["service_account_email"] - - ephemeral_storage_local_ssd_config = local.cfg["ephemeral_storage_local_ssd_config"] - - guest_accelerator = local.cfg["guest_accelerator"] - network_config = local.cfg["network_config"] - - instance_tags = local.cfg["instance_tags"] != null ? local.cfg["instance_tags"] : [] -} diff --git a/google/_modules/gke/node_pool/locals.tf b/google/cluster/node-pool/locals.tf similarity index 76% rename from google/_modules/gke/node_pool/locals.tf rename to google/cluster/node-pool/locals.tf index 88d70a0f..b3872f6a 100644 --- a/google/_modules/gke/node_pool/locals.tf +++ b/google/cluster/node-pool/locals.tf @@ -8,6 +8,5 @@ locals { "https://www.googleapis.com/auth/trace.append", ] - oauth_scopes = compact(concat(local.base_oauth_scopes, var.extra_oauth_scopes)) + oauth_scopes = compact(concat(local.base_oauth_scopes, try(coalesce(local.cfg.extra_oauth_scopes, null), []))) } - diff --git a/google/cluster/node-pool/main.tf b/google/cluster/node-pool/main.tf index c6cf0266..8e18cc53 100644 --- a/google/cluster/node-pool/main.tf +++ b/google/cluster/node-pool/main.tf @@ -1,46 +1,104 @@ -module "node_pool" { - source = "../../_modules/gke/node_pool" +module "configuration" { + source = "../../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] + + disable_per_node_pool_service_account = local.cfg.service_account_email == null ? false : true +} + +resource "google_container_node_pool" "current" { + name = local.cfg.name + project = local.cfg.project_id + cluster = var.cluster.name + location = local.cfg.location + + initial_node_count = local.cfg.initial_node_count + + autoscaling { + min_node_count = try(coalesce(local.cfg.min_node_count, null), 1) + max_node_count = try(coalesce(local.cfg.max_node_count, null), 1) + location_policy = try(coalesce(local.cfg.location_policy, null), "BALANCED") + } + + node_locations = try(coalesce(local.cfg.node_locations, null), []) + + dynamic "network_config" { + for_each = local.cfg.network_config == null ? [] : [1] + + content { + enable_private_nodes = local.cfg.network_config.enable_private_nodes + create_pod_range = local.cfg.network_config.create_pod_range + pod_ipv4_cidr_block = local.cfg.network_config.pod_ipv4_cidr_block + } + } + + # + # + # Node config + node_config { + service_account = local.disable_per_node_pool_service_account ? local.cfg.service_account_email : google_service_account.current[0].email - project = local.project_id + oauth_scopes = local.oauth_scopes - location = local.location - node_locations = local.node_locations + disk_size_gb = try(coalesce(local.cfg.disk_size_gb, null), 100) + disk_type = try(coalesce(local.cfg.disk_type, null), "pd-balanced") - cluster_name = var.cluster_metadata["name"] - pool_name = local.name + image_type = try(coalesce(local.cfg.image_type, null), "COS_CONTAINERD") + machine_type = try(coalesce(local.cfg.machine_type, null), "") + preemptible = try(coalesce(local.cfg.preemptible, null), false) - metadata_tags = var.cluster_metadata["tags"] - metadata_labels = var.cluster_metadata["labels"] + labels = merge(try(coalesce(local.cfg.labels, null), {}), var.cluster_metadata.labels) - initial_node_count = local.initial_node_count - min_node_count = local.min_node_count - max_node_count = local.max_node_count - location_policy = local.location_policy + tags = concat(var.cluster_metadata.tags, try(coalesce(local.cfg.instance_tags, null), [])) - extra_oauth_scopes = local.extra_oauth_scopes + workload_metadata_config { + mode = try(coalesce(local.cfg.node_workload_metadata_config, null), "GKE_METADATA") + } - disk_size_gb = local.disk_size_gb - disk_type = local.disk_type - image_type = local.image_type - machine_type = local.machine_type + dynamic "guest_accelerator" { + # Make sure to generate this only once + for_each = local.cfg.guest_accelerator == null ? [] : [1] - preemptible = local.preemptible - auto_repair = local.auto_repair - auto_upgrade = local.auto_upgrade + content { + type = local.cfg.guest_accelerator.type + count = local.cfg.guest_accelerator.count - node_workload_metadata_config = local.node_workload_metadata_config + dynamic "gpu_sharing_config" { + for_each = local.cfg.guest_accelerator.gpu_sharing_config == null ? [] : [1] - taints = local.taints - instance_tags = local.instance_tags + content { + gpu_sharing_strategy = local.cfg.guest_accelerator.gpu_sharing_config.gpu_sharing_strategy + max_shared_clients_per_gpu = local.cfg.guest_accelerator.gpu_sharing_config.max_shared_clients_per_gpu + } + } + } + } - labels = local.labels + dynamic "taint" { + for_each = try(coalesce(local.cfg.taints, null), toset([])) == null ? [] : try(coalesce(local.cfg.taints, null), toset([])) - service_account_email = local.service_account_email - disable_per_node_pool_service_account = local.service_account_email == null ? false : true + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } - ephemeral_storage_local_ssd_config = local.ephemeral_storage_local_ssd_config + dynamic "ephemeral_storage_local_ssd_config" { + for_each = local.cfg.ephemeral_storage_local_ssd_config == null ? [] : [1] - guest_accelerator = local.guest_accelerator + content { + local_ssd_count = local.cfg.ephemeral_storage_local_ssd_config.local_ssd_count + } + } + } - network_config = local.network_config + management { + auto_repair = try(coalesce(local.cfg.auto_repair, null), true) + auto_upgrade = try(coalesce(local.cfg.auto_upgrade, null), true) + } } diff --git a/google/cluster/node-pool/moved.tf b/google/cluster/node-pool/moved.tf new file mode 100644 index 00000000..77dd9de7 --- /dev/null +++ b/google/cluster/node-pool/moved.tf @@ -0,0 +1,19 @@ +moved { + from = module.node_pool.google_container_node_pool.current + to = google_container_node_pool.current +} + +moved { + from = module.node_pool.google_service_account.current + to = google_service_account.current +} + +moved { + from = module.node_pool.google_project_iam_member.log_writer + to = google_project_iam_member.log_writer +} + +moved { + from = module.node_pool.google_project_iam_member.metric_writer + to = google_project_iam_member.metric_writer +} diff --git a/google/_modules/gke/node_pool/outputs.tf b/google/cluster/node-pool/outputs.tf similarity index 69% rename from google/_modules/gke/node_pool/outputs.tf rename to google/cluster/node-pool/outputs.tf index 10dcb4a0..61201493 100644 --- a/google/_modules/gke/node_pool/outputs.tf +++ b/google/cluster/node-pool/outputs.tf @@ -1,3 +1,7 @@ +output "current_config" { + value = local.cfg +} + output "id" { value = google_container_node_pool.current.name description = "ID of the node pool." diff --git a/google/_modules/gke/node_pool/service_account.tf b/google/cluster/node-pool/service_account.tf similarity index 61% rename from google/_modules/gke/node_pool/service_account.tf rename to google/cluster/node-pool/service_account.tf index 5c413742..c4b97160 100644 --- a/google/_modules/gke/node_pool/service_account.tf +++ b/google/cluster/node-pool/service_account.tf @@ -1,28 +1,28 @@ locals { - account_id_prefix = join("-", [var.pool_name, var.cluster_name]) + account_id_prefix = join("-", [local.cfg.name, var.cluster_metadata.name]) account_id_suffix = sha512(local.account_id_prefix) account_id = "${substr(local.account_id_prefix, 0, 24)}-${substr(local.account_id_suffix, 0, 5)}" } resource "google_service_account" "current" { - count = var.disable_per_node_pool_service_account ? 0 : 1 + count = local.disable_per_node_pool_service_account ? 0 : 1 account_id = local.account_id - project = var.project + project = local.cfg.project_id } resource "google_project_iam_member" "log_writer" { - count = var.disable_per_node_pool_service_account ? 0 : 1 + count = local.disable_per_node_pool_service_account ? 0 : 1 - project = var.project + project = local.cfg.project_id role = "roles/logging.logWriter" member = "serviceAccount:${google_service_account.current[0].email}" } resource "google_project_iam_member" "metric_writer" { - count = var.disable_per_node_pool_service_account ? 0 : 1 + count = local.disable_per_node_pool_service_account ? 0 : 1 - project = var.project + project = local.cfg.project_id role = "roles/monitoring.metricWriter" member = "serviceAccount:${google_service_account.current[0].email}" } diff --git a/google/cluster/node-pool/variables.tf b/google/cluster/node-pool/variables.tf index 190be1fd..50157750 100644 --- a/google/cluster/node-pool/variables.tf +++ b/google/cluster/node-pool/variables.tf @@ -8,11 +8,11 @@ variable "configuration" { node_locations = optional(list(string)) location_policy = optional(string) - initial_node_count = optional(string) - min_node_count = optional(string) - max_node_count = optional(string) + initial_node_count = optional(number) + min_node_count = optional(number) + max_node_count = optional(number) - disk_size_gb = optional(string) + disk_size_gb = optional(number) disk_type = optional(string) image_type = optional(string) machine_type = optional(string) @@ -59,15 +59,24 @@ variable "configuration" { })) description = "Map with per workspace cluster configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false +} + +variable "cluster" { + type = any + description = "The cluster output from the cluster module." + nullable = false } variable "cluster_metadata" { type = any description = "Metadata of the cluster to attach the node pool to." + nullable = false } diff --git a/google/cluster/node-pool/versions.tf b/google/cluster/node-pool/versions.tf index 3e8b8534..2ad954b3 100644 --- a/google/cluster/node-pool/versions.tf +++ b/google/cluster/node-pool/versions.tf @@ -1,6 +1,6 @@ terraform { required_providers { - aws = { + google = { source = "hashicorp/google" } } diff --git a/google/cluster/node_pool.tf b/google/cluster/node_pool.tf new file mode 100644 index 00000000..f4d5751e --- /dev/null +++ b/google/cluster/node_pool.tf @@ -0,0 +1,45 @@ +module "node_pool" { + source = "./node-pool" + + cluster = google_container_cluster.current + + configuration = { + (terraform.workspace) = { + project_id = local.cfg.project_id + location = google_container_cluster.current.location + node_locations = local.cfg.cluster_node_locations + name = "default" + + service_account_email = google_service_account.current.email + + initial_node_count = try(coalesce(local.cfg.default_node_pool.initial_node_count, null), 1) + min_node_count = try(coalesce(local.cfg.default_node_pool.min_node_count, null), 1) + max_node_count = try(coalesce(local.cfg.default_node_pool.max_node_count, null), 1) + location_policy = try(local.cfg.default_node_pool.node_location_policy, null) + + extra_oauth_scopes = try(coalesce(local.cfg.default_node_pool.extra_oauth_scopes, null), []) + + disk_size_gb = try(coalesce(local.cfg.default_node_pool.disk_size_gb, null), 100) + disk_type = try(coalesce(local.cfg.default_node_pool.disk_type, null), "pd-standard") + image_type = try(coalesce(local.cfg.default_node_pool.image_type, null), "COS_containerd") + machine_type = try(coalesce(local.cfg.default_node_pool.machine_type, null), "") + + preemptible = try(coalesce(local.cfg.default_node_pool.preemptible, null), false) + auto_repair = try(coalesce(local.cfg.default_node_pool.auto_repair, null), true) + auto_upgrade = try(coalesce(local.cfg.default_node_pool.auto_upgrade, null), true) + + node_workload_metadata_config = try(coalesce(local.cfg.node_workload_metadata_config, null), "GKE_METADATA") + + taints = try(coalesce(local.cfg.default_node_pool.taints, null), toset([])) + instance_tags = try(coalesce(local.cfg.default_node_pool.instance_tags, null), []) + labels = try(coalesce(local.cfg.default_node_pool.labels, null), {}) + + ephemeral_storage_local_ssd_config = try(local.cfg.default_node_pool.ephemeral_storage_local_ssd_config, null) + network_config = try(local.cfg.default_node_pool.network_config, null) + guest_accelerator = null + } + } + configuration_base_key = terraform.workspace + + cluster_metadata = module.cluster_metadata +} diff --git a/google/cluster/outputs.tf b/google/cluster/outputs.tf index 589e30b8..461e37e3 100644 --- a/google/cluster/outputs.tf +++ b/google/cluster/outputs.tf @@ -1,5 +1,9 @@ +output "cluster" { + value = google_container_cluster.current +} + output "current_config" { - value = module.configuration.merged[terraform.workspace] + value = local.cfg } output "current_metadata" { @@ -7,9 +11,13 @@ output "current_metadata" { } output "kubeconfig" { - value = module.cluster.kubeconfig + value = local.kubeconfig + + # when the node pool is destroyed before the k8s namespaces + # the namespaces get stuck in terminating + depends_on = [module.node_pool] } output "default_ingress_ip" { - value = module.cluster.default_ingress_ip + value = length(google_compute_address.current) > 0 ? google_compute_address.current[0].address : null } diff --git a/google/_modules/gke/provider.tf b/google/cluster/provider.tf similarity index 100% rename from google/_modules/gke/provider.tf rename to google/cluster/provider.tf diff --git a/google/_modules/gke/service_account.tf b/google/cluster/service_account.tf similarity index 70% rename from google/_modules/gke/service_account.tf rename to google/cluster/service_account.tf index 1ba41b3e..47a08a1b 100644 --- a/google/_modules/gke/service_account.tf +++ b/google/cluster/service_account.tf @@ -1,17 +1,16 @@ resource "google_service_account" "current" { - account_id = substr(var.metadata_name, 0, 30) - project = var.project + account_id = substr(module.cluster_metadata.name, 0, 30) + project = local.cfg.project_id } resource "google_project_iam_member" "log_writer" { - project = var.project + project = local.cfg.project_id role = "roles/logging.logWriter" member = "serviceAccount:${google_service_account.current.email}" } resource "google_project_iam_member" "metric_writer" { - project = var.project + project = local.cfg.project_id role = "roles/monitoring.metricWriter" member = "serviceAccount:${google_service_account.current.email}" } - diff --git a/google/_modules/gke/templates/kubeconfig.tpl b/google/cluster/templates/kubeconfig.tpl similarity index 100% rename from google/_modules/gke/templates/kubeconfig.tpl rename to google/cluster/templates/kubeconfig.tpl diff --git a/google/cluster/variables.tf b/google/cluster/variables.tf index 8bf95161..051282b1 100644 --- a/google/cluster/variables.tf +++ b/google/cluster/variables.tf @@ -1,10 +1,108 @@ variable "configuration" { - type = map(map(string)) + type = map(object({ + # Required attributes + name_prefix = optional(string) + base_domain = optional(string) + project_id = optional(string) + region = optional(string) + cluster_min_master_version = optional(string) + + # Optional attributes without defaults + deletion_protection = optional(bool) + + cluster_node_locations = optional(list(string)) + + cluster_release_channel = optional(string) + + cluster_daily_maintenance_window_start_time = optional(string) + + cluster_maintenance_exclusion = optional(object({ + start_time = optional(string) + end_time = optional(string) + name = optional(string) + scope = optional(string) + })) + + default_node_pool = optional(object({ + remove_default_node_pool = optional(bool) + initial_node_count = optional(number) + min_node_count = optional(number) + max_node_count = optional(number) + node_location_policy = optional(string) + extra_oauth_scopes = optional(list(string)) + disk_size_gb = optional(number) + disk_type = optional(string) + image_type = optional(string) + machine_type = optional(string) + preemptible = optional(bool) + auto_repair = optional(bool) + auto_upgrade = optional(bool) + taints = optional(set(object({ + key = string + value = string + effect = string + }))) + labels = optional(map(string)) + instance_tags = optional(list(string)) + ephemeral_storage_local_ssd_config = optional(object({ + local_ssd_count = number + })) + network_config = optional(object({ + enable_private_nodes = bool + create_pod_range = bool + pod_ipv4_cidr_block = string + })) + })) + + disable_default_ingress = optional(bool) + + enable_private_nodes = optional(bool) + enable_private_endpoint = optional(bool) + master_cidr_block = optional(string) + + cluster_ipv4_cidr_block = optional(string) + services_ipv4_cidr_block = optional(string) + + cluster_database_encryption_key_name = optional(string) + + enable_cloud_nat = optional(bool) + cloud_nat_enable_endpoint_independent_mapping = optional(bool) + cloud_nat_min_ports_per_vm = optional(number) + cloud_nat_ip_count = optional(number) + + disable_workload_identity = optional(bool) + node_workload_metadata_config = optional(string) + + master_authorized_networks_config_cidr_blocks = optional(set(string)) + + enable_intranode_visibility = optional(bool) + enable_tpu = optional(bool) + + router_advertise_config = optional(object({ + groups = optional(set(string)) + ip_ranges = optional(map(string)) + mode = optional(string) + })) + router_asn = optional(number) + + logging_config = optional(object({ + enable_components = optional(list(string)) + })) + + monitoring_config = optional(object({ + enable_components = optional(list(string)) + })) + + enable_gcs_fuse_csi_driver = optional(bool) + })) + description = "Map with per workspace cluster configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } diff --git a/quickstart/src/configurations/aks/aks_zero_cluster.tf b/quickstart/src/configurations/aks/aks_zero_cluster.tf index cf71c7be..d39b87d4 100644 --- a/quickstart/src/configurations/aks/aks_zero_cluster.tf +++ b/quickstart/src/configurations/aks/aks_zero_cluster.tf @@ -21,9 +21,9 @@ module "aks_zero" { # CNI/Advanced networking configuration parameters. # Leave commented for default 'kubenet' networking - # vnet_address_space = "10.16.0.0/12" # accepts multiple comma-separated values - # subnet_address_prefixes = "10.18.0.0/16" # accepts multiple comma-separated values - # subnet_service_endpoints = null # accepts multiple comma-separated values + # vnet_address_space = ["10.16.0.0/12"] + # subnet_address_prefixes = ["10.18.0.0/16"] + # subnet_service_endpoints = ["Microsoft.Storage", "Microsoft.Sql"] # network_plugin = "azure" # network_policy = "azure" diff --git a/quickstart/src/configurations/eks/eks_zero_cluster.tf b/quickstart/src/configurations/eks/eks_zero_cluster.tf index 3a4e418c..40006d95 100644 --- a/quickstart/src/configurations/eks/eks_zero_cluster.tf +++ b/quickstart/src/configurations/eks/eks_zero_cluster.tf @@ -21,17 +21,17 @@ module "eks_zero" { # e.g. kbst-apps-eu-west-1.aws.infra.example.com base_domain = "" - cluster_instance_type = "t3.small" - cluster_desired_capacity = "1" - cluster_min_size = "1" - cluster_max_size = "3" + default_node_pool = { + instance_types = ["t3.small"] + desired_capacity = 1 + min_size = 1 + max_size = 1 + } - # Comma-separated list of zone names to deploy worker nodes in # EKS requires a min. of 2 zones # Must match region set in provider - # e.g. cluster_availability_zones = "eu-west-1a,eu-west-1b,eu-west-1c" - # FIXME: Use actual list when TF 0.12 finally supports heterogeneous maps - cluster_availability_zones = "" + # e.g. cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] + cluster_availability_zones = [] } # ops environment, inherits from apps diff --git a/quickstart/src/configurations/gke/gke_zero_cluster.tf b/quickstart/src/configurations/gke/gke_zero_cluster.tf index 60f7deae..90a70206 100644 --- a/quickstart/src/configurations/gke/gke_zero_cluster.tf +++ b/quickstart/src/configurations/gke/gke_zero_cluster.tf @@ -27,16 +27,17 @@ module "gke_zero" { cluster_min_master_version = "1.22" # Initial number of desired nodes per zone - cluster_initial_node_count = 1 + default_node_pool = { + initial_node_count = 1 + } # The Google cloud region to deploy the clusters in region = "" # Comma-separated list of zone names to deploy worker nodes in. # Must match region above. - # e.g. cluster_node_locations = "europe-west3-a,europe-west3-b,europe-west3-c" - # FIXME: Use actual list when TF 0.12 finally supports heterogeneous maps - cluster_node_locations = "" + # e.g. cluster_node_locations = ["europe-west3-a", "europe-west3-b", "europe-west3-c"] + cluster_node_locations = [] } # ops environment, inherits from apps diff --git a/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf b/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf index 48f42474..3f1ea76d 100644 --- a/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf +++ b/quickstart/src/configurations/multi-cloud/eks_zero_cluster.tf @@ -21,17 +21,17 @@ module "eks_zero" { # e.g. kbst-apps-eu-west-1.aws.infra.example.com base_domain = "" - cluster_instance_type = "t3.small" - cluster_desired_capacity = "1" - cluster_min_size = "1" - cluster_max_size = "3" + default_node_pool = { + instance_types = ["t3.small"] + desired_capacity = 1 + min_size = 1 + max_size = 1 + } - # Comma-separated list of zone names to deploy worker nodes in # EKS requires a min. of 2 zones # Must match region set in provider - # e.g. cluster_availability_zones = "eu-west-1a,eu-west-1b,eu-west-1c" - # FIXME: Use actual list when TF 0.12 finally supports heterogeneous maps - cluster_availability_zones = "" + # e.g. cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] + cluster_availability_zones = [] } # Settings for Ops-cluster @@ -39,8 +39,8 @@ module "eks_zero" { # Overwrite apps["cluster_availability_zones"] to have a smaller # ops cluster # EKS requires a min. of 2 zones - # e.g. cluster_availability_zones = "eu-west-1a,eu-west-1b" - cluster_availability_zones = "" + # e.g. cluster_availability_zones = ["eu-west-1a", "eu-west-1b"] + cluster_availability_zones = [] } } } diff --git a/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf b/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf index 96f973aa..a1e24bd5 100644 --- a/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf +++ b/quickstart/src/configurations/multi-cloud/gke_zero_cluster.tf @@ -27,16 +27,17 @@ module "gke_zero" { cluster_min_master_version = "1.22" # Initial number of desired nodes per zone - cluster_initial_node_count = 1 + default_node_pool = { + initial_node_count = 1 + } # The Google cloud region to deploy the clusters in region = "" # Comma-separated list of zone names to deploy worker nodes in. # Must match region above. - # e.g. cluster_node_locations = "europe-west3-a,europe-west3-b,europe-west3-c" - # FIXME: Use actual list when TF 0.12 finally supports heterogeneous maps - cluster_node_locations = "" + # e.g. cluster_node_locations = ["europe-west3-a", "europe-west3-b", "europe-west3-c"] + cluster_node_locations = [] } # Settings for Ops-cluster @@ -44,8 +45,8 @@ module "gke_zero" { ops = { # Overwrite apps["cluster_node_locations"] to have a smaller # ops cluster - # e.g. cluster_node_locations = "europe-west3-a" - cluster_node_locations = "" + # e.g. cluster_node_locations = ["europe-west3-a"] + cluster_node_locations = [] } } } diff --git a/tests/aks_zero_cluster.tf b/tests/aks_zero_cluster.tf index 9abf1621..a4241521 100644 --- a/tests/aks_zero_cluster.tf +++ b/tests/aks_zero_cluster.tf @@ -8,9 +8,11 @@ module "aks_zero" { name_prefix = "kbstacctest" base_domain = "infra.serverwolken.de" - default_node_pool_vm_size = "Standard_B2s" - default_node_pool_min_count = 1 - default_node_pool_max_count = 1 + default_node_pool = { + vm_size = "Standard_B2s" + min_count = 1 + max_count = 1 + } network_plugin = "azure" diff --git a/tests/aks_zero_node_pools.tf b/tests/aks_zero_node_pools.tf index 1b629fea..e1db62e7 100644 --- a/tests/aks_zero_node_pools.tf +++ b/tests/aks_zero_node_pools.tf @@ -1,8 +1,7 @@ module "aks_zero_node_pool" { source = "../azurerm/cluster/node-pool" - cluster_name = module.aks_zero.current_metadata["name"] - resource_group = module.aks_zero.current_config["resource_group"] + cluster = module.aks_zero.cluster configuration = { # Settings for Apps-cluster diff --git a/tests/eks_zero_cluster.tf b/tests/eks_zero_cluster.tf index 60620ce2..fc34fbe7 100644 --- a/tests/eks_zero_cluster.tf +++ b/tests/eks_zero_cluster.tf @@ -9,13 +9,15 @@ module "eks_zero" { configuration = { # Settings for Apps-cluster apps = { - name_prefix = "kbstacctest" - base_domain = "infra.serverwolken.de" - cluster_instance_type = "t3a.medium" - cluster_desired_capacity = "1" - cluster_min_size = "1" - cluster_max_size = "1" - cluster_availability_zones = "eu-west-1a,eu-west-1b,eu-west-1c" + name_prefix = "kbstacctest" + base_domain = "infra.serverwolken.de" + default_node_pool = { + instance_types = ["t3a.medium"] + desired_capacity = 1 + min_size = 1 + max_size = 3 + } + cluster_availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] cluster_aws_auth_map_users = <