diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bd70e252..2053546a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,7 +54,7 @@ jobs: needs: [build-test-artifacts] strategy: matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] permissions: id-token: write # required for keyless signing @@ -117,7 +117,7 @@ jobs: strategy: fail-fast: false matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] steps: - name: "Download test-artifacts" @@ -173,7 +173,7 @@ jobs: sed -i 's/region = ""/region = "eu-west-1"/g' eks_zero_providers.tf || true # EKS: set cluster_availability_zones - sed -i 's/cluster_availability_zones = ""/cluster_availability_zones = "eu-west-1a,eu-west-1b"/g' eks_zero_cluster.tf || true + sed -i 's/cluster_availability_zones = \[\]/cluster_availability_zones = ["eu-west-1a", "eu-west-1b"]/g' eks_zero_cluster.tf || true # GKE: set project_id sed -i 's/project_id = ""/project_id = "terraform-kubestack-testing"/g' gke_zero_cluster.tf || true @@ -182,7 +182,7 @@ jobs: sed -i 's/region = ""/region = "europe-west1"/g' gke_zero_cluster.tf || true # GKE: set cluster_node_locations - sed -i 's/cluster_node_locations = ""/cluster_node_locations = "europe-west1-b,europe-west1-c,europe-west1-d"/g' gke_zero_cluster.tf || true + sed -i 's/cluster_node_locations = \[\]/cluster_node_locations = ["europe-west1-b", "europe-west1-c", "europe-west1-d"]/g' gke_zero_cluster.tf || true - name: "Terraform init" working-directory: ./kubestack-starter-${{ matrix.starter }} @@ -230,7 +230,7 @@ jobs: strategy: matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] steps: - name: "Download test-artifacts" @@ -277,7 +277,7 @@ jobs: strategy: matrix: - starter: ["multi-cloud", "aks", "eks", "gke", "kind"] + starter: ["multi-cloud", "aks", "eks", "gke"] steps: - name: "Download test-artifacts" diff --git a/aws/_modules/eks/ingress.tf b/aws/_modules/eks/ingress.tf deleted file mode 100644 index 917a9aa5..00000000 --- a/aws/_modules/eks/ingress.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "aws_route53_zone" "current" { - count = var.disable_default_ingress ? 0 : 1 - - name = "${var.metadata_fqdn}." -} diff --git a/aws/_modules/eks/main.tf b/aws/_modules/eks/main.tf deleted file mode 100644 index a2f87d9f..00000000 --- a/aws/_modules/eks/main.tf +++ /dev/null @@ -1,17 +0,0 @@ -# AWS refers to labels (key-value pairs) as tags. -# EKS also requires below specifc tag set. This is why we merge this one -# and our default labels from metadata here into eks_tags to use throughout -# the eks module. -locals { - eks_tags = { - "kubernetes.io/cluster/${var.metadata_name}" = "shared" - } - - eks_metadata_tags = merge(var.metadata_labels, local.eks_tags) -} - -data "aws_partition" "current" {} - -data "aws_eks_cluster_auth" "current" { - name = aws_eks_cluster.current.name -} diff --git a/aws/_modules/eks/master.tf b/aws/_modules/eks/master.tf deleted file mode 100644 index ed8f6b0d..00000000 --- a/aws/_modules/eks/master.tf +++ /dev/null @@ -1,39 +0,0 @@ -resource "aws_eks_cluster" "current" { - name = var.metadata_name - role_arn = aws_iam_role.master.arn - - vpc_config { - security_group_ids = [aws_security_group.masters.id] - subnet_ids = aws_subnet.current.*.id - endpoint_private_access = var.cluster_endpoint_private_access - endpoint_public_access = var.cluster_endpoint_public_access - public_access_cidrs = var.cluster_public_access_cidrs - } - - dynamic "kubernetes_network_config" { - for_each = var.cluster_service_cidr != null ? toset([1]) : toset([]) - content { - service_ipv4_cidr = var.cluster_service_cidr - } - } - - dynamic "encryption_config" { - for_each = var.cluster_encryption_key_arn != null ? toset([1]) : toset([]) - content { - resources = ["secrets"] - - provider { - key_arn = var.cluster_encryption_key_arn - } - } - } - - depends_on = [ - aws_iam_role_policy_attachment.master_cluster_policy, - aws_iam_role_policy_attachment.master_service_policy, - ] - - version = var.cluster_version - - enabled_cluster_log_types = var.enabled_cluster_log_types -} diff --git a/aws/_modules/eks/node_pool.tf b/aws/_modules/eks/node_pool.tf deleted file mode 100644 index 9d22f1ed..00000000 --- a/aws/_modules/eks/node_pool.tf +++ /dev/null @@ -1,37 +0,0 @@ -module "node_pool" { - source = "./node_pool" - - metadata_labels = var.metadata_labels - eks_metadata_tags = local.eks_metadata_tags - - cluster_name = aws_eks_cluster.current.name - node_group_name = "default" - - role_arn = aws_iam_role.node.arn - - subnet_ids = var.vpc_legacy_node_subnets ? aws_subnet.current.*.id : aws_subnet.node_pool.*.id - - instance_types = var.instance_types - desired_size = var.desired_capacity - max_size = var.max_size - min_size = var.min_size - - disk_size = var.root_device_volume_size - - metadata_options = var.metadata_options - - taints = var.taints - - ami_type = var.worker_ami_type - ami_release_version = var.worker_ami_release_version - - tags = var.additional_node_tags - - kubernetes_version = aws_eks_cluster.current.version - - # force node_pool to depend on aws-auth configmap - depends-on-aws-auth = { - name = kubernetes_config_map.current.metadata[0].name - namespace = kubernetes_config_map.current.metadata[0].namespace - } -} diff --git a/aws/_modules/eks/node_pool/launch_template.tf b/aws/_modules/eks/node_pool/launch_template.tf deleted file mode 100644 index 0fbd5ffd..00000000 --- a/aws/_modules/eks/node_pool/launch_template.tf +++ /dev/null @@ -1,63 +0,0 @@ -locals { - create_launch_template = var.metadata_options != null ? true : false - - launch_template_name_parts = join("-", [var.cluster_name, var.node_group_name]) - launch_template_name_hash = sha256(local.launch_template_name_parts) - launch_template_name = "${substr(local.launch_template_name_parts, 0, 120)}-${substr(local.launch_template_name_hash, 0, 7)}" - - cpu_ami_name = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "amazon-linux-2023/arm64/standard" : "amazon-linux-2023/x86_64/standard" - is_gpu = length(data.aws_ec2_instance_type.current.gpus) > 0 - ami_name = local.is_gpu ? "amazon-linux-2023/x86_64/nvidia" : local.cpu_ami_name - ami_release_prefix = local.is_gpu ? "amazon-eks-gpu-node" : "amazon-eks-node" - ami_release_date = var.ami_release_version == null ? "" : split("-", var.ami_release_version)[1] - ami_release_name = var.ami_release_version == null ? "recommended" : "${local.ami_release_prefix}-${var.kubernetes_version}-v${local.ami_release_date}" -} - -data "aws_ssm_parameter" "eks_ami_release_version" { - count = var.disk_size != null ? 1 : 0 - - name = "/aws/service/eks/optimized-ami/${var.kubernetes_version}/${local.ami_name}/${local.ami_release_name}/image_id" -} - -data "aws_ami" "eks_optimized" { - count = var.disk_size != null ? 1 : 0 - - owners = ["amazon"] - - filter { - name = "image-id" - values = [nonsensitive(data.aws_ssm_parameter.eks_ami_release_version[0].value)] - } -} - -resource "aws_launch_template" "current" { - count = local.create_launch_template ? 1 : 0 - - name = local.launch_template_name - - tags = merge(var.tags, var.eks_metadata_tags) - - dynamic "block_device_mappings" { - for_each = var.disk_size != null ? toset([1]) : toset([]) - - content { - device_name = data.aws_ami.eks_optimized[0].root_device_name - - ebs { - volume_size = var.disk_size - } - } - } - - dynamic "metadata_options" { - for_each = var.metadata_options != null ? toset([1]) : toset([]) - - content { - http_endpoint = var.metadata_options.http_endpoint - http_tokens = var.metadata_options.http_tokens - http_put_response_hop_limit = var.metadata_options.http_put_response_hop_limit - http_protocol_ipv6 = var.metadata_options.http_protocol_ipv6 - instance_metadata_tags = var.metadata_options.instance_metadata_tags - } - } -} diff --git a/aws/_modules/eks/node_pool/main.tf b/aws/_modules/eks/node_pool/main.tf deleted file mode 100644 index 4b534c4a..00000000 --- a/aws/_modules/eks/node_pool/main.tf +++ /dev/null @@ -1,60 +0,0 @@ -data "aws_ec2_instance_type" "current" { - # ami_type is always determined by the first instance_type in the list - instance_type = element(tolist(var.instance_types), 0) -} - -locals { - cpu_ami_type = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "AL2023_ARM_64_STANDARD" : "AL2023_x86_64_STANDARD" - ami_type = length(data.aws_ec2_instance_type.current.gpus) > 0 ? "AL2023_x86_64_NVIDIA" : local.cpu_ami_type -} - -resource "aws_eks_node_group" "nodes" { - cluster_name = var.cluster_name - node_group_name = var.node_group_name - node_role_arn = var.role_arn - subnet_ids = var.subnet_ids - - scaling_config { - desired_size = var.desired_size - max_size = var.max_size - min_size = var.min_size - } - - version = var.kubernetes_version - ami_type = var.ami_type == null ? local.ami_type : var.ami_type - release_version = var.ami_release_version - instance_types = var.instance_types - - disk_size = local.create_launch_template ? null : var.disk_size - - dynamic "launch_template" { - for_each = local.create_launch_template ? toset([1]) : toset([]) - - content { - id = aws_launch_template.current[0].id - version = aws_launch_template.current[0].latest_version - } - } - - tags = merge(var.tags, var.eks_metadata_tags) - labels = merge(var.labels, var.metadata_labels) - - dynamic "taint" { - for_each = var.taints - - content { - key = taint.value["key"] - value = taint.value["value"] - effect = taint.value["effect"] - } - } - - depends_on = [var.depends-on-aws-auth] - - # when autoscaler is enabled, desired_size needs to be ignored - # better would be to handle this in the resource and not require - # desired_size, min_size and max_size in scaling_config - lifecycle { - ignore_changes = [scaling_config[0].desired_size] - } -} diff --git a/aws/_modules/eks/node_pool/variables.tf b/aws/_modules/eks/node_pool/variables.tf deleted file mode 100644 index 265d6c01..00000000 --- a/aws/_modules/eks/node_pool/variables.tf +++ /dev/null @@ -1,108 +0,0 @@ -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "eks_metadata_tags" { - type = map(any) - description = "EKS metadata tags to use." -} - -variable "cluster_name" { - type = string - description = "Cluster name of the EKS cluster." -} - -variable "node_group_name" { - type = string - description = "Name for this node pool." -} - -variable "role_arn" { - type = string - description = "ARN of the IAM role for worker nodes." -} - -variable "instance_types" { - type = set(string) - description = "Set of AWS instance types to use for nodes." -} - -variable "desired_size" { - type = string - description = "Desired number of worker nodes." -} - -variable "max_size" { - type = string - description = "Maximum number of worker nodes." -} - -variable "min_size" { - type = string - description = "Minimum number of worker nodes." -} - -variable "disk_size" { - type = string - default = "20" - description = "Will set the volume size of the root device" -} - -variable "subnet_ids" { - type = list(string) - description = "List of VPC subnet IDs to use for nodes." -} - -variable "depends-on-aws-auth" { - type = map(string) - description = "Used as a depends_on shim to first create the aws-auth configmap before creating the node_pool." -} - -variable "taints" { - type = set(object({ - key = string - value = string - effect = string - })) - description = "Kubernetes taints to set for node pool." -} - -variable "tags" { - type = map(string) - description = "AWS tags to set on the node pool. Merged with Kubestack default tags." - default = {} -} - -variable "labels" { - type = map(string) - description = "Kubernetes labels to set on the nodes created by the node pool. Merged with Kubestack default labels." - default = {} -} - -variable "ami_type" { - type = string - description = "AMI type to use for nodes of the node pool." -} - -variable "ami_release_version" { - type = string - description = "AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version" - default = null -} - -variable "kubernetes_version" { - type = string - description = "Kubernetes version to use for node pool." -} - -variable "metadata_options" { - description = "EC2 metadata service options." - type = object({ - http_endpoint = optional(string) - http_tokens = optional(string) - http_put_response_hop_limit = optional(number) - http_protocol_ipv6 = optional(string) - instance_metadata_tags = optional(string) - }) -} diff --git a/aws/_modules/eks/node_pool/versions.tf b/aws/_modules/eks/node_pool/versions.tf deleted file mode 100644 index ac97c6ac..00000000 --- a/aws/_modules/eks/node_pool/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/aws/_modules/eks/outputs.tf b/aws/_modules/eks/outputs.tf deleted file mode 100644 index 55b14881..00000000 --- a/aws/_modules/eks/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "kubeconfig" { - value = local.kubeconfig -} diff --git a/aws/_modules/eks/roles_master.tf b/aws/_modules/eks/roles_master.tf deleted file mode 100644 index dd76f32f..00000000 --- a/aws/_modules/eks/roles_master.tf +++ /dev/null @@ -1,30 +0,0 @@ -resource "aws_iam_role" "master" { - name = "${var.metadata_name}-master" - - assume_role_policy = < 0 ? 1 : 0 + count = var.cluster_default_node_pool_subnet_ids == null ? 1 : 0 filter { name = "vpc-id" - values = [data.aws_vpc.current.id] + values = [var.cluster.vpc_config[0].vpc_id] } # if the node pool is in one or more specific AZs # only link subnet_ids belonging to these AZs filter { name = "availability-zone" - values = local.availability_zones + values = coalesce(local.cfg.availability_zones, []) } # exclude control plane subnets filter { name = "subnet-id" - values = tolist(data.aws_eks_node_group.default.subnet_ids) + values = tolist(data.aws_eks_node_group.default[0].subnet_ids) } } + data "aws_internet_gateway" "current" { - count = local.vpc_subnet_newbits == null ? 0 : local.vpc_subnet_map_public_ip == false ? 0 : 1 + count = local.cfg.vpc_subnet_newbits == null ? 0 : local.cfg.vpc_subnet_map_public_ip == false ? 0 : 1 filter { name = "attachment.vpc-id" - values = [data.aws_vpc.current.id] + values = [var.cluster.vpc_config[0].vpc_id] } } data "aws_nat_gateway" "current" { - count = local.vpc_subnet_newbits == null ? 0 : local.vpc_subnet_map_public_ip == false ? length(local.availability_zones) : 0 + count = local.cfg.vpc_subnet_newbits == null ? 0 : local.cfg.vpc_subnet_map_public_ip == false ? length(lookup(local.cfg, "availability_zones", [])) : 0 - vpc_id = data.aws_vpc.current.id + vpc_id = var.cluster.vpc_config[0].vpc_id tags = { - "kubestack.com/cluster_name" = data.aws_eks_cluster.current.name - "kubestack.com/cluster_provider_zone" = local.availability_zones[count.index] + "kubestack.com/cluster_name" = var.cluster.name + "kubestack.com/cluster_provider_zone" = local.cfg.availability_zones[count.index] } } + +data "aws_ec2_instance_type" "current" { + # ami_type is always determined by the first instance_type in the list + # fall back to t3.medium when no instance_types are configured + instance_type = element(coalesce(local.cfg.instance_types, ["t3.medium"]), 0) +} diff --git a/aws/cluster/node-pool/launch_template.tf b/aws/cluster/node-pool/launch_template.tf new file mode 100644 index 00000000..db5a188f --- /dev/null +++ b/aws/cluster/node-pool/launch_template.tf @@ -0,0 +1,62 @@ +locals { + create_launch_template = local.cfg.metadata_options != null + launch_template_name_parts = join("-", [var.cluster.name, local.cfg.name]) + launch_template_name_hash = sha256(local.launch_template_name_parts) + launch_template_name = "${substr(local.launch_template_name_parts, 0, 120)}-${substr(local.launch_template_name_hash, 0, 7)}" + + cpu_ami_name = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "amazon-linux-2023/arm64/standard" : "amazon-linux-2023/x86_64/standard" + is_gpu = length(data.aws_ec2_instance_type.current.gpus) > 0 + ami_name = local.is_gpu ? "amazon-linux-2023/x86_64/nvidia" : local.cpu_ami_name + ami_release_prefix = local.is_gpu ? "amazon-eks-gpu-node" : "amazon-eks-node" + ami_release_date = local.cfg.ami_release_version == null ? "" : split("-", local.cfg.ami_release_version)[1] + ami_release_name = local.cfg.ami_release_version == null ? "recommended" : "${local.ami_release_prefix}-${var.cluster.version}-v${local.ami_release_date}" +} + +data "aws_ssm_parameter" "eks_ami_release_version" { + count = try(coalesce(local.cfg.disk_size, null), 20) != null ? 1 : 0 + + name = "/aws/service/eks/optimized-ami/${var.cluster.version}/${local.ami_name}/${local.ami_release_name}/image_id" +} + +data "aws_ami" "eks_optimized" { + count = try(coalesce(local.cfg.disk_size, null), 20) != null ? 1 : 0 + + owners = ["amazon"] + + filter { + name = "image-id" + values = [nonsensitive(data.aws_ssm_parameter.eks_ami_release_version[0].value)] + } +} + +resource "aws_launch_template" "current" { + count = local.create_launch_template ? 1 : 0 + + name = local.launch_template_name + + tags = coalesce(local.cfg.tags, {}) + + dynamic "block_device_mappings" { + for_each = try(coalesce(local.cfg.disk_size, null), 20) != null ? toset([1]) : toset([]) + + content { + device_name = data.aws_ami.eks_optimized[0].root_device_name + + ebs { + volume_size = try(coalesce(local.cfg.disk_size, null), 20) + } + } + } + + dynamic "metadata_options" { + for_each = local.cfg.metadata_options != null ? toset([1]) : toset([]) + + content { + http_endpoint = local.cfg.metadata_options.http_endpoint + http_tokens = local.cfg.metadata_options.http_tokens + http_put_response_hop_limit = local.cfg.metadata_options.http_put_response_hop_limit + http_protocol_ipv6 = local.cfg.metadata_options.http_protocol_ipv6 + instance_metadata_tags = local.cfg.metadata_options.instance_metadata_tags + } + } +} diff --git a/aws/cluster/node-pool/main.tf b/aws/cluster/node-pool/main.tf index 077fa1b3..55d9d333 100644 --- a/aws/cluster/node-pool/main.tf +++ b/aws/cluster/node-pool/main.tf @@ -1,44 +1,72 @@ -locals { - // if provider level tags are set, the node_group data source tags attr - // includes the resource level and provider level tags - // we have to exclude the provider level tags when setting them for node pools below - node_group_tag_keys = toset(keys(data.aws_eks_node_group.default.tags)) - provider_level_tag_keys = toset(keys(data.aws_default_tags.current.tags)) - tags_without_all_tags_keys = setsubtract(local.node_group_tag_keys, local.provider_level_tag_keys) - tags_without_all_tags = { for k in local.tags_without_all_tags_keys : k => data.aws_eks_node_group.default.tags[k] } +module "configuration" { + source = "../../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key } -module "node_pool" { - source = "../../_modules/eks/node_pool" - - cluster_name = data.aws_eks_cluster.current.name - metadata_labels = data.aws_eks_node_group.default.labels - eks_metadata_tags = local.tags_without_all_tags - role_arn = data.aws_eks_node_group.default.node_role_arn +locals { + cfg = module.configuration.merged[terraform.workspace] - node_group_name = local.name + cpu_ami_type = data.aws_ec2_instance_type.current.supported_architectures[0] == "arm64" ? "AL2023_ARM_64_STANDARD" : "AL2023_x86_64_STANDARD" + computed_ami_type = length(data.aws_ec2_instance_type.current.gpus) > 0 ? "AL2023_x86_64_NVIDIA" : local.cpu_ami_type +} - subnet_ids = local.vpc_subnet_newbits == null ? local.vpc_subnet_ids : aws_subnet.current.*.id +resource "aws_eks_node_group" "nodes" { + cluster_name = var.cluster.name + node_group_name = local.cfg.name + node_role_arn = data.aws_iam_role.node.arn + subnet_ids = coalesce( + # first prio, user specified subnet ids + try(coalesce(var.cluster_default_node_pool_subnet_ids, local.cfg.vpc_subnet_ids), null), + # second prio, created subnet ids + length(aws_subnet.current.*.id) > 0 ? aws_subnet.current.*.id : null, + # fall back, existing subnet ids + length(data.aws_subnets.current) == 1 ? data.aws_subnets.current[0].ids : null, + ) - instance_types = local.instance_types - ami_release_version = local.ami_release_version - desired_size = local.desired_capacity - max_size = local.max_size - min_size = local.min_size + scaling_config { + desired_size = local.cfg.desired_capacity + max_size = local.cfg.max_size + min_size = local.cfg.min_size + } - ami_type = local.ami_type + version = var.cluster.version + ami_type = local.cfg.ami_type == null ? local.computed_ami_type : local.cfg.ami_type + release_version = local.cfg.ami_release_version + instance_types = try(toset(local.cfg.instance_types), toset([])) - kubernetes_version = data.aws_eks_cluster.current.version + disk_size = local.create_launch_template ? null : try(coalesce(local.cfg.disk_size, null), 20) - disk_size = local.disk_size + dynamic "launch_template" { + for_each = local.create_launch_template ? toset([1]) : toset([]) - metadata_options = local.metadata_options + content { + id = aws_launch_template.current[0].id + version = aws_launch_template.current[0].latest_version + } + } - taints = local.taints + tags = merge( + { "kubernetes.io/cluster/${var.cluster.name}" = "shared" }, + var.cluster_metadata.labels, + coalesce(local.cfg.tags, {}) + ) + labels = merge(var.cluster_metadata.labels, coalesce(local.cfg.labels, {})) - tags = local.tags + dynamic "taint" { + for_each = coalesce(local.cfg.taints, []) - labels = local.labels + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } - depends-on-aws-auth = null + # when autoscaler is enabled, desired_size needs to be ignored + # better would be to handle this in the resource and not require + # desired_size, min_size and max_size in scaling_config + lifecycle { + ignore_changes = [scaling_config[0].desired_size] + } } diff --git a/aws/cluster/node-pool/moved.tf b/aws/cluster/node-pool/moved.tf new file mode 100644 index 00000000..44e99b61 --- /dev/null +++ b/aws/cluster/node-pool/moved.tf @@ -0,0 +1,9 @@ +moved { + from = module.node_pool.aws_eks_node_group.nodes + to = aws_eks_node_group.nodes +} + +moved { + from = module.node_pool.aws_launch_template.current + to = aws_launch_template.current +} diff --git a/aws/cluster/node-pool/outputs.tf b/aws/cluster/node-pool/outputs.tf new file mode 100644 index 00000000..871c9f88 --- /dev/null +++ b/aws/cluster/node-pool/outputs.tf @@ -0,0 +1,3 @@ +output "current_config" { + value = local.cfg +} diff --git a/aws/cluster/node-pool/variables.tf b/aws/cluster/node-pool/variables.tf index 82b0367d..5b0fcac6 100644 --- a/aws/cluster/node-pool/variables.tf +++ b/aws/cluster/node-pool/variables.tf @@ -3,12 +3,12 @@ variable "configuration" { name = optional(string) - instance_types = optional(string) + instance_types = optional(list(string)) ami_release_version = optional(string) - desired_capacity = optional(string) - min_size = optional(string) - max_size = optional(string) - disk_size = optional(string) + desired_capacity = optional(number) + min_size = optional(number) + max_size = optional(number) + disk_size = optional(number) ami_type = optional(string) @@ -20,14 +20,14 @@ variable "configuration" { instance_metadata_tags = optional(string) })) - availability_zones = optional(string) + availability_zones = optional(list(string)) - vpc_subnet_ids = optional(string) + vpc_subnet_ids = optional(list(string)) vpc_secondary_cidr = optional(string) - vpc_subnet_newbits = optional(string) - vpc_subnet_number_offset = optional(string) + vpc_subnet_newbits = optional(number) + vpc_subnet_number_offset = optional(number) vpc_subnet_map_public_ip = optional(bool) taints = optional(set(object({ @@ -41,20 +41,37 @@ variable "configuration" { labels = optional(map(string)) })) description = "Map with per workspace module configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } -variable "cluster_name" { - type = string - description = "Name of the cluster to attach the node pool to." +variable "cluster" { + type = any + description = "The cluster output from the cluster module." + nullable = false +} + +variable "cluster_metadata" { + type = any + description = "Metadata of the cluster to attach the node pool to." + nullable = false } + variable "cluster_default_node_pool_name" { type = string description = "Name of the cluster's default node pool to inherit IAM role and subnets from." default = "default" + nullable = false +} + +variable "cluster_default_node_pool_subnet_ids" { + type = list(string) + description = "Subnet IDs for the default node pool subnet." + default = null } diff --git a/aws/cluster/node-pool/versions.tf b/aws/cluster/node-pool/versions.tf index 07ac0e26..d64d1d92 100644 --- a/aws/cluster/node-pool/versions.tf +++ b/aws/cluster/node-pool/versions.tf @@ -3,6 +3,10 @@ terraform { aws = { source = "hashicorp/aws" } + + tls = { + source = "hashicorp/tls" + } } required_version = ">= 1.3.0" diff --git a/aws/cluster/node-pool/vpc.tf b/aws/cluster/node-pool/vpc.tf index 2219684a..f758288c 100644 --- a/aws/cluster/node-pool/vpc.tf +++ b/aws/cluster/node-pool/vpc.tf @@ -1,47 +1,51 @@ resource "aws_vpc_ipv4_cidr_block_association" "current" { - count = local.vpc_secondary_cidr == null ? 0 : 1 + count = local.cfg.vpc_secondary_cidr == null ? 0 : 1 - vpc_id = data.aws_vpc.current.id - cidr_block = local.vpc_secondary_cidr + vpc_id = var.cluster.vpc_config[0].vpc_id + cidr_block = local.cfg.vpc_secondary_cidr } locals { - subnet_cidr = length(aws_vpc_ipv4_cidr_block_association.current) == 1 ? aws_vpc_ipv4_cidr_block_association.current[0].cidr_block : data.aws_vpc.current.cidr_block + subnet_cidr = length(aws_vpc_ipv4_cidr_block_association.current) == 1 ? aws_vpc_ipv4_cidr_block_association.current[0].cidr_block : data.aws_vpc.cluster.cidr_block } resource "aws_subnet" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 - availability_zone = local.availability_zones[count.index] + availability_zone = local.cfg.availability_zones[count.index] cidr_block = cidrsubnet( local.subnet_cidr, - local.vpc_subnet_newbits, - local.vpc_subnet_number_offset + count.index + local.cfg.vpc_subnet_newbits, + (local.cfg.vpc_subnet_number_offset == null ? 1 : local.cfg.vpc_subnet_number_offset) + count.index ) - vpc_id = data.aws_vpc.current.id - map_public_ip_on_launch = local.vpc_subnet_map_public_ip == null ? true : local.vpc_subnet_map_public_ip + vpc_id = var.cluster.vpc_config[0].vpc_id + map_public_ip_on_launch = local.cfg.vpc_subnet_map_public_ip == null ? true : local.cfg.vpc_subnet_map_public_ip - tags = data.aws_eks_node_group.default.tags + tags = merge( + { "kubernetes.io/cluster/${var.cluster.name}" = "shared" }, + var.cluster_metadata.labels, + coalesce(local.cfg.tags, {}) + ) } resource "aws_route_table" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 - vpc_id = data.aws_vpc.current.id + vpc_id = var.cluster.vpc_config[0].vpc_id } resource "aws_route" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 route_table_id = aws_route_table.current[count.index].id - gateway_id = local.vpc_subnet_map_public_ip == false ? null : data.aws_internet_gateway.current[0].id - nat_gateway_id = local.vpc_subnet_map_public_ip == false ? data.aws_nat_gateway.current[count.index].id : null + gateway_id = local.cfg.vpc_subnet_map_public_ip == false ? null : data.aws_internet_gateway.current[0].id + nat_gateway_id = local.cfg.vpc_subnet_map_public_ip == false ? data.aws_nat_gateway.current[count.index].id : null destination_cidr_block = "0.0.0.0/0" } resource "aws_route_table_association" "current" { - count = local.vpc_subnet_newbits == null ? 0 : length(local.availability_zones) + count = local.cfg.vpc_subnet_newbits != null && local.cfg.availability_zones != null ? length(local.cfg.availability_zones) : 0 subnet_id = aws_subnet.current[count.index].id route_table_id = aws_route_table.current[count.index].id diff --git a/aws/_modules/eks/openid_connect.tf b/aws/cluster/openid_connect.tf similarity index 66% rename from aws/_modules/eks/openid_connect.tf rename to aws/cluster/openid_connect.tf index 06745c60..1dae6189 100644 --- a/aws/_modules/eks/openid_connect.tf +++ b/aws/cluster/openid_connect.tf @@ -1,11 +1,11 @@ data "tls_certificate" "current" { - count = var.disable_openid_connect_provider == false ? 1 : 0 + count = try(coalesce(local.cfg.disable_openid_connect_provider, null), false) == false ? 1 : 0 url = aws_eks_cluster.current.identity[0].oidc[0].issuer } resource "aws_iam_openid_connect_provider" "current" { - count = var.disable_openid_connect_provider == false ? 1 : 0 + count = try(coalesce(local.cfg.disable_openid_connect_provider, null), false) == false ? 1 : 0 client_id_list = ["sts.${data.aws_partition.current.dns_suffix}"] thumbprint_list = [data.tls_certificate.current[0].certificates[0].sha1_fingerprint] diff --git a/aws/cluster/outputs.tf b/aws/cluster/outputs.tf index 424f37e9..53125dc3 100644 --- a/aws/cluster/outputs.tf +++ b/aws/cluster/outputs.tf @@ -1,5 +1,9 @@ +output "cluster" { + value = aws_eks_cluster.current +} + output "current_config" { - value = module.configuration.merged[terraform.workspace] + value = local.cfg } output "current_metadata" { @@ -7,5 +11,5 @@ output "current_metadata" { } output "kubeconfig" { - value = module.cluster.kubeconfig + value = local.kubeconfig } diff --git a/aws/_modules/eks/roles_worker.tf b/aws/cluster/roles.tf similarity index 50% rename from aws/_modules/eks/roles_worker.tf rename to aws/cluster/roles.tf index f81fd16e..26d9ff41 100644 --- a/aws/_modules/eks/roles_worker.tf +++ b/aws/cluster/roles.tf @@ -1,5 +1,37 @@ +resource "aws_iam_role" "master" { + name = "${module.cluster_metadata.name}-master" + + assume_role_policy = < 0 ? azurerm_virtual_network.current[0] : null -} - -output "kubeconfig" { - sensitive = true - value = local.kubeconfig -} - -output "default_ingress_ip" { - value = length(azurerm_public_ip.current) > 0 ? azurerm_public_ip.current[0].ip_address : null -} diff --git a/azurerm/_modules/aks/service_principal.tf b/azurerm/_modules/aks/service_principal.tf deleted file mode 100644 index 4dec3756..00000000 --- a/azurerm/_modules/aks/service_principal.tf +++ /dev/null @@ -1,17 +0,0 @@ -resource "azuread_application" "current" { - count = var.disable_managed_identities == true ? 1 : 0 - - display_name = var.metadata_name -} - -resource "azuread_service_principal" "current" { - count = var.disable_managed_identities == true ? 1 : 0 - - client_id = azuread_application.current[0].client_id -} - -resource "azuread_service_principal_password" "current" { - count = var.disable_managed_identities == true ? 1 : 0 - - service_principal_id = azuread_service_principal.current[0].id -} diff --git a/azurerm/_modules/aks/variables.tf b/azurerm/_modules/aks/variables.tf deleted file mode 100644 index 2f9246ad..00000000 --- a/azurerm/_modules/aks/variables.tf +++ /dev/null @@ -1,217 +0,0 @@ -variable "metadata_name" { - type = string - description = "Metadata name to use." -} - -variable "metadata_fqdn" { - type = string - description = "DNS name of the zone. E.g. `infra.example.com`" -} - -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "metadata_label_namespace" { - type = string - description = "Prefix labels are namespaced with." -} - -variable "resource_group" { - type = string - description = "Resource group to use for the cluster." -} - -variable "legacy_vnet_name" { - type = bool - description = "Wheter to use the legacy vnet and subnet names." - default = false -} - -variable "vnet_address_space" { - type = list(string) - description = "Address space for the virtual network." -} - -variable "subnet_address_prefixes" { - type = list(string) - description = "Address space for the cluster subnet." -} - -variable "subnet_service_endpoints" { - type = list(string) - description = "List of service endpoints to expose on the subnet." -} - -variable "network_plugin" { - type = string - description = "Network plugin to use. Set to 'azure' for CNI; 'kubenet' for Basic" -} - -variable "network_policy" { - type = string - description = "Network policy to use. Set to 'azure' for CNI; 'calico' for Basic" -} - -variable "dns_prefix" { - type = string - description = "DNS prefix of AKS cluster API server." - default = "api" -} - -variable "service_cidr" { - type = string - description = "CIDR range for kubernetes service." -} - -variable "dns_service_ip" { - type = string - description = "IP address for the kube-dns service. Must be within service_cidr." -} - -variable "pod_cidr" { - type = string - description = "CIDR range for pods." -} - -variable "max_pods" { - type = number - description = "Maximum number of pods to run per node, if using CNI for networking." -} - -variable "default_node_pool_name" { - type = string - description = "Name of default node pool." - default = "default" -} - -variable "default_node_pool_type" { - type = string - description = "Type of default node pool. Defaults to VirtualMachineScaleSets, use AvailabilitySet for backwards compatibility with older clusters." - default = "VirtualMachineScaleSets" -} - -variable "default_node_pool_enable_auto_scaling" { - type = bool - description = "Wether to enable auto scaling for the default node pool. Defaults to true." - default = true -} - -variable "default_node_pool_min_count" { - type = string - description = "Min number of worker nodes" - default = "1" -} - -variable "default_node_pool_max_count" { - type = string - description = "Max number of worker nodes" - default = "1" -} - -variable "default_node_pool_node_count" { - type = string - description = "Static number of worker nodes" - default = "1" -} - -variable "default_node_pool_vm_size" { - type = string - description = "VM size of worker nodes" - default = "Standard_D1_v2" -} - -variable "default_node_pool_os_disk_size_gb" { - type = string - description = "Disk size of worker nodes (in GB)" - default = "30" -} - -variable "default_node_pool_only_critical_addons" { - type = bool - description = "Enabling will taint default node pool with CriticalAddonsOnly=true:NoSchedule taint. Changing this forces a new resource to be created." - default = false -} - -variable "disable_default_ingress" { - type = bool - description = "Whether to disable the default ingress." -} - -variable "default_ingress_ip_zones" { - type = list(string) - description = "Explicit list of zones for the ingress public IP." -} - -variable "enable_azure_policy_agent" { - type = bool - description = "whether to deploy the Azure policy agent to the cluster" - default = false -} - -variable "disable_managed_identities" { - type = bool - description = "Keep using legacy service principal instead of new managed identities." - default = false -} - -variable "user_assigned_identity_id" { - type = string - description = "ID of the UserAssigned identity to use." - default = null -} - -variable "sku_tier" { - type = string - description = "The SKU tier to use for the cluster. Possible values are 'Free' and 'Paid'. Defaults to 'Free'." - default = "Free" -} - -variable "kubernetes_version" { - type = string - description = "Version of Kubernetes to use when creating the cluster. If not specified, latest version will be used." -} - -variable "automatic_channel_upgrade" { - type = string - description = "The upgrade channel for this Kubernetes Cluster. Possible values are 'patch', 'rapid', 'node-image' and 'stable'." -} - -variable "enable_log_analytics" { - type = bool - description = "whether to deploy the Azure Log analytics with the cluster" - default = true -} - -variable "availability_zones" { - type = list(string) - description = "The list of availability zones to create the node pool in" - default = [] -} - - -variable "keda_enabled" { - type = bool - description = "Whether KEDA Autoscaler should be enabled for the cluster." -} - -variable "vertical_pod_autoscaler_enabled" { - type = bool - description = "Whether Vertical Pod Autoscaler should be enabled for the cluster." -} - -variable "upgade_settings_drain_timeout_in_minutes" { - type = number - description = "The amount of time in minutes to wait on eviction of pods and graceful termination per node." -} - -variable "upgade_settings_max_surge" { - type = string - description = "The maximum number or percentage of nodes which will be added to the Node Pool size during an upgrade." -} - -variable "upgade_settings_node_soak_duration_in_minutes" { - type = number - description = "The amount of time in minutes to wait after draining a node and before reimaging and moving on to next node." -} diff --git a/azurerm/_modules/aks/versions.tf b/azurerm/_modules/aks/versions.tf deleted file mode 100644 index 4bc72444..00000000 --- a/azurerm/_modules/aks/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ - -terraform { - required_providers { - azurerm = { - # https://registry.terraform.io/providers/hashicorp/azurerm/latest - source = "hashicorp/azurerm" - version = ">= 3.4.0" - } - - azuread = { - # https://registry.terraform.io/providers/hashicorp/azuread/latest - source = "hashicorp/azuread" - version = ">= 1.3.0" - } - } - - required_version = ">= 0.13" -} diff --git a/azurerm/_modules/aks/vnet.tf b/azurerm/_modules/aks/vnet.tf deleted file mode 100644 index 2cf18dcb..00000000 --- a/azurerm/_modules/aks/vnet.tf +++ /dev/null @@ -1,20 +0,0 @@ - -resource "azurerm_virtual_network" "current" { - count = var.network_plugin == "azure" ? 1 : 0 - - name = var.legacy_vnet_name ? "vnet-aks-${terraform.workspace}-cluster" : var.metadata_name - address_space = var.vnet_address_space - resource_group_name = data.azurerm_resource_group.current.name - location = data.azurerm_resource_group.current.location -} - -resource "azurerm_subnet" "current" { - count = var.network_plugin == "azure" ? 1 : 0 - - name = var.legacy_vnet_name ? "aks-node-subnet" : "${var.metadata_name}-${var.default_node_pool_name}-node-pool" - address_prefixes = var.subnet_address_prefixes - resource_group_name = data.azurerm_resource_group.current.name - virtual_network_name = azurerm_virtual_network.current[0].name - - service_endpoints = length(var.subnet_service_endpoints) > 0 ? var.subnet_service_endpoints : null -} diff --git a/azurerm/cluster-local/configuration.tf b/azurerm/cluster-local/configuration.tf deleted file mode 100644 index 5aafcf57..00000000 --- a/azurerm/cluster-local/configuration.tf +++ /dev/null @@ -1,41 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - # on Azure the region is determined by resource group - # in the local implementation we don't have access to that - # to still support multi-region setups locally, we hash the resource group name - # and use that as the region part of the cluster name prefixed with aks- - resource_group = local.cfg["resource_group"] - fake_region_hash = substr(sha256(local.resource_group), 0, 7) - fake_region = "aks-${local.fake_region_hash}" - - http_port_default = terraform.workspace == "apps" ? 80 : 8080 - http_port = lookup(local.cfg, "http_port", local.http_port_default) - - https_port_default = terraform.workspace == "apps" ? 443 : 8443 - https_port = lookup(local.cfg, "https_port", local.https_port_default) - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - node_image = lookup(local.cfg, "node_image", null) - - node_count = lookup(local.cfg, "default_node_pool_min_count", "1") - nodes = [ - for node, _ in range(local.node_count) : - "worker" - ] - extra_nodes = join(",", local.nodes) - -} diff --git a/azurerm/cluster-local/main.tf b/azurerm/cluster-local/main.tf deleted file mode 100644 index 2f78176c..00000000 --- a/azurerm/cluster-local/main.tf +++ /dev/null @@ -1,29 +0,0 @@ -module "cluster_metadata" { - source = "../../common/metadata" - - name_prefix = local.name_prefix - base_domain = local.base_domain - - provider_name = "azure" - provider_region = local.fake_region - - # Azure does not allow / character in labels - label_namespace = "kubestack.com-" -} - -module "cluster" { - source = "../../kind/_modules/kind" - - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_tags = module.cluster_metadata.tags - metadata_labels = module.cluster_metadata.labels - - node_image = local.node_image - extra_nodes = local.extra_nodes - - http_port = local.http_port - https_port = local.https_port - - disable_default_ingress = local.disable_default_ingress -} diff --git a/azurerm/cluster-local/outputs.tf b/azurerm/cluster-local/outputs.tf deleted file mode 100644 index 4b0efb9c..00000000 --- a/azurerm/cluster-local/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "current_config" { - value = module.configuration.merged[terraform.workspace] -} - -output "current_metadata" { - value = module.cluster_metadata -} - -output "kubeconfig" { - sensitive = true - value = module.cluster.kubeconfig -} - -output "default_ingress_ip" { - # the cluster module returns an IP as a string - # we YAML encode null for cluster-local to provide - # a unified output to consumers - value = yamlencode(null) -} diff --git a/azurerm/cluster-local/variables.tf b/azurerm/cluster-local/variables.tf deleted file mode 100644 index 8bf95161..00000000 --- a/azurerm/cluster-local/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "configuration" { - type = map(map(string)) - description = "Map with per workspace cluster configuration." -} - -variable "configuration_base_key" { - type = string - description = "The key in the configuration map all other keys inherit from." - default = "apps" -} diff --git a/azurerm/cluster-local/versions.tf b/azurerm/cluster-local/versions.tf deleted file mode 100644 index 2606a5ac..00000000 --- a/azurerm/cluster-local/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.13" -} diff --git a/azurerm/cluster/configuration.tf b/azurerm/cluster/configuration.tf deleted file mode 100644 index a6cdfb75..00000000 --- a/azurerm/cluster/configuration.tf +++ /dev/null @@ -1,76 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - resource_group = local.cfg["resource_group"] - - dns_prefix = lookup(local.cfg, "dns_prefix", "api") - - sku_tier = lookup(local.cfg, "sku_tier", "Free") - - legacy_vnet_name = lookup(local.cfg, "legacy_vnet_name", false) - vnet_address_space = split(",", lookup(local.cfg, "vnet_address_space", "10.0.0.0/8")) - subnet_address_prefixes = split(",", lookup(local.cfg, "subnet_address_prefixes", "10.1.0.0/16")) - - subnet_service_endpoints_lookup = lookup(local.cfg, "subnet_service_endpoints", "") - subnet_service_endpoints = local.subnet_service_endpoints_lookup != "" ? split(",", local.subnet_service_endpoints_lookup) : [] - - network_plugin = lookup(local.cfg, "network_plugin", "kubenet") - network_policy = lookup(local.cfg, "network_policy", "calico") - service_cidr = lookup(local.cfg, "service_cidr", "10.0.0.0/16") - dns_service_ip = lookup(local.cfg, "dns_service_ip", "10.0.0.10") - pod_cidr = lookup(local.cfg, "pod_cidr", "10.244.0.0/16") - max_pods = lookup(local.cfg, "max_pods", null) - - default_node_pool_name = lookup(local.cfg, "default_node_pool_name", "default") - default_node_pool_type = lookup(local.cfg, "default_node_pool_type", "VirtualMachineScaleSets") - - default_node_pool_enable_auto_scaling = lookup(local.cfg, "default_node_pool_enable_auto_scaling", true) - default_node_pool_min_count = lookup(local.cfg, "default_node_pool_min_count", "1") - default_node_pool_max_count = lookup(local.cfg, "default_node_pool_max_count", "1") - default_node_pool_node_count = lookup(local.cfg, "default_node_pool_node_count", "1") - - default_node_pool_vm_size = lookup(local.cfg, "default_node_pool_vm_size", "Standard_B2s") - default_node_pool_only_critical_addons = lookup(local.cfg, "default_node_pool_only_critical_addons", false) - default_node_pool_os_disk_size_gb = lookup(local.cfg, "default_node_pool_os_disk_size_gb", "30") - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - default_ingress_ip_zones_lookup = lookup(local.cfg, "default_ingress_ip_zones", "") - default_ingress_ip_zones = local.default_ingress_ip_zones_lookup != "" ? split(",", local.default_ingress_ip_zones_lookup) : [] - - enable_azure_policy_agent = lookup(local.cfg, "enable_azure_policy_agent", false) - - disable_managed_identities = lookup(local.cfg, "disable_managed_identities", false) - user_assigned_identity_id = lookup(local.cfg, "user_assigned_identity_id", null) - - enable_log_analytics = lookup(local.cfg, "enable_log_analytics", true) - - kubernetes_version = lookup(local.cfg, "kubernetes_version", null) - automatic_channel_upgrade = lookup(local.cfg, "automatic_channel_upgrade", null) - - availability_zones_lookup = lookup(local.cfg, "availability_zones", "") - availability_zones = local.availability_zones_lookup != "" ? split(",", local.availability_zones_lookup) : [] - - additional_metadata_labels_lookup = lookup(local.cfg, "additional_metadata_labels", "") - additional_metadata_labels_tuples = [for t in split(",", local.additional_metadata_labels_lookup) : split("=", t)] - additional_metadata_labels = { for t in local.additional_metadata_labels_tuples : t[0] => t[1] if length(t) == 2 } - - keda_enabled = lookup(local.cfg, "keda_enabled", false) - vertical_pod_autoscaler_enabled = lookup(local.cfg, "vertical_pod_autoscaler_enabled", false) - - upgade_settings_max_surge = lookup(local.cfg, "upgade_settings_max_surge", "10%") - upgade_settings_drain_timeout_in_minutes = lookup(local.cfg, "upgade_settings_drain_timeout_in_minutes", 0) - upgade_settings_node_soak_duration_in_minutes = lookup(local.cfg, "upgade_settings_node_soak_duration_in_minutes", 0) -} diff --git a/azurerm/_modules/aks/ingress.tf b/azurerm/cluster/ingress.tf similarity index 53% rename from azurerm/_modules/aks/ingress.tf rename to azurerm/cluster/ingress.tf index 4730b392..8c615ff2 100644 --- a/azurerm/_modules/aks/ingress.tf +++ b/azurerm/cluster/ingress.tf @@ -1,30 +1,30 @@ resource "azurerm_public_ip" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - name = var.metadata_name + name = module.cluster_metadata.name location = azurerm_kubernetes_cluster.current.location resource_group_name = azurerm_kubernetes_cluster.current.node_resource_group allocation_method = "Static" sku = "Standard" - zones = var.default_ingress_ip_zones + zones = try(coalesce(local.cfg.default_ingress_ip_zones, null), []) - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) depends_on = [azurerm_kubernetes_cluster.current] } resource "azurerm_dns_zone" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - name = var.metadata_fqdn + name = module.cluster_metadata.fqdn resource_group_name = data.azurerm_resource_group.current.name - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } resource "azurerm_dns_a_record" "host" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 name = "@" zone_name = azurerm_dns_zone.current[0].name @@ -32,11 +32,11 @@ resource "azurerm_dns_a_record" "host" { ttl = 300 records = [azurerm_public_ip.current[0].ip_address] - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } resource "azurerm_dns_a_record" "wildcard" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 name = "*" zone_name = azurerm_dns_zone.current[0].name @@ -44,5 +44,5 @@ resource "azurerm_dns_a_record" "wildcard" { ttl = 300 records = [azurerm_public_ip.current[0].ip_address] - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } diff --git a/azurerm/_modules/aks/kubeconfig.tf b/azurerm/cluster/kubeconfig.tf similarity index 100% rename from azurerm/_modules/aks/kubeconfig.tf rename to azurerm/cluster/kubeconfig.tf diff --git a/azurerm/_modules/aks/log_analytics.tf b/azurerm/cluster/log_analytics.tf similarity index 67% rename from azurerm/_modules/aks/log_analytics.tf rename to azurerm/cluster/log_analytics.tf index 879219b2..d63e2ff8 100644 --- a/azurerm/_modules/aks/log_analytics.tf +++ b/azurerm/cluster/log_analytics.tf @@ -1,15 +1,15 @@ resource "azurerm_log_analytics_workspace" "current" { - count = var.enable_log_analytics == true ? 1 : 0 - name = var.metadata_name + count = try(coalesce(local.cfg.enable_log_analytics, null), true) == true ? 1 : 0 + name = module.cluster_metadata.name location = data.azurerm_resource_group.current.location resource_group_name = data.azurerm_resource_group.current.name sku = "PerGB2018" - tags = var.metadata_labels + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } resource "azurerm_log_analytics_solution" "current" { - count = var.enable_log_analytics == true ? 1 : 0 + count = try(coalesce(local.cfg.enable_log_analytics, null), true) == true ? 1 : 0 solution_name = "ContainerInsights" location = data.azurerm_resource_group.current.location resource_group_name = data.azurerm_resource_group.current.name @@ -21,4 +21,3 @@ resource "azurerm_log_analytics_solution" "current" { product = "OMSGallery/ContainerInsights" } } - diff --git a/azurerm/cluster/main.tf b/azurerm/cluster/main.tf index 67516f9a..ac607395 100644 --- a/azurerm/cluster/main.tf +++ b/azurerm/cluster/main.tf @@ -1,76 +1,117 @@ -data "azurerm_resource_group" "current" { - name = local.resource_group +module "configuration" { + source = "../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] } module "cluster_metadata" { source = "../../common/metadata" - name_prefix = local.name_prefix - base_domain = local.base_domain + name_prefix = local.cfg.name_prefix + base_domain = local.cfg.base_domain provider_name = "azure" - provider_region = data.azurerm_resource_group.current.location + provider_region = data.azurerm_resource_group.current.location # references cluster.tf data source # Azure does not allow / character in labels label_namespace = "kubestack.com-" } -module "cluster" { - source = "../_modules/aks" +data "azurerm_resource_group" "current" { + name = local.cfg.resource_group +} + +resource "azurerm_kubernetes_cluster" "current" { + name = module.cluster_metadata.name + location = data.azurerm_resource_group.current.location + resource_group_name = data.azurerm_resource_group.current.name + dns_prefix = try(coalesce(local.cfg.dns_prefix, null), "api") + sku_tier = try(coalesce(local.cfg.sku_tier, null), "Free") + kubernetes_version = local.cfg.kubernetes_version + automatic_upgrade_channel = local.cfg.automatic_channel_upgrade + + role_based_access_control_enabled = true + + default_node_pool { + name = try(coalesce(local.cfg.default_node_pool.name, null), "default") + type = try(coalesce(local.cfg.default_node_pool.type, null), "VirtualMachineScaleSets") + + auto_scaling_enabled = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) + + # set min and max count only if autoscaling is _enabled_ + min_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? try(coalesce(local.cfg.default_node_pool.min_count, null), 1) : null + max_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? try(coalesce(local.cfg.default_node_pool.max_count, null), 1) : null + + # set node count only if auto scaling is _disabled_ + node_count = try(coalesce(local.cfg.default_node_pool.enable_auto_scaling, null), true) ? null : try(coalesce(local.cfg.default_node_pool.node_count, null), 1) + + vm_size = try(coalesce(local.cfg.default_node_pool.vm_size, null), "Standard_D1_v2") + os_disk_size_gb = try(coalesce(local.cfg.default_node_pool.os_disk_size_gb, null), 30) + + vnet_subnet_id = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? azurerm_subnet.current[0].id : null + max_pods = local.cfg.max_pods - resource_group = local.resource_group + only_critical_addons_enabled = try(coalesce(local.cfg.default_node_pool.only_critical_addons, null), false) - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_labels = merge(module.cluster_metadata.labels, local.additional_metadata_labels) - metadata_label_namespace = module.cluster_metadata.label_namespace + zones = try(coalesce(local.cfg.availability_zones, null), []) - dns_prefix = local.dns_prefix + upgrade_settings { + max_surge = try(coalesce(try(local.cfg.default_node_pool.upgrade_settings, null).max_surge, null), "10%") + drain_timeout_in_minutes = try(coalesce(try(local.cfg.default_node_pool.upgrade_settings, null).drain_timeout_in_minutes, null), 0) + node_soak_duration_in_minutes = try(coalesce(try(local.cfg.default_node_pool.upgrade_settings, null).node_soak_duration_in_minutes, null), 0) + } + } - sku_tier = local.sku_tier + network_profile { + network_plugin = try(coalesce(local.cfg.network_plugin, null), "kubenet") + network_policy = try(coalesce(local.cfg.network_policy, null), "calico") - legacy_vnet_name = local.legacy_vnet_name - vnet_address_space = local.vnet_address_space - subnet_address_prefixes = local.subnet_address_prefixes - subnet_service_endpoints = local.subnet_service_endpoints + service_cidr = try(coalesce(local.cfg.service_cidr, null), "10.0.0.0/16") + dns_service_ip = try(coalesce(local.cfg.dns_service_ip, null), "10.0.0.10") + pod_cidr = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? null : try(coalesce(local.cfg.pod_cidr, null), "10.244.0.0/16") + } - network_plugin = local.network_plugin - network_policy = local.network_policy - service_cidr = local.service_cidr - dns_service_ip = local.dns_service_ip - pod_cidr = local.pod_cidr - max_pods = local.max_pods + dynamic "identity" { + for_each = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? toset([]) : toset([1]) - default_node_pool_name = local.default_node_pool_name - default_node_pool_type = local.default_node_pool_type + content { + type = local.cfg.user_assigned_identity_id == null ? "SystemAssigned" : "UserAssigned" - default_node_pool_enable_auto_scaling = local.default_node_pool_enable_auto_scaling - default_node_pool_min_count = local.default_node_pool_min_count - default_node_pool_max_count = local.default_node_pool_max_count - default_node_pool_node_count = local.default_node_pool_node_count + identity_ids = local.cfg.user_assigned_identity_id == null ? null : [local.cfg.user_assigned_identity_id] + } + } - default_node_pool_only_critical_addons = local.default_node_pool_only_critical_addons - default_node_pool_vm_size = local.default_node_pool_vm_size - default_node_pool_os_disk_size_gb = local.default_node_pool_os_disk_size_gb + dynamic "service_principal" { + for_each = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? toset([1]) : toset([]) - disable_default_ingress = local.disable_default_ingress - default_ingress_ip_zones = local.default_ingress_ip_zones + content { + client_id = azuread_application.current[0].application_id + client_secret = azuread_service_principal_password.current[0].value + } + } - enable_azure_policy_agent = local.enable_azure_policy_agent + azure_policy_enabled = try(coalesce(local.cfg.enable_azure_policy_agent, null), false) - disable_managed_identities = local.disable_managed_identities - user_assigned_identity_id = local.user_assigned_identity_id + dynamic "oms_agent" { + for_each = try(coalesce(local.cfg.enable_log_analytics, null), true) ? toset([1]) : toset([]) - kubernetes_version = local.kubernetes_version - automatic_channel_upgrade = local.automatic_channel_upgrade - enable_log_analytics = local.enable_log_analytics + content { + log_analytics_workspace_id = azurerm_log_analytics_workspace.current[0].id + } + } - availability_zones = local.availability_zones + dynamic "workload_autoscaler_profile" { + for_each = try(coalesce(local.cfg.keda_enabled, null), false) || try(coalesce(local.cfg.vertical_pod_autoscaler_enabled, null), false) ? toset([1]) : toset([]) - keda_enabled = local.keda_enabled - vertical_pod_autoscaler_enabled = local.vertical_pod_autoscaler_enabled + content { + keda_enabled = try(coalesce(local.cfg.keda_enabled, null), false) + vertical_pod_autoscaler_enabled = try(coalesce(local.cfg.vertical_pod_autoscaler_enabled, null), false) + } + } - upgade_settings_max_surge = local.upgade_settings_max_surge - upgade_settings_drain_timeout_in_minutes = local.upgade_settings_drain_timeout_in_minutes - upgade_settings_node_soak_duration_in_minutes = local.upgade_settings_node_soak_duration_in_minutes + tags = merge(module.cluster_metadata.labels, try(local.cfg.additional_metadata_labels, {})) } diff --git a/azurerm/cluster/moved.tf b/azurerm/cluster/moved.tf new file mode 100644 index 00000000..097a9f03 --- /dev/null +++ b/azurerm/cluster/moved.tf @@ -0,0 +1,59 @@ +moved { + from = module.cluster.azurerm_kubernetes_cluster.current + to = azurerm_kubernetes_cluster.current +} + +moved { + from = module.cluster.azurerm_virtual_network.current + to = azurerm_virtual_network.current +} + +moved { + from = module.cluster.azurerm_subnet.current + to = azurerm_subnet.current +} + +moved { + from = module.cluster.azurerm_log_analytics_workspace.current + to = azurerm_log_analytics_workspace.current +} + +moved { + from = module.cluster.azurerm_log_analytics_solution.current + to = azurerm_log_analytics_solution.current +} + +moved { + from = module.cluster.azurerm_public_ip.current + to = azurerm_public_ip.current +} + +moved { + from = module.cluster.azurerm_dns_zone.current + to = azurerm_dns_zone.current +} + +moved { + from = module.cluster.azurerm_dns_a_record.host + to = azurerm_dns_a_record.host +} + +moved { + from = module.cluster.azurerm_dns_a_record.wildcard + to = azurerm_dns_a_record.wildcard +} + +moved { + from = module.cluster.azuread_application.current + to = azuread_application.current +} + +moved { + from = module.cluster.azuread_service_principal.current + to = azuread_service_principal.current +} + +moved { + from = module.cluster.azuread_service_principal_password.current + to = azuread_service_principal_password.current +} diff --git a/azurerm/cluster/node-pool/configuration.tf b/azurerm/cluster/node-pool/configuration.tf deleted file mode 100644 index c7c4a307..00000000 --- a/azurerm/cluster/node-pool/configuration.tf +++ /dev/null @@ -1,43 +0,0 @@ -module "configuration" { - source = "../../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - cfg = module.configuration.merged[terraform.workspace] - - node_pool_name = local.cfg["node_pool_name"] - vm_size = local.cfg["vm_size"] != null ? local.cfg["vm_size"] : "Standard_B2s" - max_pods = local.cfg["max_pods"] != null ? local.cfg["max_pods"] : "110" - os_disk_type = local.cfg["os_disk_type"] != null ? local.cfg["os_disk_type"] : "Managed" - os_disk_size_gb = local.cfg["os_disk_size_gb"] - availability_zones_list = local.cfg["availability_zones"] != null ? local.cfg["availability_zones"] : [] - availability_zones = length(local.availability_zones_list) == 0 ? null : local.availability_zones_list - - enable_auto_scaling = local.cfg["enable_auto_scaling"] != null ? local.cfg["enable_auto_scaling"] : true - max_count_string = local.cfg["max_count"] != null ? local.cfg["max_count"] : "1" - min_count_string = local.cfg["min_count"] != null ? local.cfg["min_count"] : "1" - max_count = local.enable_auto_scaling ? local.max_count_string : null - min_count = local.enable_auto_scaling ? local.min_count_string : null - node_count = local.cfg["node_count"] != null ? local.cfg["node_count"] : "1" - - use_spot = local.cfg["use_spot"] != null ? local.cfg["use_spot"] : false - priority = local.use_spot ? "Spot" : "Regular" - eviction_policy = local.use_spot ? local.cfg["eviction_policy"] : null - max_spot_price_value = local.cfg["max_spot_price"] != null ? local.cfg["max_spot_price"] : "-1" - max_spot_price = local.use_spot ? local.max_spot_price_value : null - - user_node_labels = local.cfg["node_labels"] != null ? local.cfg["node_labels"] : {} - user_node_taints = local.cfg["node_taints"] != null ? local.cfg["node_taints"] : [] - spot_taints = [ - "kubernetes.azure.com/scalesetpriority=spot:NoSchedule", - ] - spot_labels = { "kubernetes.azure.com/scalesetpriority" = "spot" } - - taints = compact(concat(local.use_spot ? local.spot_taints : [], local.user_node_taints)) - labels = merge(local.use_spot ? local.spot_labels : {}, local.user_node_labels) - node_labels = length(local.labels) == 0 ? null : local.labels - node_taints = length(local.taints) == 0 ? null : local.taints -} diff --git a/azurerm/cluster/node-pool/main.tf b/azurerm/cluster/node-pool/main.tf index 65d5a3e8..2d8ba627 100644 --- a/azurerm/cluster/node-pool/main.tf +++ b/azurerm/cluster/node-pool/main.tf @@ -1,22 +1,63 @@ -module "node_pool" { - source = "../../_modules/aks/node_pool" - - cluster_name = var.cluster_name - resource_group = var.resource_group - - node_pool_name = local.node_pool_name - enable_auto_scaling = local.enable_auto_scaling - max_count = local.max_count - min_count = local.min_count - node_count = local.node_count - vm_size = local.vm_size - eviction_policy = local.eviction_policy - priority = local.priority - max_spot_price = local.max_spot_price - node_labels = local.node_labels - node_taints = local.node_taints - availability_zones = local.availability_zones - os_disk_type = local.os_disk_type - os_disk_size_gb = local.os_disk_size_gb - max_pods = local.max_pods +module "configuration" { + source = "../../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] + + vnet_subnets = compact(var.cluster.default_node_pool.*.vnet_subnet_id) + + # Conditional logic for auto_scaling + effective_max_count = try(coalesce(local.cfg.enable_auto_scaling, null), true) == true ? local.cfg.max_count : null + effective_min_count = try(coalesce(local.cfg.enable_auto_scaling, null), true) == true ? local.cfg.min_count : null + + # Conditional logic for spot instances + effective_priority = try(coalesce(local.cfg.use_spot, null), false) == true ? "Spot" : "Regular" + effective_eviction_policy = try(coalesce(local.cfg.use_spot, null), false) == true ? local.cfg.eviction_policy : null + effective_max_spot_price = try(coalesce(local.cfg.use_spot, null), false) == true ? try(coalesce(local.cfg.max_spot_price, null), -1) : null + + # Spot instance labels and taints + spot_label = try(coalesce(local.cfg.use_spot, null), false) == true ? { "kubernetes.azure.com/scalesetpriority" = "spot" } : {} + spot_taints = try(coalesce(local.cfg.use_spot, null), false) == true ? ["kubernetes.azure.com/scalesetpriority=spot:NoSchedule"] : [] + + # Merge labels and taints + node_labels_merged = merge(local.spot_label, try(coalesce(local.cfg.node_labels, null), {})) + node_taints_merged = compact(concat(local.spot_taints, try(coalesce(local.cfg.node_taints, null), []))) +} + +resource "azurerm_kubernetes_cluster_node_pool" "current" { + name = local.cfg.node_pool_name + kubernetes_cluster_id = var.cluster.id + auto_scaling_enabled = try(coalesce(local.cfg.enable_auto_scaling, null), true) + max_count = local.effective_max_count + min_count = local.effective_min_count + node_count = try(coalesce(local.cfg.node_count, null), 1) + vm_size = try(coalesce(local.cfg.vm_size, null), "Standard_B2s") + node_labels = local.node_labels_merged + node_taints = local.node_taints_merged + zones = local.cfg.availability_zones + max_pods = try(coalesce(local.cfg.max_pods, null), 110) + os_disk_type = try(coalesce(local.cfg.os_disk_type, null), "Managed") + os_disk_size_gb = local.cfg.os_disk_size_gb + priority = local.effective_priority + eviction_policy = local.effective_eviction_policy + spot_max_price = local.effective_max_spot_price + + upgrade_settings { + max_surge = try(coalesce(try(local.cfg.upgrade_settings, null).max_surge, null), "10%") + drain_timeout_in_minutes = try(coalesce(try(local.cfg.upgrade_settings, null).drain_timeout_in_minutes, null), 0) + node_soak_duration_in_minutes = try(coalesce(try(local.cfg.upgrade_settings, null).node_soak_duration_in_minutes, null), 0) + } + + # The cluster's agent_pool_profile may contain empty strings for vnet_subnet_id + # in some configurations. In that case we rely on the defaults. + vnet_subnet_id = length(local.vnet_subnets) == 0 ? null : coalesce(tolist(local.vnet_subnets)...) + + # When autoscaling acts, the node_count gets changed, but it should not be + # forced to match the config + lifecycle { + ignore_changes = [node_count] + } } diff --git a/azurerm/cluster/node-pool/moved.tf b/azurerm/cluster/node-pool/moved.tf new file mode 100644 index 00000000..20b14c3f --- /dev/null +++ b/azurerm/cluster/node-pool/moved.tf @@ -0,0 +1,4 @@ +moved { + from = module.node_pool.azurerm_kubernetes_cluster_node_pool.current + to = azurerm_kubernetes_cluster_node_pool.current +} diff --git a/azurerm/cluster/node-pool/outputs.tf b/azurerm/cluster/node-pool/outputs.tf new file mode 100644 index 00000000..871c9f88 --- /dev/null +++ b/azurerm/cluster/node-pool/outputs.tf @@ -0,0 +1,3 @@ +output "current_config" { + value = local.cfg +} diff --git a/azurerm/cluster/node-pool/variables.tf b/azurerm/cluster/node-pool/variables.tf index e62c97b2..a2cc1c8f 100644 --- a/azurerm/cluster/node-pool/variables.tf +++ b/azurerm/cluster/node-pool/variables.tf @@ -2,39 +2,43 @@ variable "configuration" { type = map(object({ node_pool_name = optional(string) vm_size = optional(string) - node_count = optional(string) + node_count = optional(number) enable_auto_scaling = optional(bool) - max_count = optional(string) - min_count = optional(string) + max_count = optional(number) + min_count = optional(number) eviction_policy = optional(string) - max_pods = optional(string) + max_pods = optional(number) os_disk_type = optional(string) - os_disk_size_gb = optional(string) + os_disk_size_gb = optional(number) use_spot = optional(bool) - max_spot_price = optional(string) + max_spot_price = optional(number) node_labels = optional(map(string)) node_taints = optional(list(string)) availability_zones = optional(list(string)) + + upgrade_settings = optional(object({ + max_surge = optional(string) + drain_timeout_in_minutes = optional(number) + node_soak_duration_in_minutes = optional(number) + })) })) description = "Map with per workspace node pool configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } -variable "cluster_name" { - type = string - description = "The name of the cluster to attach the node pool to" -} - -variable "resource_group" { - type = string - description = "The resource group of the cluster to attach the node pool to" +variable "cluster" { + type = any + description = "The cluster output from the cluster module." + nullable = false } diff --git a/azurerm/cluster/node-pool/versions.tf b/azurerm/cluster/node-pool/versions.tf index e4967d8e..cf41232e 100644 --- a/azurerm/cluster/node-pool/versions.tf +++ b/azurerm/cluster/node-pool/versions.tf @@ -1,6 +1,6 @@ terraform { required_providers { - azure = { + azurerm = { source = "hashicorp/azurerm" } } diff --git a/azurerm/cluster/outputs.tf b/azurerm/cluster/outputs.tf index d0d3deda..7a27b44d 100644 --- a/azurerm/cluster/outputs.tf +++ b/azurerm/cluster/outputs.tf @@ -1,9 +1,13 @@ +output "cluster" { + value = azurerm_kubernetes_cluster.current +} + output "aks_vnet" { - value = module.cluster.aks_vnet + value = length(azurerm_virtual_network.current) > 0 ? azurerm_virtual_network.current[0] : null } output "current_config" { - value = module.configuration.merged[terraform.workspace] + value = local.cfg } output "current_metadata" { @@ -12,9 +16,9 @@ output "current_metadata" { output "kubeconfig" { sensitive = true - value = module.cluster.kubeconfig + value = local.kubeconfig } output "default_ingress_ip" { - value = module.cluster.default_ingress_ip + value = length(azurerm_public_ip.current) > 0 ? azurerm_public_ip.current[0].ip_address : null } diff --git a/azurerm/_modules/aks/provider.tf b/azurerm/cluster/provider.tf similarity index 100% rename from azurerm/_modules/aks/provider.tf rename to azurerm/cluster/provider.tf diff --git a/azurerm/cluster/service_principal.tf b/azurerm/cluster/service_principal.tf new file mode 100644 index 00000000..376158de --- /dev/null +++ b/azurerm/cluster/service_principal.tf @@ -0,0 +1,17 @@ +resource "azuread_application" "current" { + count = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? 1 : 0 + + display_name = module.cluster_metadata.name +} + +resource "azuread_service_principal" "current" { + count = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? 1 : 0 + + client_id = azuread_application.current[0].client_id +} + +resource "azuread_service_principal_password" "current" { + count = try(coalesce(local.cfg.disable_managed_identities, null), false) == true ? 1 : 0 + + service_principal_id = azuread_service_principal.current[0].id +} diff --git a/azurerm/_modules/aks/templates/kubeconfig.tpl b/azurerm/cluster/templates/kubeconfig.tpl similarity index 100% rename from azurerm/_modules/aks/templates/kubeconfig.tpl rename to azurerm/cluster/templates/kubeconfig.tpl diff --git a/azurerm/cluster/variables.tf b/azurerm/cluster/variables.tf index 8bf95161..2fd729d4 100644 --- a/azurerm/cluster/variables.tf +++ b/azurerm/cluster/variables.tf @@ -1,10 +1,71 @@ variable "configuration" { - type = map(map(string)) + type = map(object({ + # Required attributes + name_prefix = optional(string) + base_domain = optional(string) + resource_group = optional(string) + + # Optional attributes without defaults + dns_prefix = optional(string) + sku_tier = optional(string) + + legacy_vnet_name = optional(bool) + vnet_address_space = optional(list(string)) + subnet_address_prefixes = optional(list(string)) + subnet_service_endpoints = optional(list(string)) + + network_plugin = optional(string) + network_policy = optional(string) + service_cidr = optional(string) + dns_service_ip = optional(string) + pod_cidr = optional(string) + max_pods = optional(number) + + default_node_pool = optional(object({ + name = optional(string) + type = optional(string) + enable_auto_scaling = optional(bool) + min_count = optional(number) + max_count = optional(number) + node_count = optional(number) + vm_size = optional(string) + only_critical_addons = optional(bool) + os_disk_size_gb = optional(number) + upgrade_settings = optional(object({ + max_surge = optional(string) + drain_timeout_in_minutes = optional(number) + node_soak_duration_in_minutes = optional(number) + })) + })) + + disable_default_ingress = optional(bool) + + default_ingress_ip_zones = optional(list(string)) + + enable_azure_policy_agent = optional(bool) + + disable_managed_identities = optional(bool) + user_assigned_identity_id = optional(string) + + enable_log_analytics = optional(bool) + + kubernetes_version = optional(string) + automatic_channel_upgrade = optional(string) + + availability_zones = optional(list(string)) + + additional_metadata_labels = optional(map(string)) + + keda_enabled = optional(bool) + vertical_pod_autoscaler_enabled = optional(bool) + })) description = "Map with per workspace cluster configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } diff --git a/azurerm/cluster/versions.tf b/azurerm/cluster/versions.tf index 2606a5ac..f1424a25 100644 --- a/azurerm/cluster/versions.tf +++ b/azurerm/cluster/versions.tf @@ -1,4 +1,13 @@ - terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + + azuread = { + source = "hashicorp/azuread" + } + } + required_version = ">= 0.13" } diff --git a/azurerm/cluster/vnet.tf b/azurerm/cluster/vnet.tf new file mode 100644 index 00000000..c59f4776 --- /dev/null +++ b/azurerm/cluster/vnet.tf @@ -0,0 +1,19 @@ +resource "azurerm_virtual_network" "current" { + count = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? 1 : 0 + + name = try(coalesce(local.cfg.legacy_vnet_name, null), false) ? "vnet-aks-${terraform.workspace}-cluster" : module.cluster_metadata.name + address_space = try(coalesce(local.cfg.vnet_address_space, null), ["10.0.0.0/8"]) + resource_group_name = data.azurerm_resource_group.current.name + location = data.azurerm_resource_group.current.location +} + +resource "azurerm_subnet" "current" { + count = try(coalesce(local.cfg.network_plugin, null), "kubenet") == "azure" ? 1 : 0 + + name = try(coalesce(local.cfg.legacy_vnet_name, null), false) ? "aks-node-subnet" : "${module.cluster_metadata.name}-${try(coalesce(local.cfg.default_node_pool.name, null), "default")}-node-pool" + address_prefixes = try(coalesce(local.cfg.subnet_address_prefixes, null), ["10.1.0.0/16"]) + resource_group_name = data.azurerm_resource_group.current.name + virtual_network_name = azurerm_virtual_network.current[0].name + + service_endpoints = length(try(coalesce(local.cfg.subnet_service_endpoints, null), [])) > 0 ? try(coalesce(local.cfg.subnet_service_endpoints, null), []) : null +} diff --git a/common/configuration/outputs.tf b/common/configuration/outputs.tf index 37695c95..010c38a3 100644 --- a/common/configuration/outputs.tf +++ b/common/configuration/outputs.tf @@ -1,14 +1,32 @@ locals { - base_config = var.configuration[var.base_key] + base_cfg = var.configuration[var.base_key] merged = { - for env_key, env in var.configuration : + # loop through all environments + for env_key, env_cfg in var.configuration : env_key => { - # loop through all config keys in base_key environment and current env - # if current env has that key, use the value from current env - # if not, use the value from the base_key environment - for key in setunion(keys(env), keys(local.base_config)) : - key => lookup(env, key, null) != null ? env[key] : local.base_config[key] + # loop through all keys in current and base_key environment config + for cfg_key in setunion(keys(env_cfg), keys(local.base_cfg)) : + cfg_key => try( + # try one level of nesting + { + for nested_key in setunion(keys(env_cfg[cfg_key]), keys(local.base_cfg[cfg_key])) : + nested_key => try( + # use current environment's nested value or base env's value + coalesce( + lookup(env_cfg[cfg_key], nested_key, null), + lookup(local.base_cfg[cfg_key], nested_key, null), + ), + local.base_cfg[cfg_key][nested_key] + ) + }, + # fall back to use current environment's value or base env's value + coalesce( + lookup(env_cfg, cfg_key, null), + lookup(local.base_cfg, cfg_key, null), + ), + local.base_cfg[cfg_key] + ) } } } diff --git a/common/configuration/tests/non_string_values/test_hash.tf b/common/configuration/tests/non_string_values/test_hash.tf index 10e47496..b0d40b19 100644 --- a/common/configuration/tests/non_string_values/test_hash.tf +++ b/common/configuration/tests/non_string_values/test_hash.tf @@ -22,18 +22,18 @@ module "mut_hash" { base_key = "apps" } -resource "test_assertions" "overwrite_hash" { - component = "overwrite_hash" +resource "test_assertions" "merge_hash" { + component = "merge_hash" equal "scheme" { - description = "can overwrite hashes" + description = "can merge hashes" got = module.mut_hash.merged want = { "apps" = { - "test_hash" = tomap({ + "test_hash" = { "from_apps_1" = "from_apps_1" "from_apps_2" = "from_apps_2" - }) + } } "loc" = { "test_hash" = tomap({ @@ -42,9 +42,11 @@ resource "test_assertions" "overwrite_hash" { }) } "ops" = { - "test_hash" = tomap({ - "from_ops_1" = "from_ops_1" - }) + "test_hash" = { + "from_apps_1" = "from_apps_1" + "from_apps_2" = "from_apps_2" + "from_ops_1" = "from_ops_1" + } } } } diff --git a/common/configuration/tests/non_string_values/test_object.tf b/common/configuration/tests/non_string_values/test_object.tf index c45e4239..99638891 100644 --- a/common/configuration/tests/non_string_values/test_object.tf +++ b/common/configuration/tests/non_string_values/test_object.tf @@ -2,11 +2,11 @@ module "mut_object" { source = "./wrapper" } -resource "test_assertions" "overwrite_object" { - component = "overwrite_object" +resource "test_assertions" "merge_object" { + component = "merge_object" equal "scheme" { - description = "can overwrite objects" + description = "can merge objects" got = module.mut_object.merged want = { "apps" = { @@ -18,9 +18,14 @@ resource "test_assertions" "overwrite_object" { "test_list_string" = tolist([ "from_apps", ]) - "test_map_string" = tomap({ + "test_map_object" = { + "env" = { + "key" = "from_apps" + } + } + "test_map_string" = { "key" = "from_apps" - }) + } "test_object" = { "key" = "from_apps" } @@ -35,6 +40,11 @@ resource "test_assertions" "overwrite_object" { "test_list_string" = tolist([ "from_apps", ]) + "test_map_object" = tomap({ + "env" = { + "key" = "from_apps" + } + }) "test_map_string" = tomap({ "key" = "from_apps" }) @@ -52,9 +62,14 @@ resource "test_assertions" "overwrite_object" { "test_list_string" = tolist([ "from_ops", ]) - "test_map_string" = tomap({ + "test_map_object" = { + "env" = { + "key" = "from_ops" + } + } + "test_map_string" = { "key" = "from_ops" - }) + } "test_object" = { "key" = "from_ops" } diff --git a/common/configuration/tests/non_string_values/wrapper/main.tf b/common/configuration/tests/non_string_values/wrapper/main.tf index ed2a2e7b..3a16f872 100644 --- a/common/configuration/tests/non_string_values/wrapper/main.tf +++ b/common/configuration/tests/non_string_values/wrapper/main.tf @@ -13,6 +13,9 @@ variable "test_configuration" { test_object = optional(object({ key = string })) + test_map_object = optional(map(object({ + key = string + }))) })) description = "Map with per workspace module configuration." default = { @@ -34,6 +37,10 @@ variable "test_configuration" { test_object = { key = "from_apps" } + + test_map_object = { + env = { key = "from_apps" } + } } ops = { @@ -54,6 +61,10 @@ variable "test_configuration" { test_object = { key = "from_ops" } + + test_map_object = { + env = { key = "from_ops" } + } } loc = { diff --git a/common/metadata/variables.tf b/common/metadata/variables.tf index fface973..6044891e 100644 --- a/common/metadata/variables.tf +++ b/common/metadata/variables.tf @@ -35,4 +35,3 @@ variable "label_namespace" { description = "Prefix labels are namespaced with." default = "kubestack.com/" } - diff --git a/google/_modules/gke/cluster.tf b/google/_modules/gke/cluster.tf deleted file mode 100644 index b0f445dd..00000000 --- a/google/_modules/gke/cluster.tf +++ /dev/null @@ -1,131 +0,0 @@ -resource "google_container_cluster" "current" { - project = var.project - name = var.metadata_name - - deletion_protection = var.deletion_protection - - location = var.location - node_locations = var.node_locations - - min_master_version = var.min_master_version - - release_channel { - channel = var.release_channel - } - - remove_default_node_pool = var.remove_default_node_pool - initial_node_count = var.initial_node_count - - master_auth { - client_certificate_config { - issue_client_certificate = false - } - } - - network = google_compute_network.current.self_link - - dynamic "workload_identity_config" { - for_each = var.disable_workload_identity == false ? toset([1]) : toset([]) - content { - workload_pool = "${var.project}.svc.id.goog" - } - } - - dynamic "database_encryption" { - for_each = var.cluster_database_encryption_key_name != null ? toset([1]) : toset([]) - content { - state = "ENCRYPTED" - key_name = var.cluster_database_encryption_key_name - } - } - - # - # - # Addon config - addons_config { - http_load_balancing { - disabled = true - } - - horizontal_pod_autoscaling { - disabled = false - } - - network_policy_config { - disabled = false - } - - dynamic "gcs_fuse_csi_driver_config" { - for_each = var.enable_gcs_fuse_csi_driver != null ? [1] : [] - - content { - enabled = var.enable_gcs_fuse_csi_driver - } - } - } - - network_policy { - enabled = true - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.daily_maintenance_window_start_time - } - - dynamic "maintenance_exclusion" { - for_each = var.maintenance_exclusion_start_time != "" ? [1] : [] - - content { - start_time = var.maintenance_exclusion_start_time - end_time = var.maintenance_exclusion_end_time - exclusion_name = var.maintenance_exclusion_name - - exclusion_options { - scope = var.maintenance_exclusion_scope - } - } - } - } - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config_cidr_blocks == null ? toset([]) : toset([1]) - - content { - dynamic "cidr_blocks" { - for_each = var.master_authorized_networks_config_cidr_blocks - - content { - cidr_block = cidr_blocks.value - display_name = "terraform-kubestack_${cidr_blocks.value}" - } - } - } - } - - logging_config { - enable_components = var.logging_config_enable_components - } - - monitoring_config { - enable_components = var.monitoring_config_enable_components - } - - private_cluster_config { - enable_private_nodes = var.enable_private_nodes - enable_private_endpoint = false - master_ipv4_cidr_block = var.master_cidr_block - } - - dynamic "ip_allocation_policy" { - for_each = var.enable_private_nodes ? toset([1]) : [] - - content { - cluster_ipv4_cidr_block = var.cluster_ipv4_cidr_block - services_ipv4_cidr_block = var.services_ipv4_cidr_block - } - } - - enable_intranode_visibility = var.enable_intranode_visibility - enable_tpu = var.enable_tpu -} diff --git a/google/_modules/gke/network.tf b/google/_modules/gke/network.tf deleted file mode 100644 index c398a04b..00000000 --- a/google/_modules/gke/network.tf +++ /dev/null @@ -1,78 +0,0 @@ -resource "google_compute_network" "current" { - name = var.metadata_name - project = var.project - auto_create_subnetworks = "true" -} - -resource "google_compute_address" "nat" { - count = var.enable_cloud_nat ? var.cloud_nat_ip_count : 0 - - region = google_container_cluster.current.location - project = var.project - - name = "nat-${var.metadata_name}-${count.index}" -} - -resource "google_compute_router" "current" { - count = var.enable_cloud_nat ? 1 : 0 - - project = var.project - name = var.metadata_name - region = google_container_cluster.current.location - - network = google_compute_network.current.name - - bgp { - advertise_mode = ( - var.router_advertise_config == null - ? null - : var.router_advertise_config.mode - ) - advertised_groups = ( - var.router_advertise_config == null ? null : ( - var.router_advertise_config.mode != "CUSTOM" - ? null - : var.router_advertise_config.groups - ) - ) - dynamic "advertised_ip_ranges" { - for_each = ( - var.router_advertise_config == null ? {} : ( - var.router_advertise_config.mode != "CUSTOM" - ? {} - : var.router_advertise_config.ip_ranges - ) - ) - iterator = range - content { - range = range.key - description = range.value - } - } - - # expected "bgp.0.asn" to be a RFC6996-compliant Local ASN: - # must be either in the private ASN ranges: [64512..65534], [4200000000..4294967294]; - # or be the value of [16550] - asn = var.router_asn != null ? var.router_asn : 16550 - } -} - -resource "google_compute_router_nat" "nat" { - count = var.enable_cloud_nat ? 1 : 0 - - project = var.project - name = var.metadata_name - region = google_compute_router.current[0].region - router = google_compute_router.current[0].name - - enable_endpoint_independent_mapping = var.cloud_nat_endpoint_independent_mapping - min_ports_per_vm = var.cloud_nat_min_ports_per_vm - nat_ip_allocate_option = var.cloud_nat_ip_count > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" - source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" - nat_ips = var.cloud_nat_ip_count > 0 ? google_compute_address.nat.*.self_link : null - - log_config { - enable = true - filter = "ERRORS_ONLY" - } -} diff --git a/google/_modules/gke/node_pool.tf b/google/_modules/gke/node_pool.tf deleted file mode 100644 index 05d938fa..00000000 --- a/google/_modules/gke/node_pool.tf +++ /dev/null @@ -1,38 +0,0 @@ -module "node_pool" { - source = "./node_pool" - - project = var.project - location = google_container_cluster.current.location - - cluster_name = google_container_cluster.current.name - pool_name = "default" - - service_account_email = google_service_account.current.email - disable_per_node_pool_service_account = true - - metadata_tags = var.metadata_tags - metadata_labels = var.metadata_labels - - initial_node_count = var.initial_node_count - min_node_count = var.min_node_count - max_node_count = var.max_node_count - location_policy = var.location_policy - - extra_oauth_scopes = var.extra_oauth_scopes - - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - image_type = var.image_type - machine_type = var.machine_type - - # Whether to use preemptible nodes for this node pool - preemptible = var.preemptible - # Whether the nodes will be automatically repaired - auto_repair = var.auto_repair - # Whether the nodes will be automatically upgraded - auto_upgrade = var.auto_upgrade - - node_workload_metadata_config = var.node_workload_metadata_config - - guest_accelerator = var.guest_accelerator -} diff --git a/google/_modules/gke/node_pool/main.tf b/google/_modules/gke/node_pool/main.tf deleted file mode 100644 index 9504cfaa..00000000 --- a/google/_modules/gke/node_pool/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -resource "google_container_node_pool" "current" { - name = var.pool_name - project = var.project - cluster = var.cluster_name - location = var.location - - initial_node_count = var.initial_node_count - - autoscaling { - min_node_count = var.min_node_count - max_node_count = var.max_node_count - location_policy = var.location_policy - } - - node_locations = var.node_locations - - dynamic "network_config" { - for_each = var.network_config == null ? [] : [1] - - content { - enable_private_nodes = var.network_config["enable_private_nodes"] - create_pod_range = var.network_config["create_pod_range"] - pod_ipv4_cidr_block = var.network_config["pod_ipv4_cidr_block"] - } - } - - # - # - # Node config - node_config { - service_account = var.disable_per_node_pool_service_account ? var.service_account_email : google_service_account.current[0].email - - oauth_scopes = local.oauth_scopes - - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - - image_type = var.image_type - machine_type = var.machine_type - preemptible = var.preemptible - - labels = merge(var.labels, var.metadata_labels) - - tags = concat(var.metadata_tags, var.instance_tags) - - workload_metadata_config { - mode = var.node_workload_metadata_config - } - - dynamic "guest_accelerator" { - # Make sure to generate this only once - for_each = var.guest_accelerator == null ? [] : [1] - - content { - type = var.guest_accelerator.type - count = var.guest_accelerator.count - - dynamic "gpu_sharing_config" { - for_each = var.guest_accelerator.gpu_sharing_config == null ? [] : [1] - - content { - gpu_sharing_strategy = var.guest_accelerator.gpu_sharing_config.gpu_sharing_strategy - max_shared_clients_per_gpu = var.guest_accelerator.gpu_sharing_config.max_shared_clients_per_gpu - } - } - } - } - - dynamic "taint" { - for_each = var.taints == null ? [] : var.taints - - content { - key = taint.value["key"] - value = taint.value["value"] - effect = taint.value["effect"] - } - } - - dynamic "ephemeral_storage_local_ssd_config" { - for_each = var.ephemeral_storage_local_ssd_config == null ? [] : [1] - - content { - local_ssd_count = var.ephemeral_storage_local_ssd_config.local_ssd_count - } - } - } - - management { - auto_repair = var.auto_repair - auto_upgrade = var.auto_upgrade - } -} diff --git a/google/_modules/gke/node_pool/variables.tf b/google/_modules/gke/node_pool/variables.tf deleted file mode 100644 index 7eb53a4d..00000000 --- a/google/_modules/gke/node_pool/variables.tf +++ /dev/null @@ -1,170 +0,0 @@ -variable "project" { - type = string - description = "Project the cluster belongs to." -} - -variable "cluster_name" { - type = string - description = "Name of the cluster for this node pool." -} - -variable "metadata_tags" { - type = list(string) - description = "Metadata tags to use." -} - -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "pool_name" { - description = "Name of the node pool." - type = string -} - -variable "location" { - type = string - description = "location of the cluster this node pool belongs to." -} - -variable "initial_node_count" { - description = "Initial number of nodes for this node pool." - type = string -} - -variable "min_node_count" { - description = "Min number of nodes for this node pool." - type = string -} - -variable "max_node_count" { - description = "Max number of nodes for this node pool." - type = string -} - -variable "location_policy" { - type = string - description = "Location policy specifies the algorithm used when scaling-up the node pool." -} - -variable "service_account_email" { - description = "The service account email to use for this node pool." - type = string - default = null -} - -variable "disable_per_node_pool_service_account" { - description = "Skip creating a dedicated service account to use for this node pool." - type = string - default = false -} - -variable "extra_oauth_scopes" { - description = "List of additional oauth scopes for workers." - type = list(string) -} - -variable "disk_size_gb" { - description = "The disk size of nodes in this pool." - type = string -} - -variable "disk_type" { - description = "The disk type of nodes in this pool." - type = string -} - -variable "image_type" { - description = "The image type for nodes in this pool." - type = string - default = "COS" -} - -variable "machine_type" { - description = "The machine type for nodes in this pool." - type = string -} - -variable "preemptible" { - description = "Whether to use preemptible nodes for this node pool." - type = string - default = false -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = string - default = true -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = string - default = true -} - -variable "node_workload_metadata_config" { - description = "How to expose the node metadata to the workload running on the node." - type = string -} - -variable "taints" { - type = set(object({ - key = string - value = string - effect = string - })) - description = "Taints to configure for the node pool." - default = null -} - -variable "instance_tags" { - type = list(string) - description = "List of instance tags to apply to nodes." - default = [] -} - -variable "node_locations" { - type = list(string) - description = "List of zones in the cluster's region to start worker nodes in. Defaults to cluster's node locations." - default = null -} - -variable "guest_accelerator" { - type = object({ - type = string - count = number - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = optional(string) - max_shared_clients_per_gpu = optional(number) - })) - }) - description = "`guest_accelerator` block supports during node_group creation, useful to provision GPU-capable nodes. Default to `null` or `{}` which will disable GPUs." - default = null -} - -variable "ephemeral_storage_local_ssd_config" { - type = object({ - local_ssd_count = number - }) - description = "`ephemeral_storage_local_ssd_config` block, useful for node groups with local SSD. Defaults to `null`" - default = null -} - -variable "labels" { - type = map(string) - description = "Kubernetes labels to set on the nodes created by the node pool. Merged with Kubestack default labels." - default = {} -} - -variable "network_config" { - type = object({ - enable_private_nodes = bool - create_pod_range = bool - pod_ipv4_cidr_block = string - }) - description = "Configure additional pod IP address range for the node pool. Defaults to `null`" - default = null -} diff --git a/google/_modules/gke/node_pool/versions.tf b/google/_modules/gke/node_pool/versions.tf deleted file mode 100644 index ac97c6ac..00000000 --- a/google/_modules/gke/node_pool/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/google/_modules/gke/outputs.tf b/google/_modules/gke/outputs.tf deleted file mode 100644 index 4ef9ff49..00000000 --- a/google/_modules/gke/outputs.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "kubeconfig" { - value = local.kubeconfig - - # when the node pool is destroyed before the k8s namespaces - # the namespaces get stuck in terminating - depends_on = [module.node_pool] -} - -output "default_ingress_ip" { - value = length(google_compute_address.current) > 0 ? google_compute_address.current[0].address : null -} diff --git a/google/_modules/gke/variables.tf b/google/_modules/gke/variables.tf deleted file mode 100644 index a06143ee..00000000 --- a/google/_modules/gke/variables.tf +++ /dev/null @@ -1,258 +0,0 @@ -variable "project" { - type = string - description = "Project the cluster belongs to." -} - -variable "deletion_protection" { - type = bool - description = "Must be set to false to destroy clusters." -} - -variable "metadata_name" { - type = string - description = "Metadata name to use." -} - -variable "metadata_fqdn" { - type = string - description = "DNS name of the zone. E.g. `infra.example.com`" -} - -variable "metadata_tags" { - type = list(string) - description = "Metadata tags to use." -} - -variable "metadata_labels" { - type = map(string) - description = "Metadata labels to use." -} - -variable "location" { - type = string - description = "Location accepts a region or zone and starts a regional or zonal cluster respectively. Kubestack is only supported for regional clusters." -} - -variable "location_policy" { - type = string - description = "Location policy specifies the algorithm used when scaling-up the node pool." -} - -variable "min_master_version" { - type = string - description = "Minimum GKE master version." -} - -variable "release_channel" { - type = string - description = "The selected release channel. Accepted values are: UNSPECIFIED, RAPID, REGULAR, STABLE" -} - -variable "daily_maintenance_window_start_time" { - type = string - description = "Start time of the daily maintenance window." -} - -variable "maintenance_exclusion_start_time" { - type = string - description = "Maintenance exclusion start time" -} - -variable "maintenance_exclusion_end_time" { - type = string - description = "Maintenance exclusion end time" -} - -variable "maintenance_exclusion_name" { - type = string - description = "Maintenance exclusion name" -} - -variable "maintenance_exclusion_scope" { - type = string - description = "Maintenance exclusion scope" -} - -variable "remove_default_node_pool" { - type = string - description = "Whether to remove the default node pool. Leave true, except for upgrading from Kubestack v0.2.0-beta.0." -} - -variable "initial_node_count" { - description = "Initial number of nodes for this node pool." - type = string -} - -variable "min_node_count" { - description = "Min number of nodes for this node pool." - type = string -} - -variable "max_node_count" { - description = "Max number of nodes for this node pool." - type = string -} - -variable "node_locations" { - type = list(string) - description = "List of zones in the cluster's region to start worker nodes in." -} - -variable "extra_oauth_scopes" { - description = "List of additional oauth scopes for workers." - type = list(string) -} - -variable "disk_size_gb" { - description = "The disk size of nodes in this pool." - type = string -} - -variable "disk_type" { - description = "The disk type of nodes in this pool." - type = string -} - -variable "image_type" { - description = "The image type for nodes in this pool." - type = string - default = "COS_containerd" -} - -variable "machine_type" { - description = "The machine type for nodes in this pool." - type = string -} - -variable "preemptible" { - description = "Whether to use preemptible nodes for this node pool." - type = string -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = string -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = string -} - -variable "disable_default_ingress" { - type = bool - description = "Whether to disable the default ingress." -} - -variable "enable_private_nodes" { - type = bool - description = "Whether to enable private nodes" -} - -variable "master_cidr_block" { - type = string - description = "The IP range for the master network" -} - -variable "cluster_ipv4_cidr_block" { - type = string - description = "The CIDR block used for Kubernetes pods." -} - -variable "services_ipv4_cidr_block" { - type = string - description = "The CIDR block used for Kubernetes services." -} - -variable "enable_cloud_nat" { - type = bool - description = "Whether to enable cloud nat and allow internet access for private nodes." -} - -variable "cloud_nat_endpoint_independent_mapping" { - type = bool - description = "Whether to enable cloud nat endpoint independent mapping" -} - -variable "cloud_nat_min_ports_per_vm" { - type = number - description = "The min amount of ports per VM allowed for NAT mapping" -} - -variable "cloud_nat_ip_count" { - type = number - description = "The amount of IP addresses to attach to the NAT gateway, changes NAT to use MANUAL_ONLY" -} - -variable "disable_workload_identity" { - description = "Wheter to disable workload identity support." - type = bool -} - -variable "node_workload_metadata_config" { - description = "How to expose the node metadata to the workload running on the node." - type = string -} - -variable "master_authorized_networks_config_cidr_blocks" { - description = "The list of CIDR blocks of external networks that can access the Kubernetes cluster master through HTTPS. Setting cidr_blocks to an empty list disallows public access." - type = list(string) -} - -variable "enable_intranode_visibility" { - description = "Whether to enable GKE intranode visibility." - type = bool -} - -variable "enable_tpu" { - description = "Whether to enable GKE cloud TPU support." - type = bool -} - -variable "cluster_database_encryption_key_name" { - type = string - description = "Cloud KMS key name for enabling cluster database encryption." -} - -variable "router_advertise_config" { - description = "Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions." - type = object({ - groups = list(string) - ip_ranges = map(string) - mode = string - }) -} - -variable "router_asn" { - description = "Router ASN used for auto-created router." - type = number -} - -variable "logging_config_enable_components" { - description = "Logging config components to enable." - type = list(string) -} - -variable "monitoring_config_enable_components" { - description = "Monitoring config components to enable." - type = list(string) -} - -variable "enable_gcs_fuse_csi_driver" { - description = "Whether to enable GCSFuse CSI driver addon." - type = bool -} - -variable "guest_accelerator" { - type = object({ - type = string - count = number - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = optional(string) - max_shared_clients_per_gpu = optional(number) - })) - }) - description = "`guest_accelerator` block supports during node_group creation, useful to provision GPU-capable nodes. Default to `null` or `{}` which will disable GPUs." - default = null -} \ No newline at end of file diff --git a/google/_modules/gke/versions.tf b/google/_modules/gke/versions.tf deleted file mode 100644 index 3496e5a4..00000000 --- a/google/_modules/gke/versions.tf +++ /dev/null @@ -1,16 +0,0 @@ - -terraform { - required_providers { - google = { - # https://registry.terraform.io/providers/hashicorp/google/latest - source = "hashicorp/google" - version = ">= 4.76.0" - } - - kubernetes = { - source = "hashicorp/kubernetes" - } - } - - required_version = ">= 0.13" -} diff --git a/google/cluster-local/configuration.tf b/google/cluster-local/configuration.tf deleted file mode 100644 index c469498d..00000000 --- a/google/cluster-local/configuration.tf +++ /dev/null @@ -1,42 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - # while we have the real region for GKE - # we still hash and prefix it with gke- - # to align with the local implementations - # for AKS end EKS - fake_region_hash = substr(sha256(local.cfg["region"]), 0, 7) - fake_region = "gke-${local.fake_region_hash}" - - http_port_default = terraform.workspace == "apps" ? 80 : 8080 - http_port = lookup(local.cfg, "http_port", local.http_port_default) - - https_port_default = terraform.workspace == "apps" ? 443 : 8443 - https_port = lookup(local.cfg, "https_port", local.https_port_default) - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - node_image = lookup(local.cfg, "node_image", null) - - # technically it should be min_node_count times number of AZs - # but it seems better to keep node count low in the dev env - node_count = lookup(local.cfg, "cluster_min_node_count", 1) - nodes = [ - for node, _ in range(local.node_count) : - "worker" - ] - extra_nodes = join(",", local.nodes) - -} diff --git a/google/cluster-local/main.tf b/google/cluster-local/main.tf deleted file mode 100644 index 1dfa3cf0..00000000 --- a/google/cluster-local/main.tf +++ /dev/null @@ -1,26 +0,0 @@ -module "cluster_metadata" { - source = "../../common/metadata" - - name_prefix = local.name_prefix - base_domain = local.base_domain - - provider_name = "gcp" - provider_region = local.fake_region -} - -module "cluster" { - source = "../../kind/_modules/kind" - - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_tags = module.cluster_metadata.tags - metadata_labels = module.cluster_metadata.labels - - node_image = local.node_image - extra_nodes = local.extra_nodes - - http_port = local.http_port - https_port = local.https_port - - disable_default_ingress = local.disable_default_ingress -} diff --git a/google/cluster-local/outputs.tf b/google/cluster-local/outputs.tf deleted file mode 100644 index cba3bd30..00000000 --- a/google/cluster-local/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -output "current_config" { - value = module.configuration.merged[terraform.workspace] -} - -output "current_metadata" { - value = module.cluster_metadata -} - -output "kubeconfig" { - value = module.cluster.kubeconfig -} - -output "default_ingress_ip" { - # the cluster module returns an IP as a string - # we YAML encode null for cluster-local to provide - # a unified output to consumers - value = yamlencode(null) -} diff --git a/google/cluster-local/variables.tf b/google/cluster-local/variables.tf deleted file mode 100644 index 8bf95161..00000000 --- a/google/cluster-local/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "configuration" { - type = map(map(string)) - description = "Map with per workspace cluster configuration." -} - -variable "configuration_base_key" { - type = string - description = "The key in the configuration map all other keys inherit from." - default = "apps" -} diff --git a/google/cluster-local/versions.tf b/google/cluster-local/versions.tf deleted file mode 100644 index 2606a5ac..00000000 --- a/google/cluster-local/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.13" -} diff --git a/google/_modules/gke/cluster_role_binding.tf b/google/cluster/cluster_role_binding.tf similarity index 100% rename from google/_modules/gke/cluster_role_binding.tf rename to google/cluster/cluster_role_binding.tf diff --git a/google/cluster/configuration.tf b/google/cluster/configuration.tf deleted file mode 100644 index f33be750..00000000 --- a/google/cluster/configuration.tf +++ /dev/null @@ -1,104 +0,0 @@ -module "configuration" { - source = "../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - name_prefix = local.cfg["name_prefix"] - - base_domain = local.cfg["base_domain"] - - project_id = local.cfg["project_id"] - - region = local.cfg["region"] - - deletion_protection = lookup(local.cfg, "deletion_protection", null) - - cluster_node_locations_lookup = lookup(local.cfg, "cluster_node_locations", "") - cluster_node_locations = split(",", local.cluster_node_locations_lookup) - - cluster_min_master_version = local.cfg["cluster_min_master_version"] - cluster_release_channel = lookup(local.cfg, "cluster_release_channel", "STABLE") - - cluster_daily_maintenance_window_start_time = lookup( - local.cfg, - "cluster_daily_maintenance_window_start_time", - "03:00", - ) - - cluster_maintenance_exclusion_start_time = lookup(local.cfg, "cluster_maintenance_exclusion_start_time", "") - cluster_maintenance_exclusion_end_time = lookup(local.cfg, "cluster_maintenance_exclusion_end_time", "") - cluster_maintenance_exclusion_name = lookup(local.cfg, "cluster_maintenance_exclusion_name", "") - cluster_maintenance_exclusion_scope = lookup(local.cfg, "cluster_maintenance_exclusion_scope", "") - - remove_default_node_pool = lookup(local.cfg, "remove_default_node_pool", true) - - cluster_initial_node_count = lookup(local.cfg, "cluster_initial_node_count", 1) - - cluster_min_node_count = lookup(local.cfg, "cluster_min_node_count", 1) - cluster_max_node_count = lookup(local.cfg, "cluster_max_node_count", 1) - cluster_node_location_policy = lookup(local.cfg, "cluster_node_location_policy", null) - - cluster_extra_oauth_scopes_lookup = lookup(local.cfg, "cluster_extra_oauth_scopes", "") - cluster_extra_oauth_scopes = split(",", local.cluster_extra_oauth_scopes_lookup) - - cluster_disk_size_gb = lookup(local.cfg, "cluster_disk_size_gb", 100) - - cluster_disk_type = lookup(local.cfg, "cluster_disk_type", "pd-standard") - - cluster_image_type = lookup(local.cfg, "cluster_image_type", null) - - cluster_machine_type = lookup(local.cfg, "cluster_machine_type", "") - - cluster_preemptible = lookup(local.cfg, "cluster_preemptible", false) - - cluster_auto_repair = lookup(local.cfg, "cluster_auto_repair", true) - - cluster_auto_upgrade = lookup(local.cfg, "cluster_auto_upgrade", true) - - disable_default_ingress = lookup(local.cfg, "disable_default_ingress", false) - - enable_private_nodes = lookup(local.cfg, "enable_private_nodes", true) - master_cidr_block = lookup(local.cfg, "master_cidr_block", "172.16.0.32/28") - - cluster_ipv4_cidr_block = lookup(local.cfg, "cluster_ipv4_cidr_block", null) - services_ipv4_cidr_block = lookup(local.cfg, "services_ipv4_cidr_block", null) - - cluster_database_encryption_key_name = lookup(local.cfg, "cluster_database_encryption_key_name", null) - - # by default include cloud_nat when private nodes are enabled - enable_cloud_nat = lookup(local.cfg, "enable_cloud_nat", local.enable_private_nodes) - cloud_nat_endpoint_independent_mapping = lookup(local.cfg, "cloud_nat_enable_endpoint_independent_mapping", null) - cloud_nat_min_ports_per_vm = lookup(local.cfg, "cloud_nat_min_ports_per_vm", null) - cloud_nat_ip_count = lookup(local.cfg, "cloud_nat_ip_count", 0) - - disable_workload_identity = lookup(local.cfg, "disable_workload_identity", false) - default_node_workload_metadata_config = tobool(local.disable_workload_identity) == false ? "GKE_METADATA" : "MODE_UNSPECIFIED" - node_workload_metadata_config = lookup(local.cfg, "node_workload_metadata_config", local.default_node_workload_metadata_config) - - master_authorized_networks_config_cidr_blocks_lookup = lookup(local.cfg, "master_authorized_networks_config_cidr_blocks", null) - master_authorized_networks_config_cidr_blocks = local.master_authorized_networks_config_cidr_blocks_lookup == null ? null : split(",", local.master_authorized_networks_config_cidr_blocks_lookup) - - enable_intranode_visibility = lookup(local.cfg, "enable_intranode_visibility", false) - enable_tpu = lookup(local.cfg, "enable_tpu", false) - - router_advertise_config_groups_lookup = lookup(local.cfg, "router_advertise_config_groups", "") - router_advertise_config_groups = compact(split(",", local.router_advertise_config_groups_lookup)) - router_advertise_config_ip_ranges_lookup = lookup(local.cfg, "router_advertise_config_ip_ranges", "") - router_advertise_config_ip_ranges = compact(split(",", local.router_advertise_config_ip_ranges_lookup)) - router_advertise_config_mode = lookup(local.cfg, "router_advertise_config_mode", null) - router_asn = lookup(local.cfg, "router_asn", null) - - logging_config_enable_components_lookup = lookup(local.cfg, "logging_config_enable_components", "SYSTEM_COMPONENTS,WORKLOADS") - logging_config_enable_components = compact(split(",", local.logging_config_enable_components_lookup)) - - monitoring_config_enable_components_lookup = lookup(local.cfg, "monitoring_config_enable_components", "SYSTEM_COMPONENTS") - monitoring_config_enable_components = compact(split(",", local.monitoring_config_enable_components_lookup)) - - enable_gcs_fuse_csi_driver = lookup(local.cfg, "enable_gcs_fuse_csi_driver", null) -} diff --git a/google/_modules/gke/ingress.tf b/google/cluster/ingress.tf similarity index 52% rename from google/_modules/gke/ingress.tf rename to google/cluster/ingress.tf index f0c6f1fd..27432489 100644 --- a/google/_modules/gke/ingress.tf +++ b/google/cluster/ingress.tf @@ -1,25 +1,25 @@ resource "google_compute_address" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 region = google_container_cluster.current.location - project = var.project + project = local.cfg.project_id - name = var.metadata_name + name = module.cluster_metadata.name } resource "google_dns_managed_zone" "current" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - project = var.project + project = local.cfg.project_id - name = var.metadata_name - dns_name = "${var.metadata_fqdn}." + name = module.cluster_metadata.name + dns_name = "${module.cluster_metadata.fqdn}." } resource "google_dns_record_set" "host" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - project = var.project + project = local.cfg.project_id name = google_dns_managed_zone.current[0].dns_name type = "A" @@ -31,9 +31,9 @@ resource "google_dns_record_set" "host" { } resource "google_dns_record_set" "wildcard" { - count = var.disable_default_ingress ? 0 : 1 + count = try(coalesce(local.cfg.disable_default_ingress, null), false) ? 0 : 1 - project = var.project + project = local.cfg.project_id name = "*.${google_dns_managed_zone.current[0].dns_name}" type = "A" diff --git a/google/_modules/gke/kubeconfig.tf b/google/cluster/kubeconfig.tf similarity index 100% rename from google/_modules/gke/kubeconfig.tf rename to google/cluster/kubeconfig.tf diff --git a/google/cluster/main.tf b/google/cluster/main.tf index 8c39748b..8cc49879 100644 --- a/google/cluster/main.tf +++ b/google/cluster/main.tf @@ -1,91 +1,151 @@ +module "configuration" { + source = "../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] +} + module "cluster_metadata" { source = "../../common/metadata" - name_prefix = local.name_prefix - base_domain = local.base_domain + name_prefix = local.cfg.name_prefix + base_domain = local.cfg.base_domain provider_name = "gcp" - provider_region = local.region + provider_region = local.cfg.region } -module "cluster" { - source = "../_modules/gke" - - project = local.project_id +resource "google_container_cluster" "current" { + project = local.cfg.project_id + name = module.cluster_metadata.name - deletion_protection = local.deletion_protection + deletion_protection = local.cfg.deletion_protection - metadata_name = module.cluster_metadata.name - metadata_fqdn = module.cluster_metadata.fqdn - metadata_tags = module.cluster_metadata.tags - metadata_labels = module.cluster_metadata.labels + location = local.cfg.region + node_locations = try(coalesce(local.cfg.cluster_node_locations, null), []) - location = local.region - node_locations = local.cluster_node_locations + min_master_version = local.cfg.cluster_min_master_version - min_master_version = local.cluster_min_master_version - release_channel = local.cluster_release_channel + release_channel { + channel = try(coalesce(local.cfg.cluster_release_channel, null), "STABLE") + } - daily_maintenance_window_start_time = local.cluster_daily_maintenance_window_start_time + remove_default_node_pool = true + initial_node_count = 1 - maintenance_exclusion_start_time = local.cluster_maintenance_exclusion_start_time - maintenance_exclusion_end_time = local.cluster_maintenance_exclusion_end_time - maintenance_exclusion_name = local.cluster_maintenance_exclusion_name - maintenance_exclusion_scope = local.cluster_maintenance_exclusion_scope + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } - remove_default_node_pool = local.remove_default_node_pool + network = google_compute_network.current.self_link - initial_node_count = local.cluster_initial_node_count - min_node_count = local.cluster_min_node_count - max_node_count = local.cluster_max_node_count - location_policy = local.cluster_node_location_policy + dynamic "workload_identity_config" { + for_each = try(coalesce(local.cfg.disable_workload_identity, null), false) == false ? toset([1]) : toset([]) + content { + workload_pool = "${local.cfg.project_id}.svc.id.goog" + } + } - extra_oauth_scopes = local.cluster_extra_oauth_scopes + dynamic "database_encryption" { + for_each = local.cfg.cluster_database_encryption_key_name != null ? [1] : [] + content { + state = "ENCRYPTED" + key_name = local.cfg.cluster_database_encryption_key_name + } + } - disk_size_gb = local.cluster_disk_size_gb - disk_type = local.cluster_disk_type - image_type = local.cluster_image_type - machine_type = local.cluster_machine_type + # + # + # Addon config + addons_config { + http_load_balancing { + disabled = true + } + + horizontal_pod_autoscaling { + disabled = false + } + + network_policy_config { + disabled = false + } + + dynamic "gcs_fuse_csi_driver_config" { + for_each = local.cfg.enable_gcs_fuse_csi_driver != null ? [1] : [] + + content { + enabled = local.cfg.enable_gcs_fuse_csi_driver + } + } + } - preemptible = local.cluster_preemptible + network_policy { + enabled = true + } - auto_repair = local.cluster_auto_repair + maintenance_policy { + daily_maintenance_window { + start_time = try(coalesce(local.cfg.cluster_daily_maintenance_window_start_time, null), "03:00") + } - auto_upgrade = local.cluster_auto_upgrade + dynamic "maintenance_exclusion" { + for_each = try(local.cfg.cluster_maintenance_exclusion.start_time, "") != "" ? [1] : [] - disable_default_ingress = local.disable_default_ingress + content { + start_time = try(local.cfg.cluster_maintenance_exclusion.start_time, "") + end_time = try(local.cfg.cluster_maintenance_exclusion.end_time, "") + exclusion_name = try(local.cfg.cluster_maintenance_exclusion.name, "") - enable_private_nodes = local.enable_private_nodes - master_cidr_block = local.master_cidr_block + exclusion_options { + scope = try(local.cfg.cluster_maintenance_exclusion.scope, "") + } + } + } + } - cluster_ipv4_cidr_block = local.cluster_ipv4_cidr_block - services_ipv4_cidr_block = local.services_ipv4_cidr_block + dynamic "master_authorized_networks_config" { + for_each = local.cfg.master_authorized_networks_config_cidr_blocks == null ? [] : [1] - enable_cloud_nat = local.enable_cloud_nat - cloud_nat_endpoint_independent_mapping = local.cloud_nat_endpoint_independent_mapping - cloud_nat_ip_count = local.cloud_nat_ip_count + content { + dynamic "cidr_blocks" { + for_each = local.cfg.master_authorized_networks_config_cidr_blocks - master_authorized_networks_config_cidr_blocks = local.master_authorized_networks_config_cidr_blocks + content { + cidr_block = cidr_blocks.value + display_name = "terraform-kubestack_${cidr_blocks.value}" + } + } + } + } - cloud_nat_min_ports_per_vm = local.cloud_nat_min_ports_per_vm + logging_config { + enable_components = try(coalesce(local.cfg.logging_config.enable_components, null), ["SYSTEM_COMPONENTS", "WORKLOADS"]) + } - disable_workload_identity = local.disable_workload_identity - node_workload_metadata_config = local.node_workload_metadata_config + monitoring_config { + enable_components = try(coalesce(local.cfg.monitoring_config.enable_components, null), ["SYSTEM_COMPONENTS"]) + } - cluster_database_encryption_key_name = local.cluster_database_encryption_key_name + private_cluster_config { + enable_private_nodes = try(coalesce(local.cfg.enable_private_nodes, null), true) + enable_private_endpoint = try(coalesce(local.cfg.enable_private_endpoint, null), false) + master_ipv4_cidr_block = try(coalesce(local.cfg.master_cidr_block, null), "172.16.0.32/28") + } - enable_intranode_visibility = local.enable_intranode_visibility - enable_tpu = local.enable_tpu + dynamic "ip_allocation_policy" { + for_each = try(coalesce(local.cfg.enable_private_nodes, null), true) ? [1] : [] - router_advertise_config = { - groups = local.router_advertise_config_groups - ip_ranges = { for ip in local.router_advertise_config_ip_ranges : ip => null } - mode = local.router_advertise_config_mode + content { + cluster_ipv4_cidr_block = local.cfg.cluster_ipv4_cidr_block + services_ipv4_cidr_block = local.cfg.services_ipv4_cidr_block + } } - router_asn = local.router_asn - - logging_config_enable_components = local.logging_config_enable_components - monitoring_config_enable_components = local.monitoring_config_enable_components - enable_gcs_fuse_csi_driver = local.enable_gcs_fuse_csi_driver + enable_intranode_visibility = try(coalesce(local.cfg.enable_intranode_visibility, null), false) + enable_tpu = try(coalesce(local.cfg.enable_tpu, null), false) } diff --git a/google/cluster/moved.tf b/google/cluster/moved.tf new file mode 100644 index 00000000..4ec91436 --- /dev/null +++ b/google/cluster/moved.tf @@ -0,0 +1,69 @@ +moved { + from = module.cluster.google_container_cluster.current + to = google_container_cluster.current +} + +moved { + from = module.cluster.google_compute_network.current + to = google_compute_network.current +} + +moved { + from = module.cluster.google_compute_address.nat + to = google_compute_address.nat +} + +moved { + from = module.cluster.google_compute_router.current + to = google_compute_router.current +} + +moved { + from = module.cluster.google_compute_router_nat.nat + to = google_compute_router_nat.nat +} + +moved { + from = module.cluster.google_service_account.current + to = google_service_account.current +} + +moved { + from = module.cluster.google_project_iam_member.log_writer + to = google_project_iam_member.log_writer +} + +moved { + from = module.cluster.google_project_iam_member.metric_writer + to = google_project_iam_member.metric_writer +} + +moved { + from = module.cluster.google_compute_address.current + to = google_compute_address.current +} + +moved { + from = module.cluster.google_dns_managed_zone.current + to = google_dns_managed_zone.current +} + +moved { + from = module.cluster.google_dns_record_set.host + to = google_dns_record_set.host +} + +moved { + from = module.cluster.google_dns_record_set.wildcard + to = google_dns_record_set.wildcard +} + +moved { + from = module.cluster.kubernetes_cluster_role_binding.current + to = kubernetes_cluster_role_binding.current +} + +moved { + from = module.cluster.module.node_pool + to = module.node_pool +} diff --git a/google/cluster/network.tf b/google/cluster/network.tf new file mode 100644 index 00000000..cb64d802 --- /dev/null +++ b/google/cluster/network.tf @@ -0,0 +1,78 @@ +resource "google_compute_network" "current" { + name = module.cluster_metadata.name + project = local.cfg.project_id + auto_create_subnetworks = true +} + +resource "google_compute_address" "nat" { + count = try(coalesce(local.cfg.enable_cloud_nat, null), true) ? try(coalesce(local.cfg.cloud_nat_ip_count, null), 0) : 0 + + region = google_container_cluster.current.location + project = local.cfg.project_id + + name = "nat-${module.cluster_metadata.name}-${count.index}" +} + +resource "google_compute_router" "current" { + count = try(coalesce(local.cfg.enable_cloud_nat, null), true) ? 1 : 0 + + project = local.cfg.project_id + name = module.cluster_metadata.name + region = google_container_cluster.current.location + + network = google_compute_network.current.name + + bgp { + advertise_mode = ( + local.cfg.router_advertise_config == null + ? null + : local.cfg.router_advertise_config.mode + ) + advertised_groups = ( + local.cfg.router_advertise_config == null ? null : ( + local.cfg.router_advertise_config.mode != "CUSTOM" + ? null + : try(coalesce(local.cfg.router_advertise_config.groups, null), toset([])) + ) + ) + dynamic "advertised_ip_ranges" { + for_each = ( + local.cfg.router_advertise_config == null ? [] : ( + local.cfg.router_advertise_config.mode != "CUSTOM" + ? [] + : try(coalesce(local.cfg.router_advertise_config.ip_ranges, null), tomap({})) + ) + ) + iterator = range + content { + range = range.key + description = range.value + } + } + + # expected "bgp.0.asn" to be a RFC6996-compliant Local ASN: + # must be either in the private ASN ranges: [64512..65534], [4200000000..4294967294]; + # or be the value of [16550] + asn = local.cfg.router_asn != null ? local.cfg.router_asn : 16550 + } +} + +resource "google_compute_router_nat" "nat" { + count = try(coalesce(local.cfg.enable_cloud_nat, null), true) ? 1 : 0 + + project = local.cfg.project_id + name = module.cluster_metadata.name + region = google_compute_router.current[0].region + router = google_compute_router.current[0].name + + enable_endpoint_independent_mapping = local.cfg.cloud_nat_enable_endpoint_independent_mapping + min_ports_per_vm = local.cfg.cloud_nat_min_ports_per_vm + nat_ip_allocate_option = try(coalesce(local.cfg.cloud_nat_ip_count, null), 0) > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + nat_ips = try(coalesce(local.cfg.cloud_nat_ip_count, null), 0) > 0 ? google_compute_address.nat[*].self_link : null + + log_config { + enable = true + filter = "ERRORS_ONLY" + } +} diff --git a/google/cluster/node-pool/configuration.tf b/google/cluster/node-pool/configuration.tf deleted file mode 100644 index 2b4f843a..00000000 --- a/google/cluster/node-pool/configuration.tf +++ /dev/null @@ -1,49 +0,0 @@ -module "configuration" { - source = "../../../common/configuration" - - configuration = var.configuration - base_key = var.configuration_base_key -} - -locals { - # current workspace config - cfg = module.configuration.merged[terraform.workspace] - - project_id = local.cfg["project_id"] - - name = lookup(local.cfg, "name") - - location = local.cfg["location"] - node_locations = local.cfg["node_locations"] - - initial_node_count = local.cfg["initial_node_count"] - min_node_count = local.cfg["min_node_count"] - max_node_count = local.cfg["max_node_count"] - location_policy = local.cfg["location_policy"] != null ? local.cfg["location_policy"] : "BALANCED" - - disk_size_gb = local.cfg["disk_size_gb"] - disk_type = local.cfg["disk_type"] - image_type = local.cfg["image_type"] - machine_type = local.cfg["machine_type"] - - preemptible = local.cfg["preemptible"] != null ? local.cfg["preemptible"] : false - auto_repair = local.cfg["auto_repair"] != null ? local.cfg["auto_repair"] : true - auto_upgrade = local.cfg["auto_upgrade"] != null ? local.cfg["auto_upgrade"] : true - - taints = local.cfg["taints"] - - labels = local.cfg["labels"] - - extra_oauth_scopes = local.cfg["extra_oauth_scopes"] != null ? local.cfg["extra_oauth_scopes"] : [] - - node_workload_metadata_config = local.cfg["node_workload_metadata_config"] != null ? local.cfg["node_workload_metadata_config"] : "GKE_METADATA" - - service_account_email = local.cfg["service_account_email"] - - ephemeral_storage_local_ssd_config = local.cfg["ephemeral_storage_local_ssd_config"] - - guest_accelerator = local.cfg["guest_accelerator"] - network_config = local.cfg["network_config"] - - instance_tags = local.cfg["instance_tags"] != null ? local.cfg["instance_tags"] : [] -} diff --git a/google/_modules/gke/node_pool/locals.tf b/google/cluster/node-pool/locals.tf similarity index 76% rename from google/_modules/gke/node_pool/locals.tf rename to google/cluster/node-pool/locals.tf index 88d70a0f..b3872f6a 100644 --- a/google/_modules/gke/node_pool/locals.tf +++ b/google/cluster/node-pool/locals.tf @@ -8,6 +8,5 @@ locals { "https://www.googleapis.com/auth/trace.append", ] - oauth_scopes = compact(concat(local.base_oauth_scopes, var.extra_oauth_scopes)) + oauth_scopes = compact(concat(local.base_oauth_scopes, try(coalesce(local.cfg.extra_oauth_scopes, null), []))) } - diff --git a/google/cluster/node-pool/main.tf b/google/cluster/node-pool/main.tf index c6cf0266..8e18cc53 100644 --- a/google/cluster/node-pool/main.tf +++ b/google/cluster/node-pool/main.tf @@ -1,46 +1,104 @@ -module "node_pool" { - source = "../../_modules/gke/node_pool" +module "configuration" { + source = "../../../common/configuration" + configuration = var.configuration + base_key = var.configuration_base_key +} + +locals { + cfg = module.configuration.merged[terraform.workspace] + + disable_per_node_pool_service_account = local.cfg.service_account_email == null ? false : true +} + +resource "google_container_node_pool" "current" { + name = local.cfg.name + project = local.cfg.project_id + cluster = var.cluster.name + location = local.cfg.location + + initial_node_count = local.cfg.initial_node_count + + autoscaling { + min_node_count = try(coalesce(local.cfg.min_node_count, null), 1) + max_node_count = try(coalesce(local.cfg.max_node_count, null), 1) + location_policy = try(coalesce(local.cfg.location_policy, null), "BALANCED") + } + + node_locations = try(coalesce(local.cfg.node_locations, null), []) + + dynamic "network_config" { + for_each = local.cfg.network_config == null ? [] : [1] + + content { + enable_private_nodes = local.cfg.network_config.enable_private_nodes + create_pod_range = local.cfg.network_config.create_pod_range + pod_ipv4_cidr_block = local.cfg.network_config.pod_ipv4_cidr_block + } + } + + # + # + # Node config + node_config { + service_account = local.disable_per_node_pool_service_account ? local.cfg.service_account_email : google_service_account.current[0].email - project = local.project_id + oauth_scopes = local.oauth_scopes - location = local.location - node_locations = local.node_locations + disk_size_gb = try(coalesce(local.cfg.disk_size_gb, null), 100) + disk_type = try(coalesce(local.cfg.disk_type, null), "pd-balanced") - cluster_name = var.cluster_metadata["name"] - pool_name = local.name + image_type = try(coalesce(local.cfg.image_type, null), "COS_CONTAINERD") + machine_type = try(coalesce(local.cfg.machine_type, null), "") + preemptible = try(coalesce(local.cfg.preemptible, null), false) - metadata_tags = var.cluster_metadata["tags"] - metadata_labels = var.cluster_metadata["labels"] + labels = merge(try(coalesce(local.cfg.labels, null), {}), var.cluster_metadata.labels) - initial_node_count = local.initial_node_count - min_node_count = local.min_node_count - max_node_count = local.max_node_count - location_policy = local.location_policy + tags = concat(var.cluster_metadata.tags, try(coalesce(local.cfg.instance_tags, null), [])) - extra_oauth_scopes = local.extra_oauth_scopes + workload_metadata_config { + mode = try(coalesce(local.cfg.node_workload_metadata_config, null), "GKE_METADATA") + } - disk_size_gb = local.disk_size_gb - disk_type = local.disk_type - image_type = local.image_type - machine_type = local.machine_type + dynamic "guest_accelerator" { + # Make sure to generate this only once + for_each = local.cfg.guest_accelerator == null ? [] : [1] - preemptible = local.preemptible - auto_repair = local.auto_repair - auto_upgrade = local.auto_upgrade + content { + type = local.cfg.guest_accelerator.type + count = local.cfg.guest_accelerator.count - node_workload_metadata_config = local.node_workload_metadata_config + dynamic "gpu_sharing_config" { + for_each = local.cfg.guest_accelerator.gpu_sharing_config == null ? [] : [1] - taints = local.taints - instance_tags = local.instance_tags + content { + gpu_sharing_strategy = local.cfg.guest_accelerator.gpu_sharing_config.gpu_sharing_strategy + max_shared_clients_per_gpu = local.cfg.guest_accelerator.gpu_sharing_config.max_shared_clients_per_gpu + } + } + } + } - labels = local.labels + dynamic "taint" { + for_each = try(coalesce(local.cfg.taints, null), toset([])) == null ? [] : try(coalesce(local.cfg.taints, null), toset([])) - service_account_email = local.service_account_email - disable_per_node_pool_service_account = local.service_account_email == null ? false : true + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } - ephemeral_storage_local_ssd_config = local.ephemeral_storage_local_ssd_config + dynamic "ephemeral_storage_local_ssd_config" { + for_each = local.cfg.ephemeral_storage_local_ssd_config == null ? [] : [1] - guest_accelerator = local.guest_accelerator + content { + local_ssd_count = local.cfg.ephemeral_storage_local_ssd_config.local_ssd_count + } + } + } - network_config = local.network_config + management { + auto_repair = try(coalesce(local.cfg.auto_repair, null), true) + auto_upgrade = try(coalesce(local.cfg.auto_upgrade, null), true) + } } diff --git a/google/cluster/node-pool/moved.tf b/google/cluster/node-pool/moved.tf new file mode 100644 index 00000000..77dd9de7 --- /dev/null +++ b/google/cluster/node-pool/moved.tf @@ -0,0 +1,19 @@ +moved { + from = module.node_pool.google_container_node_pool.current + to = google_container_node_pool.current +} + +moved { + from = module.node_pool.google_service_account.current + to = google_service_account.current +} + +moved { + from = module.node_pool.google_project_iam_member.log_writer + to = google_project_iam_member.log_writer +} + +moved { + from = module.node_pool.google_project_iam_member.metric_writer + to = google_project_iam_member.metric_writer +} diff --git a/google/_modules/gke/node_pool/outputs.tf b/google/cluster/node-pool/outputs.tf similarity index 69% rename from google/_modules/gke/node_pool/outputs.tf rename to google/cluster/node-pool/outputs.tf index 10dcb4a0..61201493 100644 --- a/google/_modules/gke/node_pool/outputs.tf +++ b/google/cluster/node-pool/outputs.tf @@ -1,3 +1,7 @@ +output "current_config" { + value = local.cfg +} + output "id" { value = google_container_node_pool.current.name description = "ID of the node pool." diff --git a/google/_modules/gke/node_pool/service_account.tf b/google/cluster/node-pool/service_account.tf similarity index 61% rename from google/_modules/gke/node_pool/service_account.tf rename to google/cluster/node-pool/service_account.tf index 5c413742..c4b97160 100644 --- a/google/_modules/gke/node_pool/service_account.tf +++ b/google/cluster/node-pool/service_account.tf @@ -1,28 +1,28 @@ locals { - account_id_prefix = join("-", [var.pool_name, var.cluster_name]) + account_id_prefix = join("-", [local.cfg.name, var.cluster_metadata.name]) account_id_suffix = sha512(local.account_id_prefix) account_id = "${substr(local.account_id_prefix, 0, 24)}-${substr(local.account_id_suffix, 0, 5)}" } resource "google_service_account" "current" { - count = var.disable_per_node_pool_service_account ? 0 : 1 + count = local.disable_per_node_pool_service_account ? 0 : 1 account_id = local.account_id - project = var.project + project = local.cfg.project_id } resource "google_project_iam_member" "log_writer" { - count = var.disable_per_node_pool_service_account ? 0 : 1 + count = local.disable_per_node_pool_service_account ? 0 : 1 - project = var.project + project = local.cfg.project_id role = "roles/logging.logWriter" member = "serviceAccount:${google_service_account.current[0].email}" } resource "google_project_iam_member" "metric_writer" { - count = var.disable_per_node_pool_service_account ? 0 : 1 + count = local.disable_per_node_pool_service_account ? 0 : 1 - project = var.project + project = local.cfg.project_id role = "roles/monitoring.metricWriter" member = "serviceAccount:${google_service_account.current[0].email}" } diff --git a/google/cluster/node-pool/variables.tf b/google/cluster/node-pool/variables.tf index 190be1fd..50157750 100644 --- a/google/cluster/node-pool/variables.tf +++ b/google/cluster/node-pool/variables.tf @@ -8,11 +8,11 @@ variable "configuration" { node_locations = optional(list(string)) location_policy = optional(string) - initial_node_count = optional(string) - min_node_count = optional(string) - max_node_count = optional(string) + initial_node_count = optional(number) + min_node_count = optional(number) + max_node_count = optional(number) - disk_size_gb = optional(string) + disk_size_gb = optional(number) disk_type = optional(string) image_type = optional(string) machine_type = optional(string) @@ -59,15 +59,24 @@ variable "configuration" { })) description = "Map with per workspace cluster configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false +} + +variable "cluster" { + type = any + description = "The cluster output from the cluster module." + nullable = false } variable "cluster_metadata" { type = any description = "Metadata of the cluster to attach the node pool to." + nullable = false } diff --git a/google/cluster/node-pool/versions.tf b/google/cluster/node-pool/versions.tf index 3e8b8534..2ad954b3 100644 --- a/google/cluster/node-pool/versions.tf +++ b/google/cluster/node-pool/versions.tf @@ -1,6 +1,6 @@ terraform { required_providers { - aws = { + google = { source = "hashicorp/google" } } diff --git a/google/cluster/node_pool.tf b/google/cluster/node_pool.tf new file mode 100644 index 00000000..f4d5751e --- /dev/null +++ b/google/cluster/node_pool.tf @@ -0,0 +1,45 @@ +module "node_pool" { + source = "./node-pool" + + cluster = google_container_cluster.current + + configuration = { + (terraform.workspace) = { + project_id = local.cfg.project_id + location = google_container_cluster.current.location + node_locations = local.cfg.cluster_node_locations + name = "default" + + service_account_email = google_service_account.current.email + + initial_node_count = try(coalesce(local.cfg.default_node_pool.initial_node_count, null), 1) + min_node_count = try(coalesce(local.cfg.default_node_pool.min_node_count, null), 1) + max_node_count = try(coalesce(local.cfg.default_node_pool.max_node_count, null), 1) + location_policy = try(local.cfg.default_node_pool.node_location_policy, null) + + extra_oauth_scopes = try(coalesce(local.cfg.default_node_pool.extra_oauth_scopes, null), []) + + disk_size_gb = try(coalesce(local.cfg.default_node_pool.disk_size_gb, null), 100) + disk_type = try(coalesce(local.cfg.default_node_pool.disk_type, null), "pd-standard") + image_type = try(coalesce(local.cfg.default_node_pool.image_type, null), "COS_containerd") + machine_type = try(coalesce(local.cfg.default_node_pool.machine_type, null), "") + + preemptible = try(coalesce(local.cfg.default_node_pool.preemptible, null), false) + auto_repair = try(coalesce(local.cfg.default_node_pool.auto_repair, null), true) + auto_upgrade = try(coalesce(local.cfg.default_node_pool.auto_upgrade, null), true) + + node_workload_metadata_config = try(coalesce(local.cfg.node_workload_metadata_config, null), "GKE_METADATA") + + taints = try(coalesce(local.cfg.default_node_pool.taints, null), toset([])) + instance_tags = try(coalesce(local.cfg.default_node_pool.instance_tags, null), []) + labels = try(coalesce(local.cfg.default_node_pool.labels, null), {}) + + ephemeral_storage_local_ssd_config = try(local.cfg.default_node_pool.ephemeral_storage_local_ssd_config, null) + network_config = try(local.cfg.default_node_pool.network_config, null) + guest_accelerator = null + } + } + configuration_base_key = terraform.workspace + + cluster_metadata = module.cluster_metadata +} diff --git a/google/cluster/outputs.tf b/google/cluster/outputs.tf index 589e30b8..461e37e3 100644 --- a/google/cluster/outputs.tf +++ b/google/cluster/outputs.tf @@ -1,5 +1,9 @@ +output "cluster" { + value = google_container_cluster.current +} + output "current_config" { - value = module.configuration.merged[terraform.workspace] + value = local.cfg } output "current_metadata" { @@ -7,9 +11,13 @@ output "current_metadata" { } output "kubeconfig" { - value = module.cluster.kubeconfig + value = local.kubeconfig + + # when the node pool is destroyed before the k8s namespaces + # the namespaces get stuck in terminating + depends_on = [module.node_pool] } output "default_ingress_ip" { - value = module.cluster.default_ingress_ip + value = length(google_compute_address.current) > 0 ? google_compute_address.current[0].address : null } diff --git a/google/_modules/gke/provider.tf b/google/cluster/provider.tf similarity index 100% rename from google/_modules/gke/provider.tf rename to google/cluster/provider.tf diff --git a/google/_modules/gke/service_account.tf b/google/cluster/service_account.tf similarity index 70% rename from google/_modules/gke/service_account.tf rename to google/cluster/service_account.tf index 1ba41b3e..47a08a1b 100644 --- a/google/_modules/gke/service_account.tf +++ b/google/cluster/service_account.tf @@ -1,17 +1,16 @@ resource "google_service_account" "current" { - account_id = substr(var.metadata_name, 0, 30) - project = var.project + account_id = substr(module.cluster_metadata.name, 0, 30) + project = local.cfg.project_id } resource "google_project_iam_member" "log_writer" { - project = var.project + project = local.cfg.project_id role = "roles/logging.logWriter" member = "serviceAccount:${google_service_account.current.email}" } resource "google_project_iam_member" "metric_writer" { - project = var.project + project = local.cfg.project_id role = "roles/monitoring.metricWriter" member = "serviceAccount:${google_service_account.current.email}" } - diff --git a/google/_modules/gke/templates/kubeconfig.tpl b/google/cluster/templates/kubeconfig.tpl similarity index 100% rename from google/_modules/gke/templates/kubeconfig.tpl rename to google/cluster/templates/kubeconfig.tpl diff --git a/google/cluster/variables.tf b/google/cluster/variables.tf index 8bf95161..051282b1 100644 --- a/google/cluster/variables.tf +++ b/google/cluster/variables.tf @@ -1,10 +1,108 @@ variable "configuration" { - type = map(map(string)) + type = map(object({ + # Required attributes + name_prefix = optional(string) + base_domain = optional(string) + project_id = optional(string) + region = optional(string) + cluster_min_master_version = optional(string) + + # Optional attributes without defaults + deletion_protection = optional(bool) + + cluster_node_locations = optional(list(string)) + + cluster_release_channel = optional(string) + + cluster_daily_maintenance_window_start_time = optional(string) + + cluster_maintenance_exclusion = optional(object({ + start_time = optional(string) + end_time = optional(string) + name = optional(string) + scope = optional(string) + })) + + default_node_pool = optional(object({ + remove_default_node_pool = optional(bool) + initial_node_count = optional(number) + min_node_count = optional(number) + max_node_count = optional(number) + node_location_policy = optional(string) + extra_oauth_scopes = optional(list(string)) + disk_size_gb = optional(number) + disk_type = optional(string) + image_type = optional(string) + machine_type = optional(string) + preemptible = optional(bool) + auto_repair = optional(bool) + auto_upgrade = optional(bool) + taints = optional(set(object({ + key = string + value = string + effect = string + }))) + labels = optional(map(string)) + instance_tags = optional(list(string)) + ephemeral_storage_local_ssd_config = optional(object({ + local_ssd_count = number + })) + network_config = optional(object({ + enable_private_nodes = bool + create_pod_range = bool + pod_ipv4_cidr_block = string + })) + })) + + disable_default_ingress = optional(bool) + + enable_private_nodes = optional(bool) + enable_private_endpoint = optional(bool) + master_cidr_block = optional(string) + + cluster_ipv4_cidr_block = optional(string) + services_ipv4_cidr_block = optional(string) + + cluster_database_encryption_key_name = optional(string) + + enable_cloud_nat = optional(bool) + cloud_nat_enable_endpoint_independent_mapping = optional(bool) + cloud_nat_min_ports_per_vm = optional(number) + cloud_nat_ip_count = optional(number) + + disable_workload_identity = optional(bool) + node_workload_metadata_config = optional(string) + + master_authorized_networks_config_cidr_blocks = optional(set(string)) + + enable_intranode_visibility = optional(bool) + enable_tpu = optional(bool) + + router_advertise_config = optional(object({ + groups = optional(set(string)) + ip_ranges = optional(map(string)) + mode = optional(string) + })) + router_asn = optional(number) + + logging_config = optional(object({ + enable_components = optional(list(string)) + })) + + monitoring_config = optional(object({ + enable_components = optional(list(string)) + })) + + enable_gcs_fuse_csi_driver = optional(bool) + })) + description = "Map with per workspace cluster configuration." + nullable = false } variable "configuration_base_key" { type = string description = "The key in the configuration map all other keys inherit from." default = "apps" + nullable = false } diff --git a/kind/_modules/kind/main.tf b/kind/_modules/kind/main.tf deleted file mode 100644 index 97067805..00000000 --- a/kind/_modules/kind/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -locals { - kubeadm_config_patches = [ - <