From 2dc566533300120577ed35807b8773a97735f6cf Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 10:51:38 +0800 Subject: [PATCH 01/16] commit for now --- examples/application_gateway_ingress/main.tf | 1 + .../application_gateway_ingress_v4/data.tf | 1 + .../k8s_workload.tf | 1 + .../application_gateway_ingress_v4/main.tf | 1 + .../main_override.tf | 5 + .../application_gateway_ingress_v4/outputs.tf | 1 + .../providers.tf | 1 + .../providers_override.tf | 8 + .../variables.tf | 1 + examples/multiple_node_pools/main.tf | 1 - examples/multiple_node_pools/main_override.tf | 3 + examples/multiple_node_pools_v4/main.tf | 1 + .../multiple_node_pools_v4/main_override.tf | 8 + examples/multiple_node_pools_v4/outputs.tf | 1 + examples/multiple_node_pools_v4/providers.tf | 1 + .../providers_override.tf | 12 + examples/multiple_node_pools_v4/variables.tf | 1 + extra_node_pool.tf | 10 - extra_node_pool_override.tf | 15 + log_analytics.tf | 1 + main.tf | 7 + main_override.tf | 246 +++++++++++++++ v4/extra_node_pool.tf | 1 + v4/extra_node_pool_override.tf | 11 + v4/locals.tf | 1 + v4/log_analytics.tf | 1 + v4/main.tf | 1 + v4/main_override.tf | 298 ++++++++++++++++++ v4/outputs.tf | 1 + v4/role_assignments.tf | 1 + v4/variables.tf | 1 + v4/variables_override.tf | 15 + v4/versions.tf | 1 + v4/versions_override.tf | 20 ++ 34 files changed, 668 insertions(+), 11 deletions(-) create mode 100644 examples/application_gateway_ingress_v4/data.tf create mode 120000 examples/application_gateway_ingress_v4/k8s_workload.tf create mode 120000 examples/application_gateway_ingress_v4/main.tf create mode 100644 examples/application_gateway_ingress_v4/main_override.tf create mode 120000 examples/application_gateway_ingress_v4/outputs.tf create mode 120000 examples/application_gateway_ingress_v4/providers.tf create mode 100644 examples/application_gateway_ingress_v4/providers_override.tf create mode 120000 examples/application_gateway_ingress_v4/variables.tf create mode 100644 examples/multiple_node_pools/main_override.tf create mode 120000 examples/multiple_node_pools_v4/main.tf create mode 100644 examples/multiple_node_pools_v4/main_override.tf create mode 120000 examples/multiple_node_pools_v4/outputs.tf create mode 120000 examples/multiple_node_pools_v4/providers.tf create mode 100644 examples/multiple_node_pools_v4/providers_override.tf create mode 120000 examples/multiple_node_pools_v4/variables.tf create mode 100644 extra_node_pool_override.tf create mode 100644 main_override.tf create mode 120000 v4/extra_node_pool.tf create mode 100644 v4/extra_node_pool_override.tf create mode 120000 v4/locals.tf create mode 120000 v4/log_analytics.tf create mode 120000 v4/main.tf create mode 100644 v4/main_override.tf create mode 120000 v4/outputs.tf create mode 120000 v4/role_assignments.tf create mode 120000 v4/variables.tf create mode 100644 v4/variables_override.tf create mode 120000 v4/versions.tf create mode 100644 v4/versions_override.tf diff --git a/examples/application_gateway_ingress/main.tf b/examples/application_gateway_ingress/main.tf index e11cfb14..cc8ad70b 100644 --- a/examples/application_gateway_ingress/main.tf +++ b/examples/application_gateway_ingress/main.tf @@ -183,6 +183,7 @@ module "aks" { private_cluster_enabled = false rbac_aad = true rbac_aad_managed = true + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id role_based_access_control_enabled = true sku_tier = "Standard" vnet_subnet_id = var.bring_your_own_vnet ? azurerm_subnet.test[0].id : null diff --git a/examples/application_gateway_ingress_v4/data.tf b/examples/application_gateway_ingress_v4/data.tf new file mode 100644 index 00000000..012126ad --- /dev/null +++ b/examples/application_gateway_ingress_v4/data.tf @@ -0,0 +1 @@ +data "azurerm_client_config" "this" {} \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/k8s_workload.tf b/examples/application_gateway_ingress_v4/k8s_workload.tf new file mode 120000 index 00000000..3a85d4c7 --- /dev/null +++ b/examples/application_gateway_ingress_v4/k8s_workload.tf @@ -0,0 +1 @@ +../application_gateway_ingress/k8s_workload.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/main.tf b/examples/application_gateway_ingress_v4/main.tf new file mode 120000 index 00000000..025a83a1 --- /dev/null +++ b/examples/application_gateway_ingress_v4/main.tf @@ -0,0 +1 @@ +../application_gateway_ingress/main.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/main_override.tf b/examples/application_gateway_ingress_v4/main_override.tf new file mode 100644 index 00000000..370cc63a --- /dev/null +++ b/examples/application_gateway_ingress_v4/main_override.tf @@ -0,0 +1,5 @@ +module "aks" { + #checkov:skip=CKV_AZURE_141:We enable admin account here so we can provision K8s resources directly in this simple example + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id +} \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/outputs.tf b/examples/application_gateway_ingress_v4/outputs.tf new file mode 120000 index 00000000..b7f9f6ed --- /dev/null +++ b/examples/application_gateway_ingress_v4/outputs.tf @@ -0,0 +1 @@ +../application_gateway_ingress/outputs.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/providers.tf b/examples/application_gateway_ingress_v4/providers.tf new file mode 120000 index 00000000..66780034 --- /dev/null +++ b/examples/application_gateway_ingress_v4/providers.tf @@ -0,0 +1 @@ +../application_gateway_ingress/providers.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/providers_override.tf b/examples/application_gateway_ingress_v4/providers_override.tf new file mode 100644 index 00000000..ab616209 --- /dev/null +++ b/examples/application_gateway_ingress_v4/providers_override.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + } +} \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/variables.tf b/examples/application_gateway_ingress_v4/variables.tf new file mode 120000 index 00000000..7899944f --- /dev/null +++ b/examples/application_gateway_ingress_v4/variables.tf @@ -0,0 +1 @@ +../application_gateway_ingress/variables.tf \ No newline at end of file diff --git a/examples/multiple_node_pools/main.tf b/examples/multiple_node_pools/main.tf index 2cba335b..a8b7291f 100644 --- a/examples/multiple_node_pools/main.tf +++ b/examples/multiple_node_pools/main.tf @@ -28,7 +28,6 @@ resource "azurerm_subnet" "test" { name = "${random_id.prefix.hex}-sn" resource_group_name = local.resource_group.name virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true } locals { diff --git a/examples/multiple_node_pools/main_override.tf b/examples/multiple_node_pools/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/multiple_node_pools/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/main.tf b/examples/multiple_node_pools_v4/main.tf new file mode 120000 index 00000000..a320b6cf --- /dev/null +++ b/examples/multiple_node_pools_v4/main.tf @@ -0,0 +1 @@ +../multiple_node_pools/main.tf \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/main_override.tf b/examples/multiple_node_pools_v4/main_override.tf new file mode 100644 index 00000000..b7c945b8 --- /dev/null +++ b/examples/multiple_node_pools_v4/main_override.tf @@ -0,0 +1,8 @@ +resource "azurerm_subnet" "test" { + private_endpoint_network_policies = "Disabled" + private_link_service_network_policies_enabled = true +} + +module "aks" { + source = "../../v4" +} \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/outputs.tf b/examples/multiple_node_pools_v4/outputs.tf new file mode 120000 index 00000000..a2313596 --- /dev/null +++ b/examples/multiple_node_pools_v4/outputs.tf @@ -0,0 +1 @@ +../multiple_node_pools/outputs.tf \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/providers.tf b/examples/multiple_node_pools_v4/providers.tf new file mode 120000 index 00000000..99bf33e1 --- /dev/null +++ b/examples/multiple_node_pools_v4/providers.tf @@ -0,0 +1 @@ +../multiple_node_pools/providers.tf \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/providers_override.tf b/examples/multiple_node_pools_v4/providers_override.tf new file mode 100644 index 00000000..613b6157 --- /dev/null +++ b/examples/multiple_node_pools_v4/providers_override.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/variables.tf b/examples/multiple_node_pools_v4/variables.tf new file mode 120000 index 00000000..a2f1eeca --- /dev/null +++ b/examples/multiple_node_pools_v4/variables.tf @@ -0,0 +1 @@ +../multiple_node_pools/variables.tf \ No newline at end of file diff --git a/extra_node_pool.tf b/extra_node_pool.tf index 9b732b35..9bc68313 100644 --- a/extra_node_pool.tf +++ b/extra_node_pool.tf @@ -10,10 +10,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy name = "${each.value.name}${substr(md5(uuid()), 0, 4)}" vm_size = each.value.vm_size capacity_reservation_group_id = each.value.capacity_reservation_group_id - custom_ca_trust_enabled = each.value.custom_ca_trust_enabled - enable_auto_scaling = each.value.enable_auto_scaling - enable_host_encryption = each.value.enable_host_encryption - enable_node_public_ip = each.value.enable_node_public_ip eviction_policy = each.value.eviction_policy fips_enabled = each.value.fips_enabled gpu_instance = each.value.gpu_instance @@ -21,7 +17,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy kubelet_disk_type = each.value.kubelet_disk_type max_count = each.value.max_count max_pods = each.value.max_pods - message_of_the_day = each.value.message_of_the_day min_count = each.value.min_count mode = each.value.mode node_count = each.value.node_count @@ -171,17 +166,12 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_after_destroy" name = each.value.name vm_size = each.value.vm_size capacity_reservation_group_id = each.value.capacity_reservation_group_id - custom_ca_trust_enabled = each.value.custom_ca_trust_enabled - enable_auto_scaling = each.value.enable_auto_scaling - enable_host_encryption = each.value.enable_host_encryption - enable_node_public_ip = each.value.enable_node_public_ip eviction_policy = each.value.eviction_policy fips_enabled = each.value.fips_enabled host_group_id = each.value.host_group_id kubelet_disk_type = each.value.kubelet_disk_type max_count = each.value.max_count max_pods = each.value.max_pods - message_of_the_day = each.value.message_of_the_day min_count = each.value.min_count mode = each.value.mode node_count = each.value.node_count diff --git a/extra_node_pool_override.tf b/extra_node_pool_override.tf new file mode 100644 index 00000000..6dcae10e --- /dev/null +++ b/extra_node_pool_override.tf @@ -0,0 +1,15 @@ +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy" { + custom_ca_trust_enabled = each.value.custom_ca_trust_enabled + enable_auto_scaling = each.value.enable_auto_scaling + enable_host_encryption = each.value.enable_host_encryption + enable_node_public_ip = each.value.enable_node_public_ip + message_of_the_day = each.value.message_of_the_day +} + +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_after_destroy" { + custom_ca_trust_enabled = each.value.custom_ca_trust_enabled + enable_auto_scaling = each.value.enable_auto_scaling + enable_host_encryption = each.value.enable_host_encryption + enable_node_public_ip = each.value.enable_node_public_ip + message_of_the_day = each.value.message_of_the_day +} \ No newline at end of file diff --git a/log_analytics.tf b/log_analytics.tf index b791ecdd..1d3a2993 100644 --- a/log_analytics.tf +++ b/log_analytics.tf @@ -28,6 +28,7 @@ resource "azurerm_log_analytics_workspace" "main" { dynamic "identity" { for_each = var.log_analytics_workspace_identity == null ? [] : [var.log_analytics_workspace_identity] + content { type = identity.value.type identity_ids = identity.value.identity_ids diff --git a/main.tf b/main.tf index 44f81f94..499a8b30 100644 --- a/main.tf +++ b/main.tf @@ -327,6 +327,7 @@ resource "azurerm_kubernetes_cluster" "main" { } dynamic "http_proxy_config" { for_each = var.http_proxy_config == null ? [] : ["http_proxy_config"] + content { http_proxy = coalesce(var.http_proxy_config.http_proxy, var.http_proxy_config.https_proxy) https_proxy = coalesce(var.http_proxy_config.https_proxy, var.http_proxy_config.http_proxy) @@ -370,6 +371,7 @@ resource "azurerm_kubernetes_cluster" "main" { } dynamic "kubelet_identity" { for_each = var.kubelet_identity == null ? [] : [var.kubelet_identity] + content { client_id = kubelet_identity.value.client_id object_id = kubelet_identity.value.object_id @@ -411,6 +413,7 @@ resource "azurerm_kubernetes_cluster" "main" { } dynamic "maintenance_window_auto_upgrade" { for_each = var.maintenance_window_auto_upgrade == null ? [] : [var.maintenance_window_auto_upgrade] + content { duration = maintenance_window_auto_upgrade.value.duration frequency = maintenance_window_auto_upgrade.value.frequency @@ -424,6 +427,7 @@ resource "azurerm_kubernetes_cluster" "main" { dynamic "not_allowed" { for_each = maintenance_window_auto_upgrade.value.not_allowed == null ? [] : maintenance_window_auto_upgrade.value.not_allowed + content { end = not_allowed.value.end start = not_allowed.value.start @@ -433,6 +437,7 @@ resource "azurerm_kubernetes_cluster" "main" { } dynamic "maintenance_window_node_os" { for_each = var.maintenance_window_node_os == null ? [] : [var.maintenance_window_node_os] + content { duration = maintenance_window_node_os.value.duration frequency = maintenance_window_node_os.value.frequency @@ -446,6 +451,7 @@ resource "azurerm_kubernetes_cluster" "main" { dynamic "not_allowed" { for_each = maintenance_window_node_os.value.not_allowed == null ? [] : maintenance_window_node_os.value.not_allowed + content { end = not_allowed.value.end start = not_allowed.value.start @@ -504,6 +510,7 @@ resource "azurerm_kubernetes_cluster" "main" { } dynamic "service_mesh_profile" { for_each = var.service_mesh_profile == null ? [] : ["service_mesh_profile"] + content { mode = var.service_mesh_profile.mode external_ingress_gateway_enabled = var.service_mesh_profile.external_ingress_gateway_enabled diff --git a/main_override.tf b/main_override.tf new file mode 100644 index 00000000..86501234 --- /dev/null +++ b/main_override.tf @@ -0,0 +1,246 @@ +resource "azurerm_kubernetes_cluster" "main" { + dynamic "default_node_pool" { + for_each = var.enable_auto_scaling == true ? [] : ["default_node_pool_manually_scaled"] + + content { + name = var.agents_pool_name + vm_size = var.agents_size + enable_auto_scaling = var.enable_auto_scaling + enable_host_encryption = var.enable_host_encryption + enable_node_public_ip = var.enable_node_public_ip + fips_enabled = var.default_node_pool_fips_enabled + max_count = null + max_pods = var.agents_max_pods + min_count = null + node_count = var.agents_count + node_labels = var.agents_labels + node_taints = var.agents_taints + only_critical_addons_enabled = var.only_critical_addons_enabled + orchestrator_version = var.orchestrator_version + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + pod_subnet_id = var.pod_subnet_id + proximity_placement_group_id = var.agents_proximity_placement_group_id + scale_down_mode = var.scale_down_mode + snapshot_id = var.snapshot_id + tags = merge(var.tags, var.agents_tags) + temporary_name_for_rotation = var.temporary_name_for_rotation + type = var.agents_type + ultra_ssd_enabled = var.ultra_ssd_enabled + vnet_subnet_id = var.vnet_subnet_id + zones = var.agents_availability_zones + + dynamic "kubelet_config" { + for_each = var.agents_pool_kubelet_configs + + content { + allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls + container_log_max_line = kubelet_config.value.container_log_max_line + container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb + cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled + cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period + cpu_manager_policy = kubelet_config.value.cpu_manager_policy + image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold + image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold + pod_max_pid = kubelet_config.value.pod_max_pid + topology_manager_policy = kubelet_config.value.topology_manager_policy + } + } + dynamic "linux_os_config" { + for_each = var.agents_pool_linux_os_configs + + content { + swap_file_size_mb = linux_os_config.value.swap_file_size_mb + transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag + transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled + + dynamic "sysctl_config" { + for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs + + content { + fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr + fs_file_max = sysctl_config.value.fs_file_max + fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches + fs_nr_open = sysctl_config.value.fs_nr_open + kernel_threads_max = sysctl_config.value.kernel_threads_max + net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog + net_core_optmem_max = sysctl_config.value.net_core_optmem_max + net_core_rmem_default = sysctl_config.value.net_core_rmem_default + net_core_rmem_max = sysctl_config.value.net_core_rmem_max + net_core_somaxconn = sysctl_config.value.net_core_somaxconn + net_core_wmem_default = sysctl_config.value.net_core_wmem_default + net_core_wmem_max = sysctl_config.value.net_core_wmem_max + net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max + net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min + net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 + net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 + net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 + net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout + net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl + net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes + net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time + net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog + net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets + net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse + net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets + net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max + vm_max_map_count = sysctl_config.value.vm_max_map_count + vm_swappiness = sysctl_config.value.vm_swappiness + vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure + } + } + } + } + dynamic "upgrade_settings" { + for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.agents_pool_max_surge + drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes + node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes + } + } + } + } + dynamic "default_node_pool" { + for_each = var.enable_auto_scaling == true ? ["default_node_pool_auto_scaled"] : [] + + content { + name = var.agents_pool_name + vm_size = var.agents_size + enable_auto_scaling = var.enable_auto_scaling + enable_host_encryption = var.enable_host_encryption + enable_node_public_ip = var.enable_node_public_ip + fips_enabled = var.default_node_pool_fips_enabled + max_count = var.agents_max_count + max_pods = var.agents_max_pods + min_count = var.agents_min_count + node_labels = var.agents_labels + node_taints = var.agents_taints + only_critical_addons_enabled = var.only_critical_addons_enabled + orchestrator_version = var.orchestrator_version + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + pod_subnet_id = var.pod_subnet_id + proximity_placement_group_id = var.agents_proximity_placement_group_id + scale_down_mode = var.scale_down_mode + snapshot_id = var.snapshot_id + tags = merge(var.tags, var.agents_tags) + temporary_name_for_rotation = var.temporary_name_for_rotation + type = var.agents_type + ultra_ssd_enabled = var.ultra_ssd_enabled + vnet_subnet_id = var.vnet_subnet_id + zones = var.agents_availability_zones + + dynamic "kubelet_config" { + for_each = var.agents_pool_kubelet_configs + + content { + allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls + container_log_max_line = kubelet_config.value.container_log_max_line + container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb + cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled + cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period + cpu_manager_policy = kubelet_config.value.cpu_manager_policy + image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold + image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold + pod_max_pid = kubelet_config.value.pod_max_pid + topology_manager_policy = kubelet_config.value.topology_manager_policy + } + } + dynamic "linux_os_config" { + for_each = var.agents_pool_linux_os_configs + + content { + swap_file_size_mb = linux_os_config.value.swap_file_size_mb + transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag + transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled + + dynamic "sysctl_config" { + for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs + + content { + fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr + fs_file_max = sysctl_config.value.fs_file_max + fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches + fs_nr_open = sysctl_config.value.fs_nr_open + kernel_threads_max = sysctl_config.value.kernel_threads_max + net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog + net_core_optmem_max = sysctl_config.value.net_core_optmem_max + net_core_rmem_default = sysctl_config.value.net_core_rmem_default + net_core_rmem_max = sysctl_config.value.net_core_rmem_max + net_core_somaxconn = sysctl_config.value.net_core_somaxconn + net_core_wmem_default = sysctl_config.value.net_core_wmem_default + net_core_wmem_max = sysctl_config.value.net_core_wmem_max + net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max + net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min + net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 + net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 + net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 + net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout + net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl + net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes + net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time + net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog + net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets + net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse + net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets + net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max + vm_max_map_count = sysctl_config.value.vm_max_map_count + vm_swappiness = sysctl_config.value.vm_swappiness + vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure + } + } + } + } + dynamic "upgrade_settings" { + for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.agents_pool_max_surge + drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes + node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes + } + } + } + } + network_profile { + network_plugin = var.network_plugin + dns_service_ip = var.net_profile_dns_service_ip + ebpf_data_plane = var.ebpf_data_plane + load_balancer_sku = var.load_balancer_sku + network_plugin_mode = var.network_plugin_mode + network_policy = var.network_policy + outbound_type = var.net_profile_outbound_type + pod_cidr = var.net_profile_pod_cidr + service_cidr = var.net_profile_service_cidr + + dynamic "load_balancer_profile" { + for_each = var.load_balancer_profile_enabled && var.load_balancer_sku == "standard" ? [ + "load_balancer_profile" + ] : [] + + content { + idle_timeout_in_minutes = var.load_balancer_profile_idle_timeout_in_minutes + managed_outbound_ip_count = var.load_balancer_profile_managed_outbound_ip_count + managed_outbound_ipv6_count = var.load_balancer_profile_managed_outbound_ipv6_count + outbound_ip_address_ids = var.load_balancer_profile_outbound_ip_address_ids + outbound_ip_prefix_ids = var.load_balancer_profile_outbound_ip_prefix_ids + outbound_ports_allocated = var.load_balancer_profile_outbound_ports_allocated + } + } + } + + lifecycle { + ignore_changes = [ + http_application_routing_enabled, + http_proxy_config[0].no_proxy, + kubernetes_version, + public_network_access_enabled, + # we might have a random suffix in cluster's name so we have to ignore it here, but we've traced user supplied cluster name by `null_resource.kubernetes_cluster_name_keeper` so when the name is changed we'll recreate this resource. + name, + ] + } +} \ No newline at end of file diff --git a/v4/extra_node_pool.tf b/v4/extra_node_pool.tf new file mode 120000 index 00000000..9cbc2968 --- /dev/null +++ b/v4/extra_node_pool.tf @@ -0,0 +1 @@ +../extra_node_pool.tf \ No newline at end of file diff --git a/v4/extra_node_pool_override.tf b/v4/extra_node_pool_override.tf new file mode 100644 index 00000000..21ad94b3 --- /dev/null +++ b/v4/extra_node_pool_override.tf @@ -0,0 +1,11 @@ +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy" { + auto_scaling_enabled = each.value.enable_auto_scaling + host_encryption_enabled = each.value.enable_host_encryption + node_public_ip_enabled = each.value.enable_node_public_ip +} + +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_after_destroy" { + auto_scaling_enabled = each.value.enable_auto_scaling + host_encryption_enabled = each.value.enable_host_encryption + node_public_ip_enabled = each.value.enable_node_public_ip +} \ No newline at end of file diff --git a/v4/locals.tf b/v4/locals.tf new file mode 120000 index 00000000..1b032e65 --- /dev/null +++ b/v4/locals.tf @@ -0,0 +1 @@ +../locals.tf \ No newline at end of file diff --git a/v4/log_analytics.tf b/v4/log_analytics.tf new file mode 120000 index 00000000..639a396c --- /dev/null +++ b/v4/log_analytics.tf @@ -0,0 +1 @@ +../log_analytics.tf \ No newline at end of file diff --git a/v4/main.tf b/v4/main.tf new file mode 120000 index 00000000..6c481fa3 --- /dev/null +++ b/v4/main.tf @@ -0,0 +1 @@ +../main.tf \ No newline at end of file diff --git a/v4/main_override.tf b/v4/main_override.tf new file mode 100644 index 00000000..b8648399 --- /dev/null +++ b/v4/main_override.tf @@ -0,0 +1,298 @@ +resource "azurerm_kubernetes_cluster" "main" { + automatic_upgrade_channel = var.automatic_channel_upgrade + node_os_upgrade_channel = var.node_os_channel_upgrade + + dynamic "default_node_pool" { + for_each = var.enable_auto_scaling == true ? [] : ["default_node_pool_manually_scaled"] + + content { + name = var.agents_pool_name + vm_size = var.agents_size + auto_scaling_enabled = var.enable_auto_scaling + fips_enabled = var.default_node_pool_fips_enabled + host_encryption_enabled = var.enable_host_encryption + max_count = null + max_pods = var.agents_max_pods + min_count = null + node_count = var.agents_count + node_labels = var.agents_labels + node_public_ip_enabled = var.enable_node_public_ip + only_critical_addons_enabled = var.only_critical_addons_enabled + orchestrator_version = var.orchestrator_version + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + pod_subnet_id = var.pod_subnet_id + proximity_placement_group_id = var.agents_proximity_placement_group_id + scale_down_mode = var.scale_down_mode + snapshot_id = var.snapshot_id + tags = merge(var.tags, var.agents_tags) + temporary_name_for_rotation = var.temporary_name_for_rotation + type = var.agents_type + ultra_ssd_enabled = var.ultra_ssd_enabled + vnet_subnet_id = var.vnet_subnet_id + zones = var.agents_availability_zones + + dynamic "kubelet_config" { + for_each = var.agents_pool_kubelet_configs + + content { + allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls + container_log_max_line = kubelet_config.value.container_log_max_line + container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb + cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled + cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period + cpu_manager_policy = kubelet_config.value.cpu_manager_policy + image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold + image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold + pod_max_pid = kubelet_config.value.pod_max_pid + topology_manager_policy = kubelet_config.value.topology_manager_policy + } + } + dynamic "linux_os_config" { + for_each = var.agents_pool_linux_os_configs + + content { + swap_file_size_mb = linux_os_config.value.swap_file_size_mb + transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag + transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled + + dynamic "sysctl_config" { + for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs + + content { + fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr + fs_file_max = sysctl_config.value.fs_file_max + fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches + fs_nr_open = sysctl_config.value.fs_nr_open + kernel_threads_max = sysctl_config.value.kernel_threads_max + net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog + net_core_optmem_max = sysctl_config.value.net_core_optmem_max + net_core_rmem_default = sysctl_config.value.net_core_rmem_default + net_core_rmem_max = sysctl_config.value.net_core_rmem_max + net_core_somaxconn = sysctl_config.value.net_core_somaxconn + net_core_wmem_default = sysctl_config.value.net_core_wmem_default + net_core_wmem_max = sysctl_config.value.net_core_wmem_max + net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max + net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min + net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 + net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 + net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 + net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout + net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl + net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes + net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time + net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog + net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets + net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse + net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets + net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max + vm_max_map_count = sysctl_config.value.vm_max_map_count + vm_swappiness = sysctl_config.value.vm_swappiness + vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure + } + } + } + } + dynamic "upgrade_settings" { + for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.agents_pool_max_surge + drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes + node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes + } + } + } + } + dynamic "default_node_pool" { + for_each = var.enable_auto_scaling == true ? ["default_node_pool_auto_scaled"] : [] + + content { + name = var.agents_pool_name + vm_size = var.agents_size + auto_scaling_enabled = var.enable_auto_scaling + fips_enabled = var.default_node_pool_fips_enabled + host_encryption_enabled = var.enable_host_encryption + max_count = var.agents_max_count + max_pods = var.agents_max_pods + min_count = var.agents_min_count + node_labels = var.agents_labels + node_public_ip_enabled = var.enable_node_public_ip + only_critical_addons_enabled = var.only_critical_addons_enabled + orchestrator_version = var.orchestrator_version + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + pod_subnet_id = var.pod_subnet_id + proximity_placement_group_id = var.agents_proximity_placement_group_id + scale_down_mode = var.scale_down_mode + snapshot_id = var.snapshot_id + tags = merge(var.tags, var.agents_tags) + temporary_name_for_rotation = var.temporary_name_for_rotation + type = var.agents_type + ultra_ssd_enabled = var.ultra_ssd_enabled + vnet_subnet_id = var.vnet_subnet_id + zones = var.agents_availability_zones + + dynamic "kubelet_config" { + for_each = var.agents_pool_kubelet_configs + + content { + allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls + container_log_max_line = kubelet_config.value.container_log_max_line + container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb + cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled + cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period + cpu_manager_policy = kubelet_config.value.cpu_manager_policy + image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold + image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold + pod_max_pid = kubelet_config.value.pod_max_pid + topology_manager_policy = kubelet_config.value.topology_manager_policy + } + } + dynamic "linux_os_config" { + for_each = var.agents_pool_linux_os_configs + + content { + swap_file_size_mb = linux_os_config.value.swap_file_size_mb + transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag + transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled + + dynamic "sysctl_config" { + for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs + + content { + fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr + fs_file_max = sysctl_config.value.fs_file_max + fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches + fs_nr_open = sysctl_config.value.fs_nr_open + kernel_threads_max = sysctl_config.value.kernel_threads_max + net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog + net_core_optmem_max = sysctl_config.value.net_core_optmem_max + net_core_rmem_default = sysctl_config.value.net_core_rmem_default + net_core_rmem_max = sysctl_config.value.net_core_rmem_max + net_core_somaxconn = sysctl_config.value.net_core_somaxconn + net_core_wmem_default = sysctl_config.value.net_core_wmem_default + net_core_wmem_max = sysctl_config.value.net_core_wmem_max + net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max + net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min + net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 + net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 + net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 + net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout + net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl + net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes + net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time + net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog + net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets + net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse + net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets + net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max + vm_max_map_count = sysctl_config.value.vm_max_map_count + vm_swappiness = sysctl_config.value.vm_swappiness + vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure + } + } + } + } + dynamic "upgrade_settings" { + for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.agents_pool_max_surge + drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes + node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes + } + } + } + } + dynamic "service_mesh_profile" { + for_each = var.service_mesh_profile == null ? [] : ["service_mesh_profile"] + + content { + mode = var.service_mesh_profile.mode + revisions = var.service_mesh_profile.revisions + external_ingress_gateway_enabled = var.service_mesh_profile.external_ingress_gateway_enabled + internal_ingress_gateway_enabled = var.service_mesh_profile.internal_ingress_gateway_enabled + } + } + dynamic "api_server_access_profile" { + for_each = var.api_server_authorized_ip_ranges != null || var.api_server_subnet_id != null ? [ + "api_server_access_profile" + ] : [] + + content { + authorized_ip_ranges = var.api_server_authorized_ip_ranges + } + } + dynamic "azure_active_directory_role_based_access_control" { + for_each = var.role_based_access_control_enabled && var.rbac_aad && var.rbac_aad_managed ? ["rbac"] : [] + + content { + admin_group_object_ids = var.rbac_aad_admin_group_object_ids + azure_rbac_enabled = var.rbac_aad_azure_rbac_enabled + tenant_id = var.rbac_aad_tenant_id + } + } + dynamic "azure_active_directory_role_based_access_control" { + for_each = var.role_based_access_control_enabled && var.rbac_aad && !var.rbac_aad_managed ? ["rbac"] : [] + + content { + tenant_id = var.rbac_aad_tenant_id + } + } + network_profile { + network_plugin = var.network_plugin + dns_service_ip = var.net_profile_dns_service_ip + load_balancer_sku = var.load_balancer_sku + network_data_plane = var.ebpf_data_plane + network_plugin_mode = var.network_plugin_mode + network_policy = var.network_policy + outbound_type = var.net_profile_outbound_type + pod_cidr = var.net_profile_pod_cidr + service_cidr = var.net_profile_service_cidr + + dynamic "load_balancer_profile" { + for_each = var.load_balancer_profile_enabled && var.load_balancer_sku == "standard" ? [ + "load_balancer_profile" + ] : [] + + content { + idle_timeout_in_minutes = var.load_balancer_profile_idle_timeout_in_minutes + managed_outbound_ip_count = var.load_balancer_profile_managed_outbound_ip_count + managed_outbound_ipv6_count = var.load_balancer_profile_managed_outbound_ipv6_count + outbound_ip_address_ids = var.load_balancer_profile_outbound_ip_address_ids + outbound_ip_prefix_ids = var.load_balancer_profile_outbound_ip_prefix_ids + outbound_ports_allocated = var.load_balancer_profile_outbound_ports_allocated + } + } + } + dynamic "storage_profile" { + for_each = var.storage_profile_enabled ? ["storage_profile"] : [] + + content { + blob_driver_enabled = var.storage_profile_blob_driver_enabled + disk_driver_enabled = var.storage_profile_disk_driver_enabled + file_driver_enabled = var.storage_profile_file_driver_enabled + snapshot_controller_enabled = var.storage_profile_snapshot_controller_enabled + } + } + dynamic "web_app_routing" { + for_each = var.web_app_routing == null ? [] : ["web_app_routing"] + + content { + dns_zone_ids = [var.web_app_routing.dns_zone_id] + } + } + + lifecycle { + ignore_changes = [ + http_application_routing_enabled, + http_proxy_config[0].no_proxy, + kubernetes_version, + # we might have a random suffix in cluster's name so we have to ignore it here, but we've traced user supplied cluster name by `null_resource.kubernetes_cluster_name_keeper` so when the name is changed we'll recreate this resource. + name, + api_server_access_profile, network_profile[0].load_balancer_profile[0].outbound_ip_address_ids, network_profile[0].load_balancer_profile[0].outbound_ip_prefix_ids] + } +} \ No newline at end of file diff --git a/v4/outputs.tf b/v4/outputs.tf new file mode 120000 index 00000000..1a861df4 --- /dev/null +++ b/v4/outputs.tf @@ -0,0 +1 @@ +../outputs.tf \ No newline at end of file diff --git a/v4/role_assignments.tf b/v4/role_assignments.tf new file mode 120000 index 00000000..705ff1c9 --- /dev/null +++ b/v4/role_assignments.tf @@ -0,0 +1 @@ +../role_assignments.tf \ No newline at end of file diff --git a/v4/variables.tf b/v4/variables.tf new file mode 120000 index 00000000..3a65dccd --- /dev/null +++ b/v4/variables.tf @@ -0,0 +1 @@ +../variables.tf \ No newline at end of file diff --git a/v4/variables_override.tf b/v4/variables_override.tf new file mode 100644 index 00000000..2a35b6a7 --- /dev/null +++ b/v4/variables_override.tf @@ -0,0 +1,15 @@ +variable "service_mesh_profile" { + type = object({ + mode = string + revisions = list(string) + internal_ingress_gateway_enabled = optional(bool, true) + external_ingress_gateway_enabled = optional(bool, true) + }) + default = null + description = <<-EOT + `mode` - (Required) The mode of the service mesh. Possible value is `Istio`. + `revisions` - (Required) Specify 1 or 2 Istio control plane revisions for managing minor upgrades using the canary upgrade process. For example, create the resource with `revisions` set to `["asm-1-20"]`, or leave it empty (the `revisions` will only be known after apply). To start the canary upgrade, change `revisions` to `["asm-1-20", "asm-1-21"]`. To roll back the canary upgrade, revert to `["asm-1-20"]`. To confirm the upgrade, change to `["asm-1-21"]`. + `internal_ingress_gateway_enabled` - (Optional) Is Istio Internal Ingress Gateway enabled? Defaults to `true`. + `external_ingress_gateway_enabled` - (Optional) Is Istio External Ingress Gateway enabled? Defaults to `true`. + EOT +} \ No newline at end of file diff --git a/v4/versions.tf b/v4/versions.tf new file mode 120000 index 00000000..8bd0ff14 --- /dev/null +++ b/v4/versions.tf @@ -0,0 +1 @@ +../versions.tf \ No newline at end of file diff --git a/v4/versions_override.tf b/v4/versions_override.tf new file mode 100644 index 00000000..e7d3b24b --- /dev/null +++ b/v4/versions_override.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + azapi = { + source = "Azure/azapi" + version = ">= 1.4.0, < 2.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 3.1" + } + } +} From 35faad1ed8f856c9f6996f251852894063e6ee37 Mon Sep 17 00:00:00 2001 From: zjhe Date: Mon, 9 Sep 2024 13:14:24 +0800 Subject: [PATCH 02/16] with_acr --- examples/application_gateway_ingress/main.tf | 1 - examples/named_cluster/main.tf | 1 - examples/named_cluster/main_override.tf | 3 +++ examples/named_cluster_v4/disk_encryption_set.tf | 1 + examples/named_cluster_v4/key_vault.tf | 1 + examples/named_cluster_v4/kms.tf | 1 + examples/named_cluster_v4/main.tf | 1 + examples/named_cluster_v4/main_override.tf | 5 +++++ examples/named_cluster_v4/outputs.tf | 1 + examples/named_cluster_v4/providers.tf | 1 + examples/named_cluster_v4/providers_override.tf | 16 ++++++++++++++++ examples/named_cluster_v4/variables.tf | 1 + examples/startup/main.tf | 1 - examples/startup/main_override.tf | 3 +++ examples/startup_v4/disk_encryption_set.tf | 1 + examples/startup_v4/main.tf | 1 + examples/startup_v4/main_override.tf | 4 ++++ examples/startup_v4/outputs.tf | 1 + examples/startup_v4/providers.tf | 1 + examples/startup_v4/providers_override.tf | 16 ++++++++++++++++ examples/startup_v4/variables.tf | 1 + .../uai_and_assign_role_on_subnet_v4/data.tf | 1 + .../uai_and_assign_role_on_subnet_v4/main.tf | 1 + .../main_override.tf | 4 ++++ .../providers.tf | 1 + .../providers_override.tf | 12 ++++++++++++ .../variables.tf | 1 + examples/with_acr/main.tf | 6 ------ examples/with_acr/main_override.tf | 10 ++++++++++ examples/with_acr_v4/data.tf | 1 + examples/with_acr_v4/main.tf | 1 + examples/with_acr_v4/main_override.tf | 8 ++++++++ examples/with_acr_v4/outputs.tf | 1 + examples/with_acr_v4/providers.tf | 1 + examples/with_acr_v4/providers_override.tf | 12 ++++++++++++ examples/with_acr_v4/variables.tf | 1 + 36 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 examples/named_cluster/main_override.tf create mode 120000 examples/named_cluster_v4/disk_encryption_set.tf create mode 120000 examples/named_cluster_v4/key_vault.tf create mode 120000 examples/named_cluster_v4/kms.tf create mode 120000 examples/named_cluster_v4/main.tf create mode 100644 examples/named_cluster_v4/main_override.tf create mode 120000 examples/named_cluster_v4/outputs.tf create mode 120000 examples/named_cluster_v4/providers.tf create mode 100644 examples/named_cluster_v4/providers_override.tf create mode 120000 examples/named_cluster_v4/variables.tf create mode 100644 examples/startup/main_override.tf create mode 120000 examples/startup_v4/disk_encryption_set.tf create mode 120000 examples/startup_v4/main.tf create mode 100644 examples/startup_v4/main_override.tf create mode 120000 examples/startup_v4/outputs.tf create mode 120000 examples/startup_v4/providers.tf create mode 100644 examples/startup_v4/providers_override.tf create mode 120000 examples/startup_v4/variables.tf create mode 100644 examples/uai_and_assign_role_on_subnet_v4/data.tf create mode 120000 examples/uai_and_assign_role_on_subnet_v4/main.tf create mode 100644 examples/uai_and_assign_role_on_subnet_v4/main_override.tf create mode 120000 examples/uai_and_assign_role_on_subnet_v4/providers.tf create mode 100644 examples/uai_and_assign_role_on_subnet_v4/providers_override.tf create mode 120000 examples/uai_and_assign_role_on_subnet_v4/variables.tf create mode 100644 examples/with_acr/main_override.tf create mode 100644 examples/with_acr_v4/data.tf create mode 120000 examples/with_acr_v4/main.tf create mode 100644 examples/with_acr_v4/main_override.tf create mode 120000 examples/with_acr_v4/outputs.tf create mode 120000 examples/with_acr_v4/providers.tf create mode 100644 examples/with_acr_v4/providers_override.tf create mode 120000 examples/with_acr_v4/variables.tf diff --git a/examples/application_gateway_ingress/main.tf b/examples/application_gateway_ingress/main.tf index cc8ad70b..e11cfb14 100644 --- a/examples/application_gateway_ingress/main.tf +++ b/examples/application_gateway_ingress/main.tf @@ -183,7 +183,6 @@ module "aks" { private_cluster_enabled = false rbac_aad = true rbac_aad_managed = true - rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id role_based_access_control_enabled = true sku_tier = "Standard" vnet_subnet_id = var.bring_your_own_vnet ? azurerm_subnet.test[0].id : null diff --git a/examples/named_cluster/main.tf b/examples/named_cluster/main.tf index 1c1659d8..29dc870d 100644 --- a/examples/named_cluster/main.tf +++ b/examples/named_cluster/main.tf @@ -28,7 +28,6 @@ resource "azurerm_subnet" "test" { name = "${random_id.prefix.hex}-sn" resource_group_name = local.resource_group.name virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true } resource "azurerm_user_assigned_identity" "test" { diff --git a/examples/named_cluster/main_override.tf b/examples/named_cluster/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/named_cluster/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/named_cluster_v4/disk_encryption_set.tf b/examples/named_cluster_v4/disk_encryption_set.tf new file mode 120000 index 00000000..4f249ab7 --- /dev/null +++ b/examples/named_cluster_v4/disk_encryption_set.tf @@ -0,0 +1 @@ +../named_cluster/disk_encryption_set.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/key_vault.tf b/examples/named_cluster_v4/key_vault.tf new file mode 120000 index 00000000..bcc40744 --- /dev/null +++ b/examples/named_cluster_v4/key_vault.tf @@ -0,0 +1 @@ +../named_cluster/key_vault.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/kms.tf b/examples/named_cluster_v4/kms.tf new file mode 120000 index 00000000..4bc974e9 --- /dev/null +++ b/examples/named_cluster_v4/kms.tf @@ -0,0 +1 @@ +../named_cluster/kms.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/main.tf b/examples/named_cluster_v4/main.tf new file mode 120000 index 00000000..0245d70a --- /dev/null +++ b/examples/named_cluster_v4/main.tf @@ -0,0 +1 @@ +../named_cluster/main.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/main_override.tf b/examples/named_cluster_v4/main_override.tf new file mode 100644 index 00000000..a5ae5a5f --- /dev/null +++ b/examples/named_cluster_v4/main_override.tf @@ -0,0 +1,5 @@ + +module "aks_cluster_name" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id +} diff --git a/examples/named_cluster_v4/outputs.tf b/examples/named_cluster_v4/outputs.tf new file mode 120000 index 00000000..189e2dde --- /dev/null +++ b/examples/named_cluster_v4/outputs.tf @@ -0,0 +1 @@ +../named_cluster/outputs.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/providers.tf b/examples/named_cluster_v4/providers.tf new file mode 120000 index 00000000..4d6ffa73 --- /dev/null +++ b/examples/named_cluster_v4/providers.tf @@ -0,0 +1 @@ +../named_cluster/providers.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/providers_override.tf b/examples/named_cluster_v4/providers_override.tf new file mode 100644 index 00000000..87e7a2f1 --- /dev/null +++ b/examples/named_cluster_v4/providers_override.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + curl = { + source = "anschoewe/curl" + version = "1.0.2" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/named_cluster_v4/variables.tf b/examples/named_cluster_v4/variables.tf new file mode 120000 index 00000000..99dad032 --- /dev/null +++ b/examples/named_cluster_v4/variables.tf @@ -0,0 +1 @@ +../named_cluster/variables.tf \ No newline at end of file diff --git a/examples/startup/main.tf b/examples/startup/main.tf index b0740dcc..5038d5a6 100644 --- a/examples/startup/main.tf +++ b/examples/startup/main.tf @@ -32,7 +32,6 @@ resource "azurerm_subnet" "test" { name = "${random_id.prefix.hex}-sn" resource_group_name = local.resource_group.name virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true } module "aks" { diff --git a/examples/startup/main_override.tf b/examples/startup/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/startup/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/startup_v4/disk_encryption_set.tf b/examples/startup_v4/disk_encryption_set.tf new file mode 120000 index 00000000..626703a0 --- /dev/null +++ b/examples/startup_v4/disk_encryption_set.tf @@ -0,0 +1 @@ +../startup/disk_encryption_set.tf \ No newline at end of file diff --git a/examples/startup_v4/main.tf b/examples/startup_v4/main.tf new file mode 120000 index 00000000..e6269863 --- /dev/null +++ b/examples/startup_v4/main.tf @@ -0,0 +1 @@ +../startup/main.tf \ No newline at end of file diff --git a/examples/startup_v4/main_override.tf b/examples/startup_v4/main_override.tf new file mode 100644 index 00000000..4f5f8642 --- /dev/null +++ b/examples/startup_v4/main_override.tf @@ -0,0 +1,4 @@ +module "aks" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id +} \ No newline at end of file diff --git a/examples/startup_v4/outputs.tf b/examples/startup_v4/outputs.tf new file mode 120000 index 00000000..9d7b14a6 --- /dev/null +++ b/examples/startup_v4/outputs.tf @@ -0,0 +1 @@ +../startup/outputs.tf \ No newline at end of file diff --git a/examples/startup_v4/providers.tf b/examples/startup_v4/providers.tf new file mode 120000 index 00000000..d6882a84 --- /dev/null +++ b/examples/startup_v4/providers.tf @@ -0,0 +1 @@ +../startup/providers.tf \ No newline at end of file diff --git a/examples/startup_v4/providers_override.tf b/examples/startup_v4/providers_override.tf new file mode 100644 index 00000000..87e7a2f1 --- /dev/null +++ b/examples/startup_v4/providers_override.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + curl = { + source = "anschoewe/curl" + version = "1.0.2" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/startup_v4/variables.tf b/examples/startup_v4/variables.tf new file mode 120000 index 00000000..fd59ac33 --- /dev/null +++ b/examples/startup_v4/variables.tf @@ -0,0 +1 @@ +../startup/variables.tf \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/data.tf b/examples/uai_and_assign_role_on_subnet_v4/data.tf new file mode 100644 index 00000000..012126ad --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/data.tf @@ -0,0 +1 @@ +data "azurerm_client_config" "this" {} \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/main.tf b/examples/uai_and_assign_role_on_subnet_v4/main.tf new file mode 120000 index 00000000..b9409925 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/main.tf @@ -0,0 +1 @@ +../uai_and_assign_role_on_subnet/main.tf \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/main_override.tf b/examples/uai_and_assign_role_on_subnet_v4/main_override.tf new file mode 100644 index 00000000..4b1c5307 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/main_override.tf @@ -0,0 +1,4 @@ +module "aks" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id +} \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/providers.tf b/examples/uai_and_assign_role_on_subnet_v4/providers.tf new file mode 120000 index 00000000..14f87c9b --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/providers.tf @@ -0,0 +1 @@ +../uai_and_assign_role_on_subnet/providers.tf \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf b/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf new file mode 100644 index 00000000..613b6157 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/variables.tf b/examples/uai_and_assign_role_on_subnet_v4/variables.tf new file mode 120000 index 00000000..893f5b44 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/variables.tf @@ -0,0 +1 @@ +../uai_and_assign_role_on_subnet/variables.tf \ No newline at end of file diff --git a/examples/with_acr/main.tf b/examples/with_acr/main.tf index 94cfe326..58fe475c 100644 --- a/examples/with_acr/main.tf +++ b/examples/with_acr/main.tf @@ -28,7 +28,6 @@ resource "azurerm_subnet" "test" { name = "${random_id.prefix.hex}-sn" resource_group_name = local.resource_group.name virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true } resource "random_string" "acr_suffix" { @@ -43,11 +42,6 @@ resource "azurerm_container_registry" "example" { name = "aksacrtest${random_string.acr_suffix.result}" resource_group_name = local.resource_group.name sku = "Premium" - - retention_policy { - days = 7 - enabled = true - } } module "aks" { diff --git a/examples/with_acr/main_override.tf b/examples/with_acr/main_override.tf new file mode 100644 index 00000000..35192b17 --- /dev/null +++ b/examples/with_acr/main_override.tf @@ -0,0 +1,10 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} + +resource "azurerm_container_registry" "example" { + retention_policy { + days = 7 + enabled = true + } +} \ No newline at end of file diff --git a/examples/with_acr_v4/data.tf b/examples/with_acr_v4/data.tf new file mode 100644 index 00000000..012126ad --- /dev/null +++ b/examples/with_acr_v4/data.tf @@ -0,0 +1 @@ +data "azurerm_client_config" "this" {} \ No newline at end of file diff --git a/examples/with_acr_v4/main.tf b/examples/with_acr_v4/main.tf new file mode 120000 index 00000000..a4ead1f2 --- /dev/null +++ b/examples/with_acr_v4/main.tf @@ -0,0 +1 @@ +../with_acr/main.tf \ No newline at end of file diff --git a/examples/with_acr_v4/main_override.tf b/examples/with_acr_v4/main_override.tf new file mode 100644 index 00000000..08e953f8 --- /dev/null +++ b/examples/with_acr_v4/main_override.tf @@ -0,0 +1,8 @@ +resource "azurerm_container_registry" "example" { + retention_policy_in_days = 7 +} + +module "aks" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id +} \ No newline at end of file diff --git a/examples/with_acr_v4/outputs.tf b/examples/with_acr_v4/outputs.tf new file mode 120000 index 00000000..d7a2645e --- /dev/null +++ b/examples/with_acr_v4/outputs.tf @@ -0,0 +1 @@ +../with_acr/outputs.tf \ No newline at end of file diff --git a/examples/with_acr_v4/providers.tf b/examples/with_acr_v4/providers.tf new file mode 120000 index 00000000..f54e7d78 --- /dev/null +++ b/examples/with_acr_v4/providers.tf @@ -0,0 +1 @@ +../with_acr/providers.tf \ No newline at end of file diff --git a/examples/with_acr_v4/providers_override.tf b/examples/with_acr_v4/providers_override.tf new file mode 100644 index 00000000..613b6157 --- /dev/null +++ b/examples/with_acr_v4/providers_override.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/with_acr_v4/variables.tf b/examples/with_acr_v4/variables.tf new file mode 120000 index 00000000..4c5ec50d --- /dev/null +++ b/examples/with_acr_v4/variables.tf @@ -0,0 +1 @@ +../with_acr/variables.tf \ No newline at end of file From 655170cdf3213520e9b9f66dafc3b1873277ab7d Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 13:31:37 +0800 Subject: [PATCH 03/16] without_monitor --- examples/without_monitor/main.tf | 1 - examples/without_monitor/main_override.tf | 3 +++ .../without_monitor_v4/disk_encryption_set.tf | 1 + examples/without_monitor_v4/main.tf | 1 + examples/without_monitor_v4/main_override.tf | 4 ++++ examples/without_monitor_v4/outputs.tf | 1 + examples/without_monitor_v4/providers.tf | 1 + .../without_monitor_v4/providers_override.tf | 16 ++++++++++++++++ examples/without_monitor_v4/variables.tf | 1 + 9 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 examples/without_monitor/main_override.tf create mode 120000 examples/without_monitor_v4/disk_encryption_set.tf create mode 120000 examples/without_monitor_v4/main.tf create mode 100644 examples/without_monitor_v4/main_override.tf create mode 120000 examples/without_monitor_v4/outputs.tf create mode 120000 examples/without_monitor_v4/providers.tf create mode 100644 examples/without_monitor_v4/providers_override.tf create mode 120000 examples/without_monitor_v4/variables.tf diff --git a/examples/without_monitor/main.tf b/examples/without_monitor/main.tf index 6be8a984..864b4638 100644 --- a/examples/without_monitor/main.tf +++ b/examples/without_monitor/main.tf @@ -28,7 +28,6 @@ resource "azurerm_subnet" "test" { name = "${random_id.prefix.hex}-sn" resource_group_name = local.resource_group.name virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true } resource "azurerm_dns_zone" "aks_web_app_routing" { diff --git a/examples/without_monitor/main_override.tf b/examples/without_monitor/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/without_monitor/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/without_monitor_v4/disk_encryption_set.tf b/examples/without_monitor_v4/disk_encryption_set.tf new file mode 120000 index 00000000..f8e69f92 --- /dev/null +++ b/examples/without_monitor_v4/disk_encryption_set.tf @@ -0,0 +1 @@ +../without_monitor/disk_encryption_set.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/main.tf b/examples/without_monitor_v4/main.tf new file mode 120000 index 00000000..ec81c61f --- /dev/null +++ b/examples/without_monitor_v4/main.tf @@ -0,0 +1 @@ +../without_monitor/main.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/main_override.tf b/examples/without_monitor_v4/main_override.tf new file mode 100644 index 00000000..300ae3bf --- /dev/null +++ b/examples/without_monitor_v4/main_override.tf @@ -0,0 +1,4 @@ +module "aks_without_monitor" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id +} \ No newline at end of file diff --git a/examples/without_monitor_v4/outputs.tf b/examples/without_monitor_v4/outputs.tf new file mode 120000 index 00000000..1c88a8fa --- /dev/null +++ b/examples/without_monitor_v4/outputs.tf @@ -0,0 +1 @@ +../without_monitor/outputs.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/providers.tf b/examples/without_monitor_v4/providers.tf new file mode 120000 index 00000000..4dd53a7e --- /dev/null +++ b/examples/without_monitor_v4/providers.tf @@ -0,0 +1 @@ +../without_monitor/providers.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/providers_override.tf b/examples/without_monitor_v4/providers_override.tf new file mode 100644 index 00000000..87e7a2f1 --- /dev/null +++ b/examples/without_monitor_v4/providers_override.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + curl = { + source = "anschoewe/curl" + version = "1.0.2" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/without_monitor_v4/variables.tf b/examples/without_monitor_v4/variables.tf new file mode 120000 index 00000000..b42da667 --- /dev/null +++ b/examples/without_monitor_v4/variables.tf @@ -0,0 +1 @@ +../without_monitor/variables.tf \ No newline at end of file From 2eb726778eb9ae9853bcbc9e57c0db488dc9f558 Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 13:37:15 +0800 Subject: [PATCH 04/16] pre-commit --- README.md | 2 ++ examples/multiple_node_pools/main.tf | 8 ++++---- examples/named_cluster/main.tf | 8 ++++---- examples/named_cluster_v4/main_override.tf | 2 +- examples/startup/main.tf | 8 ++++---- examples/startup_v4/main_override.tf | 2 +- .../uai_and_assign_role_on_subnet_v4/main_override.tf | 4 ++-- examples/with_acr/main.tf | 8 ++++---- examples/with_acr_v4/main_override.tf | 2 +- examples/without_monitor/main.tf | 8 ++++---- examples/without_monitor_v4/main_override.tf | 2 +- v4/main_override.tf | 2 +- 12 files changed, 29 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index b3c7be4d..f8d99bfc 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ This Terraform module deploys a Kubernetes cluster on Azure using AKS (Azure Kub -> **NOTE:** If you have not assigned `client_id` or `client_secret`, A `SystemAssigned` identity will be created. +-> **NOTE:** If you're using AuzreRM `v4`, you can use this module by setting `source` to `Azure/aks/azurerm//v4`. + ## Notice on breaking changes Please be aware that major version(e.g., from 6.8.0 to 7.0.0) update contains breaking changes that may impact your infrastructure. It is crucial to review these changes with caution before proceeding with the upgrade. diff --git a/examples/multiple_node_pools/main.tf b/examples/multiple_node_pools/main.tf index a8b7291f..26926246 100644 --- a/examples/multiple_node_pools/main.tf +++ b/examples/multiple_node_pools/main.tf @@ -24,10 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } locals { diff --git a/examples/named_cluster/main.tf b/examples/named_cluster/main.tf index 29dc870d..a9a4a430 100644 --- a/examples/named_cluster/main.tf +++ b/examples/named_cluster/main.tf @@ -24,10 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } resource "azurerm_user_assigned_identity" "test" { diff --git a/examples/named_cluster_v4/main_override.tf b/examples/named_cluster_v4/main_override.tf index a5ae5a5f..5d224163 100644 --- a/examples/named_cluster_v4/main_override.tf +++ b/examples/named_cluster_v4/main_override.tf @@ -1,5 +1,5 @@ module "aks_cluster_name" { - source = "../../v4" + source = "../../v4" rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id } diff --git a/examples/startup/main.tf b/examples/startup/main.tf index 5038d5a6..42661bb9 100644 --- a/examples/startup/main.tf +++ b/examples/startup/main.tf @@ -28,10 +28,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } module "aks" { diff --git a/examples/startup_v4/main_override.tf b/examples/startup_v4/main_override.tf index 4f5f8642..ae101007 100644 --- a/examples/startup_v4/main_override.tf +++ b/examples/startup_v4/main_override.tf @@ -1,4 +1,4 @@ module "aks" { - source = "../../v4" + source = "../../v4" rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id } \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/main_override.tf b/examples/uai_and_assign_role_on_subnet_v4/main_override.tf index 4b1c5307..390bc574 100644 --- a/examples/uai_and_assign_role_on_subnet_v4/main_override.tf +++ b/examples/uai_and_assign_role_on_subnet_v4/main_override.tf @@ -1,4 +1,4 @@ module "aks" { - source = "../../v4" - rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id } \ No newline at end of file diff --git a/examples/with_acr/main.tf b/examples/with_acr/main.tf index 58fe475c..48d9cc40 100644 --- a/examples/with_acr/main.tf +++ b/examples/with_acr/main.tf @@ -24,10 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } resource "random_string" "acr_suffix" { diff --git a/examples/with_acr_v4/main_override.tf b/examples/with_acr_v4/main_override.tf index 08e953f8..f5d3db65 100644 --- a/examples/with_acr_v4/main_override.tf +++ b/examples/with_acr_v4/main_override.tf @@ -3,6 +3,6 @@ resource "azurerm_container_registry" "example" { } module "aks" { - source = "../../v4" + source = "../../v4" rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id } \ No newline at end of file diff --git a/examples/without_monitor/main.tf b/examples/without_monitor/main.tf index 864b4638..d5338885 100644 --- a/examples/without_monitor/main.tf +++ b/examples/without_monitor/main.tf @@ -24,10 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } resource "azurerm_dns_zone" "aks_web_app_routing" { diff --git a/examples/without_monitor_v4/main_override.tf b/examples/without_monitor_v4/main_override.tf index 300ae3bf..2b62b4e9 100644 --- a/examples/without_monitor_v4/main_override.tf +++ b/examples/without_monitor_v4/main_override.tf @@ -1,4 +1,4 @@ module "aks_without_monitor" { - source = "../../v4" + source = "../../v4" rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id } \ No newline at end of file diff --git a/v4/main_override.tf b/v4/main_override.tf index b8648399..3f18d118 100644 --- a/v4/main_override.tf +++ b/v4/main_override.tf @@ -212,7 +212,7 @@ resource "azurerm_kubernetes_cluster" "main" { content { mode = var.service_mesh_profile.mode - revisions = var.service_mesh_profile.revisions + revisions = var.service_mesh_profile.revisions external_ingress_gateway_enabled = var.service_mesh_profile.external_ingress_gateway_enabled internal_ingress_gateway_enabled = var.service_mesh_profile.internal_ingress_gateway_enabled } From 38b4a54940338d02988756243441ae52851d099a Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 14:00:13 +0800 Subject: [PATCH 05/16] main_override.tf --- main_override.tf | 246 +---------------------------------------------- 1 file changed, 2 insertions(+), 244 deletions(-) diff --git a/main_override.tf b/main_override.tf index 86501234..5f772387 100644 --- a/main_override.tf +++ b/main_override.tf @@ -1,246 +1,4 @@ resource "azurerm_kubernetes_cluster" "main" { - dynamic "default_node_pool" { - for_each = var.enable_auto_scaling == true ? [] : ["default_node_pool_manually_scaled"] - - content { - name = var.agents_pool_name - vm_size = var.agents_size - enable_auto_scaling = var.enable_auto_scaling - enable_host_encryption = var.enable_host_encryption - enable_node_public_ip = var.enable_node_public_ip - fips_enabled = var.default_node_pool_fips_enabled - max_count = null - max_pods = var.agents_max_pods - min_count = null - node_count = var.agents_count - node_labels = var.agents_labels - node_taints = var.agents_taints - only_critical_addons_enabled = var.only_critical_addons_enabled - orchestrator_version = var.orchestrator_version - os_disk_size_gb = var.os_disk_size_gb - os_disk_type = var.os_disk_type - os_sku = var.os_sku - pod_subnet_id = var.pod_subnet_id - proximity_placement_group_id = var.agents_proximity_placement_group_id - scale_down_mode = var.scale_down_mode - snapshot_id = var.snapshot_id - tags = merge(var.tags, var.agents_tags) - temporary_name_for_rotation = var.temporary_name_for_rotation - type = var.agents_type - ultra_ssd_enabled = var.ultra_ssd_enabled - vnet_subnet_id = var.vnet_subnet_id - zones = var.agents_availability_zones - - dynamic "kubelet_config" { - for_each = var.agents_pool_kubelet_configs - - content { - allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls - container_log_max_line = kubelet_config.value.container_log_max_line - container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb - cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled - cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period - cpu_manager_policy = kubelet_config.value.cpu_manager_policy - image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold - image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold - pod_max_pid = kubelet_config.value.pod_max_pid - topology_manager_policy = kubelet_config.value.topology_manager_policy - } - } - dynamic "linux_os_config" { - for_each = var.agents_pool_linux_os_configs - - content { - swap_file_size_mb = linux_os_config.value.swap_file_size_mb - transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag - transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled - - dynamic "sysctl_config" { - for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs - - content { - fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr - fs_file_max = sysctl_config.value.fs_file_max - fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches - fs_nr_open = sysctl_config.value.fs_nr_open - kernel_threads_max = sysctl_config.value.kernel_threads_max - net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog - net_core_optmem_max = sysctl_config.value.net_core_optmem_max - net_core_rmem_default = sysctl_config.value.net_core_rmem_default - net_core_rmem_max = sysctl_config.value.net_core_rmem_max - net_core_somaxconn = sysctl_config.value.net_core_somaxconn - net_core_wmem_default = sysctl_config.value.net_core_wmem_default - net_core_wmem_max = sysctl_config.value.net_core_wmem_max - net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max - net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min - net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 - net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 - net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 - net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout - net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl - net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes - net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time - net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog - net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets - net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse - net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets - net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max - vm_max_map_count = sysctl_config.value.vm_max_map_count - vm_swappiness = sysctl_config.value.vm_swappiness - vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure - } - } - } - } - dynamic "upgrade_settings" { - for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] - - content { - max_surge = var.agents_pool_max_surge - drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes - node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes - } - } - } - } - dynamic "default_node_pool" { - for_each = var.enable_auto_scaling == true ? ["default_node_pool_auto_scaled"] : [] - - content { - name = var.agents_pool_name - vm_size = var.agents_size - enable_auto_scaling = var.enable_auto_scaling - enable_host_encryption = var.enable_host_encryption - enable_node_public_ip = var.enable_node_public_ip - fips_enabled = var.default_node_pool_fips_enabled - max_count = var.agents_max_count - max_pods = var.agents_max_pods - min_count = var.agents_min_count - node_labels = var.agents_labels - node_taints = var.agents_taints - only_critical_addons_enabled = var.only_critical_addons_enabled - orchestrator_version = var.orchestrator_version - os_disk_size_gb = var.os_disk_size_gb - os_disk_type = var.os_disk_type - os_sku = var.os_sku - pod_subnet_id = var.pod_subnet_id - proximity_placement_group_id = var.agents_proximity_placement_group_id - scale_down_mode = var.scale_down_mode - snapshot_id = var.snapshot_id - tags = merge(var.tags, var.agents_tags) - temporary_name_for_rotation = var.temporary_name_for_rotation - type = var.agents_type - ultra_ssd_enabled = var.ultra_ssd_enabled - vnet_subnet_id = var.vnet_subnet_id - zones = var.agents_availability_zones - - dynamic "kubelet_config" { - for_each = var.agents_pool_kubelet_configs - - content { - allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls - container_log_max_line = kubelet_config.value.container_log_max_line - container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb - cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled - cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period - cpu_manager_policy = kubelet_config.value.cpu_manager_policy - image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold - image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold - pod_max_pid = kubelet_config.value.pod_max_pid - topology_manager_policy = kubelet_config.value.topology_manager_policy - } - } - dynamic "linux_os_config" { - for_each = var.agents_pool_linux_os_configs - - content { - swap_file_size_mb = linux_os_config.value.swap_file_size_mb - transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag - transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled - - dynamic "sysctl_config" { - for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs - - content { - fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr - fs_file_max = sysctl_config.value.fs_file_max - fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches - fs_nr_open = sysctl_config.value.fs_nr_open - kernel_threads_max = sysctl_config.value.kernel_threads_max - net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog - net_core_optmem_max = sysctl_config.value.net_core_optmem_max - net_core_rmem_default = sysctl_config.value.net_core_rmem_default - net_core_rmem_max = sysctl_config.value.net_core_rmem_max - net_core_somaxconn = sysctl_config.value.net_core_somaxconn - net_core_wmem_default = sysctl_config.value.net_core_wmem_default - net_core_wmem_max = sysctl_config.value.net_core_wmem_max - net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max - net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min - net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 - net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 - net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 - net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout - net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl - net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes - net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time - net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog - net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets - net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse - net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets - net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max - vm_max_map_count = sysctl_config.value.vm_max_map_count - vm_swappiness = sysctl_config.value.vm_swappiness - vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure - } - } - } - } - dynamic "upgrade_settings" { - for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] - - content { - max_surge = var.agents_pool_max_surge - drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes - node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes - } - } - } - } - network_profile { - network_plugin = var.network_plugin - dns_service_ip = var.net_profile_dns_service_ip - ebpf_data_plane = var.ebpf_data_plane - load_balancer_sku = var.load_balancer_sku - network_plugin_mode = var.network_plugin_mode - network_policy = var.network_policy - outbound_type = var.net_profile_outbound_type - pod_cidr = var.net_profile_pod_cidr - service_cidr = var.net_profile_service_cidr - - dynamic "load_balancer_profile" { - for_each = var.load_balancer_profile_enabled && var.load_balancer_sku == "standard" ? [ - "load_balancer_profile" - ] : [] - - content { - idle_timeout_in_minutes = var.load_balancer_profile_idle_timeout_in_minutes - managed_outbound_ip_count = var.load_balancer_profile_managed_outbound_ip_count - managed_outbound_ipv6_count = var.load_balancer_profile_managed_outbound_ipv6_count - outbound_ip_address_ids = var.load_balancer_profile_outbound_ip_address_ids - outbound_ip_prefix_ids = var.load_balancer_profile_outbound_ip_prefix_ids - outbound_ports_allocated = var.load_balancer_profile_outbound_ports_allocated - } - } - } - - lifecycle { - ignore_changes = [ - http_application_routing_enabled, - http_proxy_config[0].no_proxy, - kubernetes_version, - public_network_access_enabled, - # we might have a random suffix in cluster's name so we have to ignore it here, but we've traced user supplied cluster name by `null_resource.kubernetes_cluster_name_keeper` so when the name is changed we'll recreate this resource. - name, - ] - } + automatic_channel_upgrade = var.automatic_channel_upgrade + node_os_channel_upgrade = var.node_os_channel_upgrade } \ No newline at end of file From 08cda8ba8cd7453021585e74cb82d70443716de1 Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 14:29:18 +0800 Subject: [PATCH 06/16] fix format --- main_override.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main_override.tf b/main_override.tf index 5f772387..fdb48813 100644 --- a/main_override.tf +++ b/main_override.tf @@ -1,4 +1,4 @@ resource "azurerm_kubernetes_cluster" "main" { - automatic_channel_upgrade = var.automatic_channel_upgrade - node_os_channel_upgrade = var.node_os_channel_upgrade + automatic_channel_upgrade = var.automatic_channel_upgrade + node_os_channel_upgrade = var.node_os_channel_upgrade } \ No newline at end of file From dad6e1e1b1fd8e9911d04331d86779826342b368 Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 14:44:29 +0800 Subject: [PATCH 07/16] try to ignore azurerm_resource_tag check on override block --- main_override.tf | 1 + v4/main_override.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/main_override.tf b/main_override.tf index fdb48813..ce1f09c9 100644 --- a/main_override.tf +++ b/main_override.tf @@ -1,4 +1,5 @@ resource "azurerm_kubernetes_cluster" "main" { + # tflint-ignore: azurerm_resource_tag automatic_channel_upgrade = var.automatic_channel_upgrade node_os_channel_upgrade = var.node_os_channel_upgrade } \ No newline at end of file diff --git a/v4/main_override.tf b/v4/main_override.tf index 3f18d118..4176a6ad 100644 --- a/v4/main_override.tf +++ b/v4/main_override.tf @@ -1,4 +1,5 @@ resource "azurerm_kubernetes_cluster" "main" { + # tflint-ignore: azurerm_resource_tag automatic_upgrade_channel = var.automatic_channel_upgrade node_os_upgrade_channel = var.node_os_channel_upgrade From a1b217b5d59cf31d09dc2c8d43cefa01dbcafd7c Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 15:03:08 +0800 Subject: [PATCH 08/16] fix tflint issues on override files --- .../providers_override.tf | 14 ++++++++++++++ .../multiple_node_pools_v4/providers_override.tf | 2 ++ examples/named_cluster_v4/providers_override.tf | 2 ++ examples/startup_v4/providers_override.tf | 2 ++ .../providers_override.tf | 2 ++ examples/with_acr_v4/providers_override.tf | 2 ++ examples/without_monitor_v4/providers_override.tf | 2 ++ extra_node_pool_override.tf | 2 ++ main_override.tf | 3 ++- v4/extra_node_pool_override.tf | 2 ++ v4/main_override.tf | 3 ++- 11 files changed, 34 insertions(+), 2 deletions(-) diff --git a/examples/application_gateway_ingress_v4/providers_override.tf b/examples/application_gateway_ingress_v4/providers_override.tf index ab616209..59b6fc4e 100644 --- a/examples/application_gateway_ingress_v4/providers_override.tf +++ b/examples/application_gateway_ingress_v4/providers_override.tf @@ -1,8 +1,22 @@ +# tflint-ignore-file: terraform_required_version_declaration + terraform { required_providers { azurerm = { source = "hashicorp/azurerm" version = "~> 4.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + time = { + source = "hashicorp/time" + version = "0.9.1" + } } } \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/providers_override.tf b/examples/multiple_node_pools_v4/providers_override.tf index 613b6157..01e99a37 100644 --- a/examples/multiple_node_pools_v4/providers_override.tf +++ b/examples/multiple_node_pools_v4/providers_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: terraform_required_version_declaration + terraform { required_providers { azurerm = { diff --git a/examples/named_cluster_v4/providers_override.tf b/examples/named_cluster_v4/providers_override.tf index 87e7a2f1..0e2a6bc8 100644 --- a/examples/named_cluster_v4/providers_override.tf +++ b/examples/named_cluster_v4/providers_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: terraform_required_version_declaration + terraform { required_providers { azurerm = { diff --git a/examples/startup_v4/providers_override.tf b/examples/startup_v4/providers_override.tf index 87e7a2f1..0e2a6bc8 100644 --- a/examples/startup_v4/providers_override.tf +++ b/examples/startup_v4/providers_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: terraform_required_version_declaration + terraform { required_providers { azurerm = { diff --git a/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf b/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf index 613b6157..01e99a37 100644 --- a/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf +++ b/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: terraform_required_version_declaration + terraform { required_providers { azurerm = { diff --git a/examples/with_acr_v4/providers_override.tf b/examples/with_acr_v4/providers_override.tf index 613b6157..01e99a37 100644 --- a/examples/with_acr_v4/providers_override.tf +++ b/examples/with_acr_v4/providers_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: terraform_required_version_declaration + terraform { required_providers { azurerm = { diff --git a/examples/without_monitor_v4/providers_override.tf b/examples/without_monitor_v4/providers_override.tf index 87e7a2f1..0e2a6bc8 100644 --- a/examples/without_monitor_v4/providers_override.tf +++ b/examples/without_monitor_v4/providers_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: terraform_required_version_declaration + terraform { required_providers { azurerm = { diff --git a/extra_node_pool_override.tf b/extra_node_pool_override.tf index 6dcae10e..500f27ec 100644 --- a/extra_node_pool_override.tf +++ b/extra_node_pool_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: azurerm_resource_tag + resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy" { custom_ca_trust_enabled = each.value.custom_ca_trust_enabled enable_auto_scaling = each.value.enable_auto_scaling diff --git a/main_override.tf b/main_override.tf index ce1f09c9..a1f53765 100644 --- a/main_override.tf +++ b/main_override.tf @@ -1,5 +1,6 @@ +# tflint-ignore-file: azurerm_resource_tag + resource "azurerm_kubernetes_cluster" "main" { - # tflint-ignore: azurerm_resource_tag automatic_channel_upgrade = var.automatic_channel_upgrade node_os_channel_upgrade = var.node_os_channel_upgrade } \ No newline at end of file diff --git a/v4/extra_node_pool_override.tf b/v4/extra_node_pool_override.tf index 21ad94b3..0d0227a6 100644 --- a/v4/extra_node_pool_override.tf +++ b/v4/extra_node_pool_override.tf @@ -1,3 +1,5 @@ +# tflint-ignore-file: azurerm_resource_tag + resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy" { auto_scaling_enabled = each.value.enable_auto_scaling host_encryption_enabled = each.value.enable_host_encryption diff --git a/v4/main_override.tf b/v4/main_override.tf index 4176a6ad..1c8baba0 100644 --- a/v4/main_override.tf +++ b/v4/main_override.tf @@ -1,5 +1,6 @@ +# tflint-ignore-file: azurerm_resource_tag + resource "azurerm_kubernetes_cluster" "main" { - # tflint-ignore: azurerm_resource_tag automatic_upgrade_channel = var.automatic_channel_upgrade node_os_upgrade_channel = var.node_os_channel_upgrade From 23db3af17033fbfe93fda558e5efa3c3e618b520 Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 15:44:56 +0800 Subject: [PATCH 09/16] add v4 examples to acc tests --- test/e2e/terraform_aks_test.go | 42 ++++++++++++++++++++++++++++--- test/upgrade/upgrade_test.go | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index de37fa0a..6839a0ad 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -4,18 +4,17 @@ import ( "fmt" "io" "os" + "path/filepath" "regexp" "strings" "testing" "time" - "github.com/hashicorp/go-retryablehttp" - - "github.com/stretchr/testify/require" - test_helper "github.com/Azure/terraform-module-test-helper" "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/hashicorp/go-retryablehttp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestExamplesStartup(t *testing.T) { @@ -178,6 +177,41 @@ func TestExamples_applicationGatewayIngress(t *testing.T) { } } +func TestExamplesForV4(t *testing.T) { + managedIdentityId := os.Getenv("MSI_ID") + if managedIdentityId != "" { + t.Setenv("TF_VAR_managed_identity_principal_id", managedIdentityId) + } + tmp, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmp) + }() + tfvars := filepath.Join(tmp, "terraform.tfvars") + require.NoError(t, os.WriteFile(tfvars, []byte(` + client_id = "" + client_secret = "" +`), 0o644)) + t.Setenv("TF_VAR_client_id", "") + t.Setenv("TF_VAR_client_secret", "") + examples, err := os.ReadDir("../../examples") + require.NoError(t, err) + for _, example := range examples { + if !example.IsDir() { + continue + } + if !strings.HasSuffix(example.Name(), "_v4") { + continue + } + t.Run(example.Name(), func(t *testing.T) { + test_helper.RunE2ETest(t, fmt.Sprintf("../../"), fmt.Sprintf("examples/%s", example.Name()), terraform.Options{ + Upgrade: true, + VarFiles: []string{tfvars}, + }, nil) + }) + } +} + func getHTML(url string) (string, error) { client := retryablehttp.NewClient() client.RetryMax = 10 diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go index efc79a3a..fd3ae14f 100644 --- a/test/upgrade/upgrade_test.go +++ b/test/upgrade/upgrade_test.go @@ -3,9 +3,13 @@ package upgrade import ( "fmt" "os" + "path/filepath" + "strings" "testing" "time" + "github.com/stretchr/testify/require" + test_helper "github.com/Azure/terraform-module-test-helper" "github.com/gruntwork-io/terratest/modules/terraform" ) @@ -148,3 +152,45 @@ func TestExampleUpgrade_applicationGw(t *testing.T) { }) } } + +func TestExamplesForV4(t *testing.T) { + managedIdentityId := os.Getenv("MSI_ID") + if managedIdentityId != "" { + t.Setenv("TF_VAR_managed_identity_principal_id", managedIdentityId) + } + tmp, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmp) + }() + tfvars := filepath.Join(tmp, "terraform.tfvars") + require.NoError(t, os.WriteFile(tfvars, []byte(` + client_id = "" + client_secret = "" +`), 0o644)) + t.Setenv("TF_VAR_client_id", "") + t.Setenv("TF_VAR_client_secret", "") + examples, err := os.ReadDir("../../examples") + require.NoError(t, err) + currentRoot, err := test_helper.GetCurrentModuleRootPath() + if err != nil { + t.FailNow() + } + currentMajorVersion, err := test_helper.GetCurrentMajorVersionFromEnv() + if err != nil { + t.FailNow() + } + for _, example := range examples { + if !example.IsDir() { + continue + } + if strings.HasSuffix(example.Name(), "_v4") { + continue + } + t.Run(example.Name(), func(t *testing.T) { + test_helper.ModuleUpgradeTest(t, "Azure", "terraform-azurerm-aks", fmt.Sprintf("examples/%s", example.Name()), currentRoot, terraform.Options{ + VarFiles: []string{tfvars}, + }, currentMajorVersion) + }) + } +} From 3dd83e7c3acd01d9515cada32ff9d6ad855beafc Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 15:50:09 +0800 Subject: [PATCH 10/16] fix golangci-lint issue --- test/e2e/terraform_aks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index 6839a0ad..80fab719 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -204,7 +204,7 @@ func TestExamplesForV4(t *testing.T) { continue } t.Run(example.Name(), func(t *testing.T) { - test_helper.RunE2ETest(t, fmt.Sprintf("../../"), fmt.Sprintf("examples/%s", example.Name()), terraform.Options{ + test_helper.RunE2ETest(t, "../../", fmt.Sprintf("examples/%s", example.Name()), terraform.Options{ Upgrade: true, VarFiles: []string{tfvars}, }, nil) From 328dd2896fe382c0091335d88f92e9372632e3d7 Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 9 Sep 2024 16:07:49 +0800 Subject: [PATCH 11/16] fix golangci-lint issue --- test/e2e/terraform_aks_test.go | 2 +- test/upgrade/upgrade_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index 80fab719..66c61d2c 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -191,7 +191,7 @@ func TestExamplesForV4(t *testing.T) { require.NoError(t, os.WriteFile(tfvars, []byte(` client_id = "" client_secret = "" -`), 0o644)) +`), 0o600)) t.Setenv("TF_VAR_client_id", "") t.Setenv("TF_VAR_client_secret", "") examples, err := os.ReadDir("../../examples") diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go index fd3ae14f..53fc7b9f 100644 --- a/test/upgrade/upgrade_test.go +++ b/test/upgrade/upgrade_test.go @@ -167,7 +167,7 @@ func TestExamplesForV4(t *testing.T) { require.NoError(t, os.WriteFile(tfvars, []byte(` client_id = "" client_secret = "" -`), 0o644)) +`), 0o600)) t.Setenv("TF_VAR_client_id", "") t.Setenv("TF_VAR_client_secret", "") examples, err := os.ReadDir("../../examples") From 7a555688c87b3eac9bbb85f2cd51ade33b16640f Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 10 Sep 2024 09:13:48 +0800 Subject: [PATCH 12/16] try to fix v4 test --- test/e2e/terraform_aks_test.go | 20 ++++++++++---------- test/upgrade/upgrade_test.go | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index 66c61d2c..465903f2 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -182,16 +182,6 @@ func TestExamplesForV4(t *testing.T) { if managedIdentityId != "" { t.Setenv("TF_VAR_managed_identity_principal_id", managedIdentityId) } - tmp, err := os.MkdirTemp("", "") - require.NoError(t, err) - defer func() { - _ = os.RemoveAll(tmp) - }() - tfvars := filepath.Join(tmp, "terraform.tfvars") - require.NoError(t, os.WriteFile(tfvars, []byte(` - client_id = "" - client_secret = "" -`), 0o600)) t.Setenv("TF_VAR_client_id", "") t.Setenv("TF_VAR_client_secret", "") examples, err := os.ReadDir("../../examples") @@ -204,6 +194,16 @@ func TestExamplesForV4(t *testing.T) { continue } t.Run(example.Name(), func(t *testing.T) { + tmp, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmp) + }() + tfvars := filepath.Join(tmp, "terraform.tfvars") + require.NoError(t, os.WriteFile(tfvars, []byte(` + client_id = "" + client_secret = "" +`), 0o600)) test_helper.RunE2ETest(t, "../../", fmt.Sprintf("examples/%s", example.Name()), terraform.Options{ Upgrade: true, VarFiles: []string{tfvars}, diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go index 53fc7b9f..53fa862f 100644 --- a/test/upgrade/upgrade_test.go +++ b/test/upgrade/upgrade_test.go @@ -163,11 +163,6 @@ func TestExamplesForV4(t *testing.T) { defer func() { _ = os.RemoveAll(tmp) }() - tfvars := filepath.Join(tmp, "terraform.tfvars") - require.NoError(t, os.WriteFile(tfvars, []byte(` - client_id = "" - client_secret = "" -`), 0o600)) t.Setenv("TF_VAR_client_id", "") t.Setenv("TF_VAR_client_secret", "") examples, err := os.ReadDir("../../examples") @@ -188,6 +183,11 @@ func TestExamplesForV4(t *testing.T) { continue } t.Run(example.Name(), func(t *testing.T) { + tfvars := filepath.Join(tmp, "terraform.tfvars") + require.NoError(t, os.WriteFile(tfvars, []byte(` + client_id = "" + client_secret = "" +`), 0o600)) test_helper.ModuleUpgradeTest(t, "Azure", "terraform-azurerm-aks", fmt.Sprintf("examples/%s", example.Name()), currentRoot, terraform.Options{ VarFiles: []string{tfvars}, }, currentMajorVersion) From 59c66f5ec72fd91e369778adda4c8e1da6b8cbfa Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 10 Sep 2024 09:14:45 +0800 Subject: [PATCH 13/16] try to fix v4 test --- test/e2e/terraform_aks_test.go | 12 ++++++------ test/upgrade/upgrade_test.go | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index 465903f2..4bece507 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -178,12 +178,6 @@ func TestExamples_applicationGatewayIngress(t *testing.T) { } func TestExamplesForV4(t *testing.T) { - managedIdentityId := os.Getenv("MSI_ID") - if managedIdentityId != "" { - t.Setenv("TF_VAR_managed_identity_principal_id", managedIdentityId) - } - t.Setenv("TF_VAR_client_id", "") - t.Setenv("TF_VAR_client_secret", "") examples, err := os.ReadDir("../../examples") require.NoError(t, err) for _, example := range examples { @@ -194,6 +188,12 @@ func TestExamplesForV4(t *testing.T) { continue } t.Run(example.Name(), func(t *testing.T) { + managedIdentityId := os.Getenv("MSI_ID") + if managedIdentityId != "" { + t.Setenv("TF_VAR_managed_identity_principal_id", managedIdentityId) + } + t.Setenv("TF_VAR_client_id", "") + t.Setenv("TF_VAR_client_secret", "") tmp, err := os.MkdirTemp("", "") require.NoError(t, err) defer func() { diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go index 53fa862f..9b6ea3e3 100644 --- a/test/upgrade/upgrade_test.go +++ b/test/upgrade/upgrade_test.go @@ -154,17 +154,6 @@ func TestExampleUpgrade_applicationGw(t *testing.T) { } func TestExamplesForV4(t *testing.T) { - managedIdentityId := os.Getenv("MSI_ID") - if managedIdentityId != "" { - t.Setenv("TF_VAR_managed_identity_principal_id", managedIdentityId) - } - tmp, err := os.MkdirTemp("", "") - require.NoError(t, err) - defer func() { - _ = os.RemoveAll(tmp) - }() - t.Setenv("TF_VAR_client_id", "") - t.Setenv("TF_VAR_client_secret", "") examples, err := os.ReadDir("../../examples") require.NoError(t, err) currentRoot, err := test_helper.GetCurrentModuleRootPath() @@ -183,6 +172,17 @@ func TestExamplesForV4(t *testing.T) { continue } t.Run(example.Name(), func(t *testing.T) { + managedIdentityId := os.Getenv("MSI_ID") + if managedIdentityId != "" { + t.Setenv("TF_VAR_managed_identity_principal_id", managedIdentityId) + } + t.Setenv("TF_VAR_client_id", "") + t.Setenv("TF_VAR_client_secret", "") + tmp, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmp) + }() tfvars := filepath.Join(tmp, "terraform.tfvars") require.NoError(t, os.WriteFile(tfvars, []byte(` client_id = "" From 49562ef11de328b3bf4b6b654d8b5c3d58d04360 Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 10 Sep 2024 11:09:51 +0800 Subject: [PATCH 14/16] remove deprecated attriubtes from v3 azruerm_kubernetes_cluster --- main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/main.tf b/main.tf index 499a8b30..c18c539b 100644 --- a/main.tf +++ b/main.tf @@ -18,7 +18,6 @@ resource "azurerm_kubernetes_cluster" "main" { location = coalesce(var.location, data.azurerm_resource_group.main.location) name = "${local.cluster_name}${var.cluster_name_random_suffix ? substr(md5(uuid()), 0, 4) : ""}" resource_group_name = data.azurerm_resource_group.main.name - automatic_channel_upgrade = var.automatic_channel_upgrade azure_policy_enabled = var.azure_policy_enabled cost_analysis_enabled = var.cost_analysis_enabled disk_encryption_set_id = var.disk_encryption_set_id @@ -27,7 +26,6 @@ resource "azurerm_kubernetes_cluster" "main" { image_cleaner_interval_hours = var.image_cleaner_interval_hours kubernetes_version = var.kubernetes_version local_account_disabled = var.local_account_disabled - node_os_channel_upgrade = var.node_os_channel_upgrade node_resource_group = var.node_resource_group oidc_issuer_enabled = var.oidc_issuer_enabled open_service_mesh_enabled = var.open_service_mesh_enabled From 38b28759fff1ce02ddc80420720ee390e7f79306 Mon Sep 17 00:00:00 2001 From: hezijie Date: Wed, 11 Sep 2024 16:33:12 +0800 Subject: [PATCH 15/16] increase acctest timeout --- .github/workflows/acc-test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/acc-test.yaml b/.github/workflows/acc-test.yaml index 5653f073..8c53c6ac 100644 --- a/.github/workflows/acc-test.yaml +++ b/.github/workflows/acc-test.yaml @@ -12,6 +12,7 @@ on: jobs: acc-tests: + timeout-minutes: 720 runs-on: [self-hosted, 1ES.Pool=terraform-azurerm-aks] environment: name: acctests @@ -21,6 +22,8 @@ jobs: run: | docker run --rm -v $(pwd):/src -w /src mcr.microsoft.com/azterraform:latest make generate - name: e2e test + env: + TEST_TIMEOUT: run: | sh scripts/ci-e2e.sh - name: upload test version snapshots From a78e7f8877bf0939d1f762f6f40c22e64c271635 Mon Sep 17 00:00:00 2001 From: hezijie Date: Wed, 11 Sep 2024 16:57:12 +0800 Subject: [PATCH 16/16] increase acctest timeout --- .github/workflows/acc-test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/acc-test.yaml b/.github/workflows/acc-test.yaml index 8c53c6ac..63cfbb82 100644 --- a/.github/workflows/acc-test.yaml +++ b/.github/workflows/acc-test.yaml @@ -22,8 +22,6 @@ jobs: run: | docker run --rm -v $(pwd):/src -w /src mcr.microsoft.com/azterraform:latest make generate - name: e2e test - env: - TEST_TIMEOUT: run: | sh scripts/ci-e2e.sh - name: upload test version snapshots