From 1fe43f65543ad5d2e65fa03888e82377ea820d42 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Wed, 11 Oct 2023 10:50:15 +0200 Subject: [PATCH 01/38] Add Microsoft Defender option Also, create documentation.yml --- README.md | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++ aks.tf | 30 ++++++++++++++ outputs.tf | 5 +++ variables.tf | 10 +++++ versions.tf | 2 +- 5 files changed, 160 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 59fe7c2..77b1ab8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,116 @@ # tf-azure-aks Terraform Module for Azure AKS + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [azurerm](#requirement\_azurerm) | >= 3.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | >= 3.0.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azurerm_kubernetes_cluster.k8s_cluster](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource | +| [azurerm_kubernetes_cluster_node_pool.aks-node](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | +| [azurerm_monitor_diagnostic_setting.aks-diagnostics](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting) | resource | +| [azurerm_subnet.k8s_agent_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | +| [azurerm_virtual_network.k8s_agent_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [admin\_username](#input\_admin\_username) | user name to add to VMs | `string` | `"azureuser"` | no | +| [agent\_net\_name](#input\_agent\_net\_name) | Optional name of the agent vnet | `string` | `"agent-net"` | no | +| [aks\_dns\_service\_ip](#input\_aks\_dns\_service\_ip) | n/a | `string` | `"10.0.0.10"` | no | +| [aks\_docker\_bridge\_cidr](#input\_aks\_docker\_bridge\_cidr) | n/a | `string` | `"172.26.0.1/16"` | no | +| [aks\_network\_plugin](#input\_aks\_network\_plugin) | n/a | `string` | `"azure"` | no | +| [aks\_network\_policy](#input\_aks\_network\_policy) | n/a | `string` | `"calico"` | no | +| [aks\_pod\_cidr](#input\_aks\_pod\_cidr) | n/a | `any` | `null` | no | +| [aks\_service\_cidr](#input\_aks\_service\_cidr) | n/a | `string` | `"10.0.0.0/16"` | no | +| [aks\_vnet\_subnet\_cidr](#input\_aks\_vnet\_subnet\_cidr) | n/a | `string` | `"10.200.0.0/24"` | no | +| [aks\_vnet\_subnet\_id](#input\_aks\_vnet\_subnet\_id) | n/a | `string` | `""` | no | +| [api\_server\_authorized\_ip\_ranges](#input\_api\_server\_authorized\_ip\_ranges) | List of IPs to whitelist for incoming to Kubernetes API | `list(string)` | `[]` | no | +| [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | n/a | `string` | `null` | no | +| [azure\_policy\_enable](#input\_azure\_policy\_enable) | Turn on Azure Policy in cluster or not | `bool` | `false` | no | +| [client\_id](#input\_client\_id) | azure client id | `any` | `null` | no | +| [client\_secret](#input\_client\_secret) | azure client secret | `any` | `null` | no | +| [cluster\_name](#input\_cluster\_name) | What the k8s cluster should be identified as | `any` | n/a | yes | +| [create\_vnet](#input\_create\_vnet) | n/a | `bool` | `true` | no | +| [default\_pool](#input\_default\_pool) | n/a | `map` | `{}` | no | +| [dns\_prefix](#input\_dns\_prefix) | n/a | `any` | n/a | yes | +| [enable\_diagnostics](#input\_enable\_diagnostics) | n/a | `bool` | `false` | no | +| [identity\_ids](#input\_identity\_ids) | n/a | `list(string)` | `[]` | no | +| [identity\_type](#input\_identity\_type) | n/a | `string` | `"SystemAssigned"` | no | +| [ingress\_application\_gateway\_enable](#input\_ingress\_application\_gateway\_enable) | Ingress Application Gateway | `bool` | `false` | no | +| [ingress\_application\_gateway\_name](#input\_ingress\_application\_gateway\_name) | n/a | `string` | `null` | no | +| [ingress\_application\_subnet\_cidr](#input\_ingress\_application\_subnet\_cidr) | n/a | `string` | `null` | no | +| [ingress\_application\_subnet\_id](#input\_ingress\_application\_subnet\_id) | n/a | `string` | `null` | no | +| [k8s\_version](#input\_k8s\_version) | What version of k8s to request from provider | `string` | `"1.11.4"` | no | +| [key\_vault\_secrets\_provider](#input\_key\_vault\_secrets\_provider) | n/a | 
map(object({
    secret_rotation_enabled  = string
    secret_rotation_interval = string
  }))
| `null` | no | +| [kubelet\_identity](#input\_kubelet\_identity) | Identity / RBAC goes here | 
object({
    client_id                 = string
    object_id                 = string
    user_assigned_identity_id = string
  })
| `null` | no | +| [load\_balancer\_sku](#input\_load\_balancer\_sku) | Networking settings. | `string` | `"standard"` | no | +| [log\_analytics](#input\_log\_analytics) | Diagnostics | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | +| [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | +| [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | +| [metrics](#input\_metrics) | n/a | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | +| [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `true` | no | +| [msd\_workspace\_id](#input\_msd\_workspace\_id) | Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to | `string` | `""` | no | +| [node\_pools](#input\_node\_pools) | Node pools to use | `list` | `[]` | no | +| [node\_resource\_group](#input\_node\_resource\_group) | n/a | `any` | `null` | no | +| [oidc\_issuer\_enabled](#input\_oidc\_issuer\_enabled) | n/a | `bool` | `false` | no | +| [oms\_agent\_enable](#input\_oms\_agent\_enable) | Enable OMS Agent profile | `bool` | `true` | no | +| [oms\_workspace\_id](#input\_oms\_workspace\_id) | Operations Management Suite Workspace ID | `string` | `""` | no | +| [outbound\_ip\_address\_ids](#input\_outbound\_ip\_address\_ids) | n/a | `list(any)` | `null` | no | +| [outbound\_ip\_prefix\_ids](#input\_outbound\_ip\_prefix\_ids) | n/a | `list(any)` | `null` | no | +| [outbound\_type](#input\_outbound\_type) | n/a | `string` | `"loadBalancer"` | no | +| [private\_cluster\_enabled](#input\_private\_cluster\_enabled) | n/a | `bool` | `false` | no | +| [private\_dns\_zone\_id](#input\_private\_dns\_zone\_id) | n/a | `string` | `null` | no | +| [rbac\_admin\_group\_ids](#input\_rbac\_admin\_group\_ids) | n/a | `list(any)` | `[]` | no | +| [rbac\_client\_app\_id](#input\_rbac\_client\_app\_id) | The Client ID of an Azure Active Directory Application | `any` | `null` | no | +| [rbac\_enable](#input\_rbac\_enable) | Should RBAC be enabled. | `bool` | `true` | no | +| [rbac\_managed](#input\_rbac\_managed) | n/a | `bool` | `false` | no | +| [rbac\_server\_app\_id](#input\_rbac\_server\_app\_id) | The Server ID of an Azure Active Directory Application | `any` | `null` | no | +| [rbac\_server\_app\_secret](#input\_rbac\_server\_app\_secret) | The Client Secret of an Azure Active Directory Application | `any` | `null` | no | +| [resource\_group\_location](#input\_resource\_group\_location) | Location of the RG the environment will run inside | `string` | `"West Europe"` | no | +| [resource\_group\_name](#input\_resource\_group\_name) | Name of RG the environment will run inside | `any` | n/a | yes | +| [ssh\_public\_key](#input\_ssh\_public\_key) | public key to add to admin\_user in VMs | `any` | n/a | yes | +| [tags](#input\_tags) | # Metadata ## | `map` | `{}` | no | +| [workload\_identity\_enabled](#input\_workload\_identity\_enabled) | n/a | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [id](#output\_id) | n/a | +| [identity](#output\_identity) | n/a | +| [kube\_client\_ca](#output\_kube\_client\_ca) | n/a | +| [kube\_client\_certificate](#output\_kube\_client\_certificate) | n/a | +| [kube\_client\_key](#output\_kube\_client\_key) | n/a | +| [kube\_cluster\_ca](#output\_kube\_cluster\_ca) | n/a | +| [kube\_cluster\_ca\_certificate](#output\_kube\_cluster\_ca\_certificate) | n/a | +| [kube\_config](#output\_kube\_config) | # Outputs ## | +| [kube\_configure](#output\_kube\_configure) | n/a | +| [kube\_host](#output\_kube\_host) | n/a | +| [kube\_password](#output\_kube\_password) | n/a | +| [kube\_username](#output\_kube\_username) | n/a | +| [kubelet\_identity](#output\_kubelet\_identity) | n/a | +| [name](#output\_name) | Re-export the AKS name for usage | +| [network\_profile](#output\_network\_profile) | n/a | +| [node\_resource\_group](#output\_node\_resource\_group) | auto-generated resource group which contains the resources for this managed kubernetes cluster | +| [oidc\_issuer\_url](#output\_oidc\_issuer\_url) | n/a | +| [private\_fqdn](#output\_private\_fqdn) | n/a | + \ No newline at end of file diff --git a/aks.tf b/aks.tf index 6001b4a..6617060 100644 --- a/aks.tf +++ b/aks.tf @@ -246,6 +246,14 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { } } + dynamic "microsoft_defender" { + for_each = var.msd_enable ? [1] : [] + + content { + log_analytics_workspace_id = var.msd_workspace_id + } + } + dynamic "oms_agent" { for_each = var.oms_agent_enable ? [1] : [] @@ -256,6 +264,21 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { tags = var.tags + + # dynamic "lifecycle" { + # for_each = lookup(var.default_pool, "enable_auto_scaling", true) ? [1] : [] + # + # content { + # ignore_changes = [tags,] + # } + # } + # lifecycle { + # ignore_changes = [ + # # Ignore changes to default_node_pools node_count , e.g. because it is managed by enable_auto_scaling + # default_node_pool[0].node_count, + # ] + # } + } resource "azurerm_kubernetes_cluster_node_pool" "aks-node" { @@ -284,6 +307,13 @@ resource "azurerm_kubernetes_cluster_node_pool" "aks-node" { priority = each.value.priority eviction_policy = each.value.eviction_policy spot_max_price = each.value.spot_max_price + + lifecycle { + ignore_changes = [ + # Ignore changes to default_node_pools node_count , e.g. because it is managed by enable_auto_scaling + node_count, + ] + } } resource "azurerm_monitor_diagnostic_setting" "aks-diagnostics" { diff --git a/outputs.tf b/outputs.tf index 27b525f..b91ef17 100644 --- a/outputs.tf +++ b/outputs.tf @@ -86,3 +86,8 @@ output "private_fqdn" { output "oidc_issuer_url" { value = azurerm_kubernetes_cluster.k8s_cluster.oidc_issuer_url } + +output "node_resource_group" { + description = "auto-generated resource group which contains the resources for this managed kubernetes cluster" + value = azurerm_kubernetes_cluster.k8s_cluster.node_resource_group +} diff --git a/variables.tf b/variables.tf index 33625e7..0dd3f9a 100644 --- a/variables.tf +++ b/variables.tf @@ -209,6 +209,16 @@ variable "oms_agent_enable" { default = true } +variable "msd_workspace_id" { + description = "Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to" + default = "" +} + +variable "msd_enable" { + description = "Enable audit logs collected by Microsoft Defender" + default = true +} + variable "enable_diagnostics" { default = false type = bool diff --git a/versions.tf b/versions.tf index 9d76625..a065769 100644 --- a/versions.tf +++ b/versions.tf @@ -3,6 +3,6 @@ terraform { required_version = ">= 1.0" required_providers { - azurerm = ">= 3.39.0" + azurerm = ">= 3.0.0" } } From 25c97c83dbb0f3d1a3122cc7e2d0ecc5e4a29454 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Wed, 11 Oct 2023 10:59:53 +0200 Subject: [PATCH 02/38] Add documentation pileline --- .github/workflows/documentation.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/documentation.yml diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..d16a556 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,19 @@ +name: Generate terraform docs +on: + - pull_request + +jobs: + docs: + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - name: Render terraform docs and push changes back to PR + uses: terraform-docs/gh-actions@main + with: + working-dir: . + output-file: README.md + output-method: inject + git-push: "true" From f77fb2ad5cce1451fe603d8e7926689702a52361 Mon Sep 17 00:00:00 2001 From: Carsten Thiel Date: Wed, 11 Oct 2023 11:15:55 +0200 Subject: [PATCH 03/38] Fixing default for msd_enable variable --- variables.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 0dd3f9a..f1a3633 100644 --- a/variables.tf +++ b/variables.tf @@ -215,8 +215,9 @@ variable "msd_workspace_id" { } variable "msd_enable" { + type = bool description = "Enable audit logs collected by Microsoft Defender" - default = true + default = false } variable "enable_diagnostics" { From 2005276769e79989963e8c3e3f30c1e1b884e9bd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 11 Oct 2023 09:17:22 +0000 Subject: [PATCH 04/38] terraform-docs: automated action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 77b1ab8..d048d57 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ No modules. | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | | [metrics](#input\_metrics) | n/a | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | -| [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `true` | no | +| [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `false` | no | | [msd\_workspace\_id](#input\_msd\_workspace\_id) | Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to | `string` | `""` | no | | [node\_pools](#input\_node\_pools) | Node pools to use | `list` | `[]` | no | | [node\_resource\_group](#input\_node\_resource\_group) | n/a | `any` | `null` | no | From fee7080cd19b87bcd747ac6aa09e71a02326022b Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Fri, 20 Oct 2023 10:39:39 +0200 Subject: [PATCH 05/38] using authorized_ip_ranges in sted of api_server_authorized_ip_ranges --- aks.tf | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/aks.tf b/aks.tf index 6617060..6e7844f 100644 --- a/aks.tf +++ b/aks.tf @@ -114,15 +114,17 @@ resource "azurerm_subnet" "k8s_agent_subnet" { } resource "azurerm_kubernetes_cluster" "k8s_cluster" { - name = var.cluster_name - location = var.resource_group_location - resource_group_name = var.resource_group_name - dns_prefix = var.dns_prefix - private_cluster_enabled = var.private_cluster_enabled - private_dns_zone_id = var.private_dns_zone_id - kubernetes_version = var.k8s_version - api_server_authorized_ip_ranges = var.api_server_authorized_ip_ranges - automatic_channel_upgrade = var.automatic_channel_upgrade + name = var.cluster_name + location = var.resource_group_location + resource_group_name = var.resource_group_name + dns_prefix = var.dns_prefix + private_cluster_enabled = var.private_cluster_enabled + private_dns_zone_id = var.private_dns_zone_id + kubernetes_version = var.k8s_version + api_server_access_profile { + authorized_ip_ranges = var.api_server_authorized_ip_ranges + } + automatic_channel_upgrade = var.automatic_channel_upgrade linux_profile { admin_username = var.admin_username From 2f6dc87452c3bf7b13a7302a9723ba01f67839e4 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Fri, 20 Oct 2023 12:33:59 +0200 Subject: [PATCH 06/38] Add dependabot --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8cce2e1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "terraform" + directory: "/" + schedule: + interval: "daily" + time: "23:00" + timezone: "Europe/Oslo" + open-pull-requests-limit: 3 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file From 8bace8d7baf8bfc3ebabfbd553caeb7698e5b481 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Fri, 20 Oct 2023 13:38:07 +0200 Subject: [PATCH 07/38] docker_bridge_cidr is depricated --- aks.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/aks.tf b/aks.tf index 6e7844f..6230181 100644 --- a/aks.tf +++ b/aks.tf @@ -205,7 +205,6 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { pod_cidr = var.aks_pod_cidr service_cidr = var.aks_service_cidr dns_service_ip = var.aks_dns_service_ip - docker_bridge_cidr = var.aks_docker_bridge_cidr dynamic "load_balancer_profile" { for_each = var.outbound_type == "loadBalancer" ? [1] : [] From 8b520f43c64f8f8542510887d2efd44aeb6dddd4 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Thu, 9 Nov 2023 09:51:16 +0100 Subject: [PATCH 08/38] make api_server_access_profile dynamic based on if api_server_authorized_ip_ranges has values --- aks.tf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/aks.tf b/aks.tf index 6230181..7022cbe 100644 --- a/aks.tf +++ b/aks.tf @@ -121,8 +121,11 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { private_cluster_enabled = var.private_cluster_enabled private_dns_zone_id = var.private_dns_zone_id kubernetes_version = var.k8s_version - api_server_access_profile { - authorized_ip_ranges = var.api_server_authorized_ip_ranges + dynamic "api_server_access_profile" { + for_each = length(var.api_server_authorized_ip_ranges) != 0 ? [1] : [] + content { + authorized_ip_ranges = var.api_server_authorized_ip_ranges + } } automatic_channel_upgrade = var.automatic_channel_upgrade From 46eb0e13549a6ffa640ea9d11ce823220d273eb0 Mon Sep 17 00:00:00 2001 From: Artjoms Laivins Date: Thu, 23 Nov 2023 12:57:44 +0100 Subject: [PATCH 09/38] Added output for node_pool_rg ID --- outputs.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/outputs.tf b/outputs.tf index b91ef17..cef8b2c 100644 --- a/outputs.tf +++ b/outputs.tf @@ -91,3 +91,8 @@ output "node_resource_group" { description = "auto-generated resource group which contains the resources for this managed kubernetes cluster" value = azurerm_kubernetes_cluster.k8s_cluster.node_resource_group } + +output "node_resource_group_id" { + description = "auto-generated resource group which contains the resources for this managed kubernetes cluster" + value = azurerm_kubernetes_cluster.k8s_cluster.node_resource_group_id +} \ No newline at end of file From 5b5d865844777a3322616f3c10dd8f8cbeccb741 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 23 Nov 2023 12:01:04 +0000 Subject: [PATCH 10/38] terraform-docs: automated action --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d048d57..beaa700 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ No modules. | [name](#output\_name) | Re-export the AKS name for usage | | [network\_profile](#output\_network\_profile) | n/a | | [node\_resource\_group](#output\_node\_resource\_group) | auto-generated resource group which contains the resources for this managed kubernetes cluster | +| [node\_resource\_group\_id](#output\_node\_resource\_group\_id) | auto-generated resource group which contains the resources for this managed kubernetes cluster | | [oidc\_issuer\_url](#output\_oidc\_issuer\_url) | n/a | | [private\_fqdn](#output\_private\_fqdn) | n/a | \ No newline at end of file From 2bd6b4557bd58281e66d9abcb09f79d1300d9444 Mon Sep 17 00:00:00 2001 From: Carsten Thiel Date: Fri, 19 Jan 2024 10:55:35 +0100 Subject: [PATCH 11/38] Adding rbac as explciit setting This is a #minor release, as this only makes the default explicit --- aks.tf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/aks.tf b/aks.tf index 7022cbe..efd135e 100644 --- a/aks.tf +++ b/aks.tf @@ -29,7 +29,7 @@ locals { os_type = lookup(p, "os_type", local.default_pool_settings.os_type) os_disk_size_gb = lookup(p, "os_disk_size_gb", local.default_pool_settings.os_disk_size_gb) os_disk_type = lookup(p, "os_disk_type", local.default_pool_settings.os_disk_type) - vnet_subnet_id = var.create_vnet ? element(concat(azurerm_subnet.k8s_agent_subnet.*.id, [""]), 0) : var.aks_vnet_subnet_id + vnet_subnet_id = var.create_vnet ? element(concat(azurerm_subnet.k8s_agent_subnet[*].id, [""]), 0) : var.aks_vnet_subnet_id zones = lookup(p, "zones", local.default_pool_settings.zones) mode = lookup(p, "mode", "User") @@ -140,6 +140,8 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { oidc_issuer_enabled = var.oidc_issuer_enabled workload_identity_enabled = var.workload_identity_enabled + role_based_access_control_enabled = true + node_resource_group = var.node_resource_group #if No aks_vnet_subnet_id is passed THEN use newly created subnet id ELSE use PASSED subnet id @@ -149,7 +151,7 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { vm_size = lookup(var.default_pool, "vm_size", local.default_pool_settings.vm_size) os_disk_size_gb = lookup(var.default_pool, "os_disk_size_gb", local.default_pool_settings.os_disk_size_gb) os_disk_type = lookup(var.default_pool, "os_disk_type", local.default_pool_settings.os_disk_type) - vnet_subnet_id = var.create_vnet ? element(concat(azurerm_subnet.k8s_agent_subnet.*.id, [""]), 0) : var.aks_vnet_subnet_id + vnet_subnet_id = var.create_vnet ? element(concat(azurerm_subnet.k8s_agent_subnet[*].id, [""]), 0) : var.aks_vnet_subnet_id zones = lookup(var.default_pool, "zones", local.default_pool_settings.zones) type = lookup(var.default_pool, "type", local.default_pool_settings.default_pool_type) enable_auto_scaling = lookup(var.default_pool, "enable_auto_scaling", true) @@ -205,9 +207,9 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { network_plugin = var.aks_network_plugin network_policy = var.aks_network_policy - pod_cidr = var.aks_pod_cidr - service_cidr = var.aks_service_cidr - dns_service_ip = var.aks_dns_service_ip + pod_cidr = var.aks_pod_cidr + service_cidr = var.aks_service_cidr + dns_service_ip = var.aks_dns_service_ip dynamic "load_balancer_profile" { for_each = var.outbound_type == "loadBalancer" ? [1] : [] From 09f23a82d65d034186a074997635e27cfad72c75 Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Tue, 5 Mar 2024 16:00:02 +0100 Subject: [PATCH 12/38] Add Windows maintenance and security updates --- aks.tf | 58 ++++++++++++++++++++++++++++++++++++++++++ variables.tf | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/aks.tf b/aks.tf index efd135e..23df933 100644 --- a/aks.tf +++ b/aks.tf @@ -129,6 +129,64 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { } automatic_channel_upgrade = var.automatic_channel_upgrade + dynamic "maintenance_window_allowed" { + for_each = var.maintenance_window.allowed + content { + allowed { + day = maintenance_window_allowed.value.day + hours = maintenance_window_allowed.value.hours + } + } + } + + dynamic "maintenance_window_not_allowed" { + for_each = var.maintenance_window.not_allowed + content { + not_allowed { + start = maintenance_window_not_allowed.value.start + end = maintenance_window_not_allowed.value.end + } + } + } + + maintenance_window_auto_upgrade { + frequency = var.frequency + interval = var.interval + duration = var.duration + day_of_week = var.day_of_week + day_of_month = var.day_of_month + week_index = var.week_index + start_time = var.start_time + utc_offset = var.utc_offset + start_date = var.start_date + not_allowed { + start = maintenance_window_not_allowed.value.start + end = maintenance_window_not_allowed.value.end + } + } + + + node_os_channel_upgrade = var.node_os_channel_upgrade + + maintenance_window_node_os { + frequency = var.frequency + interval = var.interval + duration = var.duration + day_of_week = var.day_of_week + day_of_month = var.day_of_month + week_index = var.week_index + start_time = var.start_time + utc_offset = var.utc_offset + start_date = var.start_date + not_allowed { + start = maintenance_window_not_allowed.value.start + end = maintenance_window_not_allowed.value.end + } + } + upgrade_settings { + max_surge = var.max_surge + } + linux_profile { admin_username = var.admin_username diff --git a/variables.tf b/variables.tf index f1a3633..a755d40 100644 --- a/variables.tf +++ b/variables.tf @@ -274,6 +274,78 @@ variable "automatic_channel_upgrade" { default = null } +variable "maintenance_window" { + description = "Maintenance window configuration" + type = object({ + allowed = map(object({ + day = string + hours = list(number) + })) + not_allowed = map(object({ + start = string + end = string + })) + }) +} + +variable "node_os_channel_upgrade" { + type = string + default = "NodeImage" + description = "automatically upgrades the node image to the latest version available." +} + +variable "max_surge" { + type = string + default = null + description = "The maximum percentage of nodes which will be added to the Node Pool size during an upgrade" +} + +variable "frequency" { + description = "Frequency of maintenance." + type = string +} + +variable "interval" { + description = "The interval for maintenance runs." + type = number +} + +variable "duration" { + description = "The duration of the window for maintenance to run in hours." + type = string +} + +variable "day_of_week" { + description = "The day of the week for the maintenance run." + type = string +} + +variable "day_of_month" { + description = "The day of the month for the maintenance run." + type = number +} + +variable "week_index" { + description = "Specifies on which instance of the allowed days specified in day_of_week the maintenance occurs." + type = string +} + +variable "start_time" { + description = "The time for maintenance to begin, based on the timezone determined by utc_offset." + type = string +} + +variable "utc_offset" { + description = "Used to determine the timezone for cluster maintenance." + type = string +} + +variable "start_date" { + description = "The date on which the maintenance window begins to take effect." + type = string +} + + # Ingress Application Gateway variable "ingress_application_gateway_enable" { type = bool From 0dbeb39377aaa157c3d1082ba20f91ab7f7bde6d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Mar 2024 15:00:47 +0000 Subject: [PATCH 13/38] terraform-docs: automated action --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index beaa700..611cdef 100644 --- a/README.md +++ b/README.md @@ -50,25 +50,33 @@ No modules. | [client\_secret](#input\_client\_secret) | azure client secret | `any` | `null` | no | | [cluster\_name](#input\_cluster\_name) | What the k8s cluster should be identified as | `any` | n/a | yes | | [create\_vnet](#input\_create\_vnet) | n/a | `bool` | `true` | no | +| [day\_of\_month](#input\_day\_of\_month) | The day of the month for the maintenance run. | `number` | n/a | yes | +| [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | n/a | yes | | [default\_pool](#input\_default\_pool) | n/a | `map` | `{}` | no | | [dns\_prefix](#input\_dns\_prefix) | n/a | `any` | n/a | yes | +| [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | n/a | yes | | [enable\_diagnostics](#input\_enable\_diagnostics) | n/a | `bool` | `false` | no | +| [frequency](#input\_frequency) | Frequency of maintenance. | `string` | n/a | yes | | [identity\_ids](#input\_identity\_ids) | n/a | `list(string)` | `[]` | no | | [identity\_type](#input\_identity\_type) | n/a | `string` | `"SystemAssigned"` | no | | [ingress\_application\_gateway\_enable](#input\_ingress\_application\_gateway\_enable) | Ingress Application Gateway | `bool` | `false` | no | | [ingress\_application\_gateway\_name](#input\_ingress\_application\_gateway\_name) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_cidr](#input\_ingress\_application\_subnet\_cidr) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_id](#input\_ingress\_application\_subnet\_id) | n/a | `string` | `null` | no | +| [interval](#input\_interval) | The interval for maintenance runs. | `number` | n/a | yes | | [k8s\_version](#input\_k8s\_version) | What version of k8s to request from provider | `string` | `"1.11.4"` | no | | [key\_vault\_secrets\_provider](#input\_key\_vault\_secrets\_provider) | n/a | 
map(object({
    secret_rotation_enabled  = string
    secret_rotation_interval = string
  }))
| `null` | no | | [kubelet\_identity](#input\_kubelet\_identity) | Identity / RBAC goes here | 
object({
    client_id                 = string
    object_id                 = string
    user_assigned_identity_id = string
  })
| `null` | no | | [load\_balancer\_sku](#input\_load\_balancer\_sku) | Networking settings. | `string` | `"standard"` | no | | [log\_analytics](#input\_log\_analytics) | Diagnostics | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | +| [maintenance\_window](#input\_maintenance\_window) | Maintenance window configuration | 
object({
    allowed = map(object({
      day   = string
      hours = list(number)
    }))
    not_allowed = map(object({
      start = string
      end   = string
    }))
  })
| n/a | yes | | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | +| [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `null` | no | | [metrics](#input\_metrics) | n/a | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | | [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `false` | no | | [msd\_workspace\_id](#input\_msd\_workspace\_id) | Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to | `string` | `""` | no | +| [node\_os\_channel\_upgrade](#input\_node\_os\_channel\_upgrade) | automatically upgrades the node image to the latest version available. | `string` | `"NodeImage"` | no | | [node\_pools](#input\_node\_pools) | Node pools to use | `list` | `[]` | no | | [node\_resource\_group](#input\_node\_resource\_group) | n/a | `any` | `null` | no | | [oidc\_issuer\_enabled](#input\_oidc\_issuer\_enabled) | n/a | `bool` | `false` | no | @@ -88,7 +96,11 @@ No modules. | [resource\_group\_location](#input\_resource\_group\_location) | Location of the RG the environment will run inside | `string` | `"West Europe"` | no | | [resource\_group\_name](#input\_resource\_group\_name) | Name of RG the environment will run inside | `any` | n/a | yes | | [ssh\_public\_key](#input\_ssh\_public\_key) | public key to add to admin\_user in VMs | `any` | n/a | yes | +| [start\_date](#input\_start\_date) | The date on which the maintenance window begins to take effect. | `string` | n/a | yes | +| [start\_time](#input\_start\_time) | The time for maintenance to begin, based on the timezone determined by utc\_offset. | `string` | n/a | yes | | [tags](#input\_tags) | # Metadata ## | `map` | `{}` | no | +| [utc\_offset](#input\_utc\_offset) | Used to determine the timezone for cluster maintenance. | `string` | n/a | yes | +| [week\_index](#input\_week\_index) | Specifies on which instance of the allowed days specified in day\_of\_week the maintenance occurs. | `string` | n/a | yes | | [workload\_identity\_enabled](#input\_workload\_identity\_enabled) | n/a | `bool` | `false` | no | ## Outputs From 5427e9c8a9f15bcdd186c162f891fbcc50f60ea0 Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Wed, 6 Mar 2024 12:01:29 +0100 Subject: [PATCH 14/38] Update TF module and rewrite dynamic block --- .terraform.lock.hcl | 22 ++++++++++++++++++ aks.tf | 55 ++++++++++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 .terraform.lock.hcl diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..4510869 --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.94.0" + constraints = ">= 3.0.0" + hashes = [ + "h1:t3fM/PO8PLAA5mK3esAypp01V6Vh75kjPnNqxQeVrV0=", + "zh:20d102bc63096ade82f8da81c91afaffa858aa56fe9a7ad02f24f5ae5618bc53", + "zh:3ddb9d6173a4fdb9b2352a76324ee321976915544ae66cbb863c7a60f0593f05", + "zh:4bc6c62142f67192d2def11f4fd419c54dddd89a5448af036bfc60b15eb0509a", + "zh:4c5120c2101a51524af32c4220c5e376f97a227730dd92ec0b06ac677e4b39f2", + "zh:585fa7ab876d09899cd2d842f12bc28c34556b4d47919eceadefab6fa47f909f", + "zh:59de7ea462470dee7088fc4deeff48e1ffd286eaca1185c219be68dadde745b8", + "zh:8421a46dd3bc4bc2eb56f7eb9b91cc84a66070b72195a805862c6022adee2da0", + "zh:a2fcb5a091d5944dc50f1e51f53fa4d370810a507fbf4122920d756083d8df19", + "zh:beb6b93a2a16942625bb6ac1e52bf26878e35f5562f3173279423ca66553b6d7", + "zh:c6846892ea68f49c838d90b75793d1f3a866871dd701ccb575b1eecccd4e7051", + "zh:ddd59492b6d5ce4c83f06a5b16c520048f3e9bb898bab4f3910042f5c01ffeda", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/aks.tf b/aks.tf index 23df933..7e5f87b 100644 --- a/aks.tf +++ b/aks.tf @@ -129,26 +129,6 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { } automatic_channel_upgrade = var.automatic_channel_upgrade - dynamic "maintenance_window_allowed" { - for_each = var.maintenance_window.allowed - content { - allowed { - day = maintenance_window_allowed.value.day - hours = maintenance_window_allowed.value.hours - } - } - } - - dynamic "maintenance_window_not_allowed" { - for_each = var.maintenance_window.not_allowed - content { - not_allowed { - start = maintenance_window_not_allowed.value.start - end = maintenance_window_not_allowed.value.end - } - } - } - maintenance_window_auto_upgrade { frequency = var.frequency interval = var.interval @@ -159,9 +139,21 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { start_time = var.start_time utc_offset = var.utc_offset start_date = var.start_date - not_allowed { - start = maintenance_window_not_allowed.value.start - end = maintenance_window_not_allowed.value.end + + dynamic "allowed" { + for_each = maintenance_window_auto_upgrade.value.allowed + content { + day = allowed.value.day + hours = allowed.value.hours + } + } + + dynamic "not_allowed" { + for_each = maintenance_window_auto_upgrade.value.not_allowed + content { + start = not_allowed.value.start + end = not_allowed.value.end + } } } @@ -178,9 +170,20 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { start_time = var.start_time utc_offset = var.utc_offset start_date = var.start_date - not_allowed { - start = maintenance_window_not_allowed.value.start - end = maintenance_window_not_allowed.value.end + + dynamic "allowed" { + for_each = var.maintenance_window.allowed + content { + day = allowed.value.day + hours = allowed.value.hours + } + } + dynamic "not_allowed" { + for_each = var.maintenance_window.not_allowed + content { + start = not_allowed.value.start + end = not_allowed.value.end + } } } upgrade_settings { From 514fbe90a1b6e4616fd2fe6876ef9bba35dd5857 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Mar 2024 11:01:53 +0000 Subject: [PATCH 15/38] terraform-docs: automated action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 611cdef..5faf04a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Terraform Module for Azure AKS | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | >= 3.0.0 | +| [azurerm](#provider\_azurerm) | 3.94.0 | ## Modules From cf8887542ee755aa600b762b2eeb3f61f64f3f22 Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Wed, 6 Mar 2024 12:20:09 +0100 Subject: [PATCH 16/38] Rewriting TF module --- aks.tf | 26 +++++++++++++------------- variables.tf | 24 ++++++++++++------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/aks.tf b/aks.tf index 7e5f87b..1f53f69 100644 --- a/aks.tf +++ b/aks.tf @@ -139,20 +139,19 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { start_time = var.start_time utc_offset = var.utc_offset start_date = var.start_date - dynamic "allowed" { - for_each = maintenance_window_auto_upgrade.value.allowed + for_each = var.allowed content { - day = allowed.value.day - hours = allowed.value.hours + day = var.allowed.day + hours = var.allowed.hours } } dynamic "not_allowed" { - for_each = maintenance_window_auto_upgrade.value.not_allowed + for_each = var.not_allowed content { - start = not_allowed.value.start - end = not_allowed.value.end + start = var.not_allowed.start + end = var.not_allowed.end } } } @@ -172,17 +171,18 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { start_date = var.start_date dynamic "allowed" { - for_each = var.maintenance_window.allowed + for_each = var.allowed content { - day = allowed.value.day - hours = allowed.value.hours + day = var.allowed.day + hours = var.allowed.hours } } + dynamic "not_allowed" { - for_each = var.maintenance_window.not_allowed + for_each = var.not_allowed content { - start = not_allowed.value.start - end = not_allowed.value.end + start = var.not_allowed.start + end = var.not_allowed.end } } } diff --git a/variables.tf b/variables.tf index a755d40..145dfca 100644 --- a/variables.tf +++ b/variables.tf @@ -274,18 +274,18 @@ variable "automatic_channel_upgrade" { default = null } -variable "maintenance_window" { - description = "Maintenance window configuration" - type = object({ - allowed = map(object({ - day = string - hours = list(number) - })) - not_allowed = map(object({ - start = string - end = string - })) - }) +variable "allowed" { + type = map(object({ + day = string + hours = list(number) + })) +} + +variable "not_allowed" { + type = map(object({ + start = string + end = string + })) } variable "node_os_channel_upgrade" { From 2cac22d0cc9ab276ba7d16a1d1e083d40642ed15 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Mar 2024 11:21:43 +0000 Subject: [PATCH 17/38] terraform-docs: automated action --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5faf04a..c6d65f6 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ No modules. | [aks\_service\_cidr](#input\_aks\_service\_cidr) | n/a | `string` | `"10.0.0.0/16"` | no | | [aks\_vnet\_subnet\_cidr](#input\_aks\_vnet\_subnet\_cidr) | n/a | `string` | `"10.200.0.0/24"` | no | | [aks\_vnet\_subnet\_id](#input\_aks\_vnet\_subnet\_id) | n/a | `string` | `""` | no | +| [allowed](#input\_allowed) | n/a | 
map(object({
    day   = string
    hours = list(number)
  }))
| n/a | yes | | [api\_server\_authorized\_ip\_ranges](#input\_api\_server\_authorized\_ip\_ranges) | List of IPs to whitelist for incoming to Kubernetes API | `list(string)` | `[]` | no | | [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | n/a | `string` | `null` | no | | [azure\_policy\_enable](#input\_azure\_policy\_enable) | Turn on Azure Policy in cluster or not | `bool` | `false` | no | @@ -69,7 +70,6 @@ No modules. | [kubelet\_identity](#input\_kubelet\_identity) | Identity / RBAC goes here | 
object({
    client_id                 = string
    object_id                 = string
    user_assigned_identity_id = string
  })
| `null` | no | | [load\_balancer\_sku](#input\_load\_balancer\_sku) | Networking settings. | `string` | `"standard"` | no | | [log\_analytics](#input\_log\_analytics) | Diagnostics | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | -| [maintenance\_window](#input\_maintenance\_window) | Maintenance window configuration | 
object({
    allowed = map(object({
      day   = string
      hours = list(number)
    }))
    not_allowed = map(object({
      start = string
      end   = string
    }))
  })
| n/a | yes | | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | | [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `null` | no | @@ -79,6 +79,7 @@ No modules. | [node\_os\_channel\_upgrade](#input\_node\_os\_channel\_upgrade) | automatically upgrades the node image to the latest version available. | `string` | `"NodeImage"` | no | | [node\_pools](#input\_node\_pools) | Node pools to use | `list` | `[]` | no | | [node\_resource\_group](#input\_node\_resource\_group) | n/a | `any` | `null` | no | +| [not\_allowed](#input\_not\_allowed) | n/a | 
map(object({
    start = string
    end   = string
  }))
| n/a | yes | | [oidc\_issuer\_enabled](#input\_oidc\_issuer\_enabled) | n/a | `bool` | `false` | no | | [oms\_agent\_enable](#input\_oms\_agent\_enable) | Enable OMS Agent profile | `bool` | `true` | no | | [oms\_workspace\_id](#input\_oms\_workspace\_id) | Operations Management Suite Workspace ID | `string` | `""` | no | From 1e843107d1416ea21d0164a10026e88731d6d311 Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Wed, 6 Mar 2024 12:47:15 +0100 Subject: [PATCH 18/38] Update Module --- aks.tf | 20 +++----------------- variables.tf | 7 ------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/aks.tf b/aks.tf index 1f53f69..1374b75 100644 --- a/aks.tf +++ b/aks.tf @@ -139,13 +139,6 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { start_time = var.start_time utc_offset = var.utc_offset start_date = var.start_date - dynamic "allowed" { - for_each = var.allowed - content { - day = var.allowed.day - hours = var.allowed.hours - } - } dynamic "not_allowed" { for_each = var.not_allowed @@ -170,13 +163,6 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { utc_offset = var.utc_offset start_date = var.start_date - dynamic "allowed" { - for_each = var.allowed - content { - day = var.allowed.day - hours = var.allowed.hours - } - } dynamic "not_allowed" { for_each = var.not_allowed @@ -186,9 +172,6 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { } } } - upgrade_settings { - max_surge = var.max_surge - } linux_profile { admin_username = var.admin_username @@ -221,6 +204,9 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { tags = lookup(var.default_pool, "tags", var.tags) max_pods = lookup(var.default_pool, "max_pods", local.default_pool_settings.max_pods) orchestrator_version = lookup(var.default_pool, "k8s_version", local.default_pool_settings.k8s_version) + upgrade_settings { + max_surge = var.max_surge + } } dynamic "service_principal" { diff --git a/variables.tf b/variables.tf index 145dfca..cc9a4c8 100644 --- a/variables.tf +++ b/variables.tf @@ -274,13 +274,6 @@ variable "automatic_channel_upgrade" { default = null } -variable "allowed" { - type = map(object({ - day = string - hours = list(number) - })) -} - variable "not_allowed" { type = map(object({ start = string From 7422a7e55bdada1122efd917cb18f9a08888f81f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Mar 2024 11:47:33 +0000 Subject: [PATCH 19/38] terraform-docs: automated action --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c6d65f6..5c06544 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ No modules. | [aks\_service\_cidr](#input\_aks\_service\_cidr) | n/a | `string` | `"10.0.0.0/16"` | no | | [aks\_vnet\_subnet\_cidr](#input\_aks\_vnet\_subnet\_cidr) | n/a | `string` | `"10.200.0.0/24"` | no | | [aks\_vnet\_subnet\_id](#input\_aks\_vnet\_subnet\_id) | n/a | `string` | `""` | no | -| [allowed](#input\_allowed) | n/a | 
map(object({
    day   = string
    hours = list(number)
  }))
| n/a | yes | | [api\_server\_authorized\_ip\_ranges](#input\_api\_server\_authorized\_ip\_ranges) | List of IPs to whitelist for incoming to Kubernetes API | `list(string)` | `[]` | no | | [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | n/a | `string` | `null` | no | | [azure\_policy\_enable](#input\_azure\_policy\_enable) | Turn on Azure Policy in cluster or not | `bool` | `false` | no | From a063f21c589ed5b9e499aedc5b25517f4aaa964c Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Wed, 6 Mar 2024 12:54:07 +0100 Subject: [PATCH 20/38] Add default null --- variables.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/variables.tf b/variables.tf index cc9a4c8..3c23f2d 100644 --- a/variables.tf +++ b/variables.tf @@ -296,46 +296,55 @@ variable "max_surge" { variable "frequency" { description = "Frequency of maintenance." type = string + default = null } variable "interval" { description = "The interval for maintenance runs." type = number + default = null } variable "duration" { description = "The duration of the window for maintenance to run in hours." type = string + default = null } variable "day_of_week" { description = "The day of the week for the maintenance run." type = string + default = null } variable "day_of_month" { description = "The day of the month for the maintenance run." type = number + default = null } variable "week_index" { description = "Specifies on which instance of the allowed days specified in day_of_week the maintenance occurs." type = string + default = null } variable "start_time" { description = "The time for maintenance to begin, based on the timezone determined by utc_offset." type = string + default = null } variable "utc_offset" { description = "Used to determine the timezone for cluster maintenance." type = string + default = null } variable "start_date" { description = "The date on which the maintenance window begins to take effect." type = string + default = null } From 9d5e2f8e163f6002212657d14ce4a701039d338e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Mar 2024 11:54:26 +0000 Subject: [PATCH 21/38] terraform-docs: automated action --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5c06544..eade542 100644 --- a/README.md +++ b/README.md @@ -50,20 +50,20 @@ No modules. | [client\_secret](#input\_client\_secret) | azure client secret | `any` | `null` | no | | [cluster\_name](#input\_cluster\_name) | What the k8s cluster should be identified as | `any` | n/a | yes | | [create\_vnet](#input\_create\_vnet) | n/a | `bool` | `true` | no | -| [day\_of\_month](#input\_day\_of\_month) | The day of the month for the maintenance run. | `number` | n/a | yes | -| [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | n/a | yes | +| [day\_of\_month](#input\_day\_of\_month) | The day of the month for the maintenance run. | `number` | `null` | no | +| [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | `null` | no | | [default\_pool](#input\_default\_pool) | n/a | `map` | `{}` | no | | [dns\_prefix](#input\_dns\_prefix) | n/a | `any` | n/a | yes | -| [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | n/a | yes | +| [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | `null` | no | | [enable\_diagnostics](#input\_enable\_diagnostics) | n/a | `bool` | `false` | no | -| [frequency](#input\_frequency) | Frequency of maintenance. | `string` | n/a | yes | +| [frequency](#input\_frequency) | Frequency of maintenance. | `string` | `null` | no | | [identity\_ids](#input\_identity\_ids) | n/a | `list(string)` | `[]` | no | | [identity\_type](#input\_identity\_type) | n/a | `string` | `"SystemAssigned"` | no | | [ingress\_application\_gateway\_enable](#input\_ingress\_application\_gateway\_enable) | Ingress Application Gateway | `bool` | `false` | no | | [ingress\_application\_gateway\_name](#input\_ingress\_application\_gateway\_name) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_cidr](#input\_ingress\_application\_subnet\_cidr) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_id](#input\_ingress\_application\_subnet\_id) | n/a | `string` | `null` | no | -| [interval](#input\_interval) | The interval for maintenance runs. | `number` | n/a | yes | +| [interval](#input\_interval) | The interval for maintenance runs. | `number` | `null` | no | | [k8s\_version](#input\_k8s\_version) | What version of k8s to request from provider | `string` | `"1.11.4"` | no | | [key\_vault\_secrets\_provider](#input\_key\_vault\_secrets\_provider) | n/a | 
map(object({
    secret_rotation_enabled  = string
    secret_rotation_interval = string
  }))
| `null` | no | | [kubelet\_identity](#input\_kubelet\_identity) | Identity / RBAC goes here | 
object({
    client_id                 = string
    object_id                 = string
    user_assigned_identity_id = string
  })
| `null` | no | @@ -96,11 +96,11 @@ No modules. | [resource\_group\_location](#input\_resource\_group\_location) | Location of the RG the environment will run inside | `string` | `"West Europe"` | no | | [resource\_group\_name](#input\_resource\_group\_name) | Name of RG the environment will run inside | `any` | n/a | yes | | [ssh\_public\_key](#input\_ssh\_public\_key) | public key to add to admin\_user in VMs | `any` | n/a | yes | -| [start\_date](#input\_start\_date) | The date on which the maintenance window begins to take effect. | `string` | n/a | yes | -| [start\_time](#input\_start\_time) | The time for maintenance to begin, based on the timezone determined by utc\_offset. | `string` | n/a | yes | +| [start\_date](#input\_start\_date) | The date on which the maintenance window begins to take effect. | `string` | `null` | no | +| [start\_time](#input\_start\_time) | The time for maintenance to begin, based on the timezone determined by utc\_offset. | `string` | `null` | no | | [tags](#input\_tags) | # Metadata ## | `map` | `{}` | no | -| [utc\_offset](#input\_utc\_offset) | Used to determine the timezone for cluster maintenance. | `string` | n/a | yes | -| [week\_index](#input\_week\_index) | Specifies on which instance of the allowed days specified in day\_of\_week the maintenance occurs. | `string` | n/a | yes | +| [utc\_offset](#input\_utc\_offset) | Used to determine the timezone for cluster maintenance. | `string` | `null` | no | +| [week\_index](#input\_week\_index) | Specifies on which instance of the allowed days specified in day\_of\_week the maintenance occurs. | `string` | `null` | no | | [workload\_identity\_enabled](#input\_workload\_identity\_enabled) | n/a | `bool` | `false` | no | ## Outputs From c4a7496d2a9ba9ca0cb1efd781afa89e27967dcc Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Wed, 6 Mar 2024 13:55:32 +0100 Subject: [PATCH 22/38] Fix bugs --- aks.tf | 8 ++++---- variables.tf | 16 +++++++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/aks.tf b/aks.tf index 1374b75..66bb59d 100644 --- a/aks.tf +++ b/aks.tf @@ -143,8 +143,8 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { dynamic "not_allowed" { for_each = var.not_allowed content { - start = var.not_allowed.start - end = var.not_allowed.end + start = not_allowed.value["start"] + end = not_allowed.value["end"] } } } @@ -167,8 +167,8 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { dynamic "not_allowed" { for_each = var.not_allowed content { - start = var.not_allowed.start - end = var.not_allowed.end + start = not_allowed.value["start"] + end = not_allowed.value["end"] } } } diff --git a/variables.tf b/variables.tf index 3c23f2d..c7289a1 100644 --- a/variables.tf +++ b/variables.tf @@ -22,7 +22,7 @@ variable "agent_net_name" { variable "k8s_version" { description = "What version of k8s to request from provider" - default = "1.11.4" + default = null } variable "cluster_name" { @@ -271,7 +271,7 @@ variable "azure_policy_enable" { variable "automatic_channel_upgrade" { type = string - default = null + default = "stable" } variable "not_allowed" { @@ -279,6 +279,8 @@ variable "not_allowed" { start = string end = string })) + + default = {} } variable "node_os_channel_upgrade" { @@ -289,26 +291,26 @@ variable "node_os_channel_upgrade" { variable "max_surge" { type = string - default = null + default = "10" description = "The maximum percentage of nodes which will be added to the Node Pool size during an upgrade" } variable "frequency" { description = "Frequency of maintenance." type = string - default = null + default = "Weekly" } variable "interval" { description = "The interval for maintenance runs." type = number - default = null + default = 1 } variable "duration" { description = "The duration of the window for maintenance to run in hours." type = string - default = null + default = "5" } variable "day_of_week" { @@ -338,7 +340,7 @@ variable "start_time" { variable "utc_offset" { description = "Used to determine the timezone for cluster maintenance." type = string - default = null + default = "+01:00" } variable "start_date" { From 0545c4fb116f47184c73cc5797632cf755e8f3d1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Mar 2024 12:55:52 +0000 Subject: [PATCH 23/38] terraform-docs: automated action --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index eade542..ef39a5e 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ No modules. | [aks\_vnet\_subnet\_cidr](#input\_aks\_vnet\_subnet\_cidr) | n/a | `string` | `"10.200.0.0/24"` | no | | [aks\_vnet\_subnet\_id](#input\_aks\_vnet\_subnet\_id) | n/a | `string` | `""` | no | | [api\_server\_authorized\_ip\_ranges](#input\_api\_server\_authorized\_ip\_ranges) | List of IPs to whitelist for incoming to Kubernetes API | `list(string)` | `[]` | no | -| [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | n/a | `string` | `null` | no | +| [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | n/a | `string` | `"stable"` | no | | [azure\_policy\_enable](#input\_azure\_policy\_enable) | Turn on Azure Policy in cluster or not | `bool` | `false` | no | | [client\_id](#input\_client\_id) | azure client id | `any` | `null` | no | | [client\_secret](#input\_client\_secret) | azure client secret | `any` | `null` | no | @@ -54,31 +54,31 @@ No modules. | [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | `null` | no | | [default\_pool](#input\_default\_pool) | n/a | `map` | `{}` | no | | [dns\_prefix](#input\_dns\_prefix) | n/a | `any` | n/a | yes | -| [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | `null` | no | +| [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | `"5"` | no | | [enable\_diagnostics](#input\_enable\_diagnostics) | n/a | `bool` | `false` | no | -| [frequency](#input\_frequency) | Frequency of maintenance. | `string` | `null` | no | +| [frequency](#input\_frequency) | Frequency of maintenance. | `string` | `"Weekly"` | no | | [identity\_ids](#input\_identity\_ids) | n/a | `list(string)` | `[]` | no | | [identity\_type](#input\_identity\_type) | n/a | `string` | `"SystemAssigned"` | no | | [ingress\_application\_gateway\_enable](#input\_ingress\_application\_gateway\_enable) | Ingress Application Gateway | `bool` | `false` | no | | [ingress\_application\_gateway\_name](#input\_ingress\_application\_gateway\_name) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_cidr](#input\_ingress\_application\_subnet\_cidr) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_id](#input\_ingress\_application\_subnet\_id) | n/a | `string` | `null` | no | -| [interval](#input\_interval) | The interval for maintenance runs. | `number` | `null` | no | -| [k8s\_version](#input\_k8s\_version) | What version of k8s to request from provider | `string` | `"1.11.4"` | no | +| [interval](#input\_interval) | The interval for maintenance runs. | `number` | `1` | no | +| [k8s\_version](#input\_k8s\_version) | What version of k8s to request from provider | `any` | `null` | no | | [key\_vault\_secrets\_provider](#input\_key\_vault\_secrets\_provider) | n/a | 
map(object({
    secret_rotation_enabled  = string
    secret_rotation_interval = string
  }))
| `null` | no | | [kubelet\_identity](#input\_kubelet\_identity) | Identity / RBAC goes here | 
object({
    client_id                 = string
    object_id                 = string
    user_assigned_identity_id = string
  })
| `null` | no | | [load\_balancer\_sku](#input\_load\_balancer\_sku) | Networking settings. | `string` | `"standard"` | no | | [log\_analytics](#input\_log\_analytics) | Diagnostics | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | -| [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `null` | no | +| [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `"10"` | no | | [metrics](#input\_metrics) | n/a | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | | [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `false` | no | | [msd\_workspace\_id](#input\_msd\_workspace\_id) | Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to | `string` | `""` | no | | [node\_os\_channel\_upgrade](#input\_node\_os\_channel\_upgrade) | automatically upgrades the node image to the latest version available. | `string` | `"NodeImage"` | no | | [node\_pools](#input\_node\_pools) | Node pools to use | `list` | `[]` | no | | [node\_resource\_group](#input\_node\_resource\_group) | n/a | `any` | `null` | no | -| [not\_allowed](#input\_not\_allowed) | n/a | 
map(object({
    start = string
    end   = string
  }))
| n/a | yes | +| [not\_allowed](#input\_not\_allowed) | n/a | 
map(object({
    start = string
    end   = string
  }))
| `{}` | no | | [oidc\_issuer\_enabled](#input\_oidc\_issuer\_enabled) | n/a | `bool` | `false` | no | | [oms\_agent\_enable](#input\_oms\_agent\_enable) | Enable OMS Agent profile | `bool` | `true` | no | | [oms\_workspace\_id](#input\_oms\_workspace\_id) | Operations Management Suite Workspace ID | `string` | `""` | no | @@ -99,7 +99,7 @@ No modules. | [start\_date](#input\_start\_date) | The date on which the maintenance window begins to take effect. | `string` | `null` | no | | [start\_time](#input\_start\_time) | The time for maintenance to begin, based on the timezone determined by utc\_offset. | `string` | `null` | no | | [tags](#input\_tags) | # Metadata ## | `map` | `{}` | no | -| [utc\_offset](#input\_utc\_offset) | Used to determine the timezone for cluster maintenance. | `string` | `null` | no | +| [utc\_offset](#input\_utc\_offset) | Used to determine the timezone for cluster maintenance. | `string` | `"+01:00"` | no | | [week\_index](#input\_week\_index) | Specifies on which instance of the allowed days specified in day\_of\_week the maintenance occurs. | `string` | `null` | no | | [workload\_identity\_enabled](#input\_workload\_identity\_enabled) | n/a | `bool` | `false` | no | From 7155fa0973d7c309e7f9f8491e5341a9f374c4a1 Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Thu, 7 Mar 2024 17:39:25 +0100 Subject: [PATCH 24/38] Add Tuesday as Default --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index c7289a1..0058f49 100644 --- a/variables.tf +++ b/variables.tf @@ -316,7 +316,7 @@ variable "duration" { variable "day_of_week" { description = "The day of the week for the maintenance run." type = string - default = null + default = "Tuesday" } variable "day_of_month" { From afda379ce255ff055eab8c154dbc5f1c012edc61 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Mar 2024 16:39:56 +0000 Subject: [PATCH 25/38] terraform-docs: automated action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ef39a5e..302de5d 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ No modules. | [cluster\_name](#input\_cluster\_name) | What the k8s cluster should be identified as | `any` | n/a | yes | | [create\_vnet](#input\_create\_vnet) | n/a | `bool` | `true` | no | | [day\_of\_month](#input\_day\_of\_month) | The day of the month for the maintenance run. | `number` | `null` | no | -| [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | `null` | no | +| [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | `"Tuesday"` | no | | [default\_pool](#input\_default\_pool) | n/a | `map` | `{}` | no | | [dns\_prefix](#input\_dns\_prefix) | n/a | `any` | n/a | yes | | [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | `"5"` | no | From 749c954bce979b49992103a0d122474c37bcc886 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Fri, 8 Mar 2024 08:26:52 +0100 Subject: [PATCH 26/38] restructure maintenance_window_auto_upgrade and maintenance_window_node_os --- aks.tf | 83 ++++++++++++++++-------------- variables.tf | 140 +++++++++++++++++++++++++++++---------------------- 2 files changed, 127 insertions(+), 96 deletions(-) diff --git a/aks.tf b/aks.tf index 66bb59d..7c2d1d4 100644 --- a/aks.tf +++ b/aks.tf @@ -129,46 +129,50 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { } automatic_channel_upgrade = var.automatic_channel_upgrade - maintenance_window_auto_upgrade { - frequency = var.frequency - interval = var.interval - duration = var.duration - day_of_week = var.day_of_week - day_of_month = var.day_of_month - week_index = var.week_index - start_time = var.start_time - utc_offset = var.utc_offset - start_date = var.start_date - - dynamic "not_allowed" { - for_each = var.not_allowed - content { - start = not_allowed.value["start"] - end = not_allowed.value["end"] +dynamic "maintenance_window_auto_upgrade" { + for_each = var.maintenance_window_auto_upgrade == null ? [] : [var.maintenance_window_auto_upgrade] + content { + duration = maintenance_window_auto_upgrade.value.duration + frequency = maintenance_window_auto_upgrade.value.frequency + interval = maintenance_window_auto_upgrade.value.interval + day_of_month = maintenance_window_auto_upgrade.value.day_of_month + day_of_week = maintenance_window_auto_upgrade.value.day_of_week + start_date = maintenance_window_auto_upgrade.value.start_date + start_time = maintenance_window_auto_upgrade.value.start_time + utc_offset = maintenance_window_auto_upgrade.value.utc_offset + week_index = maintenance_window_auto_upgrade.value.week_index + + dynamic "not_allowed" { + for_each = maintenance_window_auto_upgrade.value.not_allowed == null ? [] : maintenance_window_auto_upgrade.value.not_allowed + content { + end = not_allowed.value.end + start = not_allowed.value.start + } } } } - node_os_channel_upgrade = var.node_os_channel_upgrade - maintenance_window_node_os { - frequency = var.frequency - interval = var.interval - duration = var.duration - day_of_week = var.day_of_week - day_of_month = var.day_of_month - week_index = var.week_index - start_time = var.start_time - utc_offset = var.utc_offset - start_date = var.start_date - - - dynamic "not_allowed" { - for_each = var.not_allowed - content { - start = not_allowed.value["start"] - end = not_allowed.value["end"] + dynamic "maintenance_window_node_os" { + for_each = var.maintenance_window_node_os == null ? [] : [var.maintenance_window_node_os] + content { + duration = maintenance_window_node_os.value.duration + frequency = maintenance_window_node_os.value.frequency + interval = maintenance_window_node_os.value.interval + day_of_month = maintenance_window_node_os.value.day_of_month + day_of_week = maintenance_window_node_os.value.day_of_week + start_date = maintenance_window_node_os.value.start_date + start_time = maintenance_window_node_os.value.start_time + utc_offset = maintenance_window_node_os.value.utc_offset + week_index = maintenance_window_node_os.value.week_index + + dynamic "not_allowed" { + for_each = maintenance_window_node_os.value.not_allowed == null ? [] : maintenance_window_node_os.value.not_allowed + content { + end = not_allowed.value.end + start = not_allowed.value.start + } } } } @@ -204,9 +208,14 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { tags = lookup(var.default_pool, "tags", var.tags) max_pods = lookup(var.default_pool, "max_pods", local.default_pool_settings.max_pods) orchestrator_version = lookup(var.default_pool, "k8s_version", local.default_pool_settings.k8s_version) - upgrade_settings { - max_surge = var.max_surge - } + + dynamic "upgrade_settings" { + for_each = var.max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.max_surge + } + } } dynamic "service_principal" { diff --git a/variables.tf b/variables.tf index c7289a1..3d7ffd7 100644 --- a/variables.tf +++ b/variables.tf @@ -274,15 +274,6 @@ variable "automatic_channel_upgrade" { default = "stable" } -variable "not_allowed" { - type = map(object({ - start = string - end = string - })) - - default = {} -} - variable "node_os_channel_upgrade" { type = string default = "NodeImage" @@ -290,63 +281,79 @@ variable "node_os_channel_upgrade" { } variable "max_surge" { - type = string - default = "10" - description = "The maximum percentage of nodes which will be added to the Node Pool size during an upgrade" -} - -variable "frequency" { - description = "Frequency of maintenance." - type = string - default = "Weekly" -} - -variable "interval" { - description = "The interval for maintenance runs." - type = number - default = 1 -} - -variable "duration" { - description = "The duration of the window for maintenance to run in hours." - type = string - default = "5" -} - -variable "day_of_week" { - description = "The day of the week for the maintenance run." type = string default = null + description = "The maximum number or percentage of nodes which will be added to the Default Node Pool size during an upgrade." } -variable "day_of_month" { - description = "The day of the month for the maintenance run." - type = number - default = null -} - -variable "week_index" { - description = "Specifies on which instance of the allowed days specified in day_of_week the maintenance occurs." - type = string +variable "maintenance_window_auto_upgrade" { + type = object({ + day_of_month = optional(number) + day_of_week = optional(string) + duration = number + frequency = string + interval = number + start_date = optional(string) + start_time = optional(string) + utc_offset = optional(string) + week_index = optional(string) + not_allowed = optional(set(object({ + end = string + start = string + }))) + }) default = null -} - -variable "start_time" { - description = "The time for maintenance to begin, based on the timezone determined by utc_offset." - type = string + description = <<-EOT + - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive). + - `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency. + - `duration` - (Required) The duration of the window for maintenance to run in hours. + - `frequency` - (Required) Frequency of maintenance. Possible options are `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`. + - `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based. + - `start_date` - (Optional) The date on which the maintenance window begins to take effect. + - `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`. + - `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance. + - `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`. + + --- + `not_allowed` block supports the following: + - `end` - (Required) The end of a time span, formatted as an RFC3339 string. + - `start` - (Required) The start of a time span, formatted as an RFC3339 string. +EOT +} + +variable "maintenance_window_node_os" { + type = object({ + day_of_month = optional(number) + day_of_week = optional(string) + duration = number + frequency = string + interval = number + start_date = optional(string) + start_time = optional(string) + utc_offset = optional(string) + week_index = optional(string) + not_allowed = optional(set(object({ + end = string + start = string + }))) + }) default = null -} + description = <<-EOT + - `day_of_month` - + - `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency. + - `duration` - (Required) The duration of the window for maintenance to run in hours. + - `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`. + - `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based. + - `start_date` - (Optional) The date on which the maintenance window begins to take effect. + - `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`. + - `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance. + - `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`. -variable "utc_offset" { - description = "Used to determine the timezone for cluster maintenance." - type = string - default = "+01:00" -} - -variable "start_date" { - description = "The date on which the maintenance window begins to take effect." - type = string - default = null + --- + `not_allowed` block supports the following: + - `end` - (Required) The end of a time span, formatted as an RFC3339 string. + - `start` - (Required) The start of a time span, formatted as an RFC3339 string. +EOT } @@ -360,6 +367,21 @@ variable "ingress_application_gateway_name" { type = string default = null } +variable "ingress_application_gateway_subnet_id" { + type = string + default = null +} + +variable "ingress_application_gateway_subnet_cidr" { + type = string + default = null +} + +variable "ingress_application_gateway_id" { + type = string + default = null +} + variable "ingress_application_subnet_id" { type = string From 961fcf7574579e22ae62463758a6cfad407fe3e6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 8 Mar 2024 07:27:46 +0000 Subject: [PATCH 27/38] terraform-docs: automated action --- README.md | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ef39a5e..a999d6b 100644 --- a/README.md +++ b/README.md @@ -50,35 +50,34 @@ No modules. | [client\_secret](#input\_client\_secret) | azure client secret | `any` | `null` | no | | [cluster\_name](#input\_cluster\_name) | What the k8s cluster should be identified as | `any` | n/a | yes | | [create\_vnet](#input\_create\_vnet) | n/a | `bool` | `true` | no | -| [day\_of\_month](#input\_day\_of\_month) | The day of the month for the maintenance run. | `number` | `null` | no | -| [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | `null` | no | | [default\_pool](#input\_default\_pool) | n/a | `map` | `{}` | no | | [dns\_prefix](#input\_dns\_prefix) | n/a | `any` | n/a | yes | -| [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | `"5"` | no | | [enable\_diagnostics](#input\_enable\_diagnostics) | n/a | `bool` | `false` | no | -| [frequency](#input\_frequency) | Frequency of maintenance. | `string` | `"Weekly"` | no | | [identity\_ids](#input\_identity\_ids) | n/a | `list(string)` | `[]` | no | | [identity\_type](#input\_identity\_type) | n/a | `string` | `"SystemAssigned"` | no | | [ingress\_application\_gateway\_enable](#input\_ingress\_application\_gateway\_enable) | Ingress Application Gateway | `bool` | `false` | no | +| [ingress\_application\_gateway\_id](#input\_ingress\_application\_gateway\_id) | n/a | `string` | `null` | no | | [ingress\_application\_gateway\_name](#input\_ingress\_application\_gateway\_name) | n/a | `string` | `null` | no | +| [ingress\_application\_gateway\_subnet\_cidr](#input\_ingress\_application\_gateway\_subnet\_cidr) | n/a | `string` | `null` | no | +| [ingress\_application\_gateway\_subnet\_id](#input\_ingress\_application\_gateway\_subnet\_id) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_cidr](#input\_ingress\_application\_subnet\_cidr) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_id](#input\_ingress\_application\_subnet\_id) | n/a | `string` | `null` | no | -| [interval](#input\_interval) | The interval for maintenance runs. | `number` | `1` | no | | [k8s\_version](#input\_k8s\_version) | What version of k8s to request from provider | `any` | `null` | no | | [key\_vault\_secrets\_provider](#input\_key\_vault\_secrets\_provider) | n/a | 
map(object({
    secret_rotation_enabled  = string
    secret_rotation_interval = string
  }))
| `null` | no | | [kubelet\_identity](#input\_kubelet\_identity) | Identity / RBAC goes here | 
object({
    client_id                 = string
    object_id                 = string
    user_assigned_identity_id = string
  })
| `null` | no | | [load\_balancer\_sku](#input\_load\_balancer\_sku) | Networking settings. | `string` | `"standard"` | no | | [log\_analytics](#input\_log\_analytics) | Diagnostics | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | +| [maintenance\_window\_auto\_upgrade](#input\_maintenance\_window\_auto\_upgrade) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | +| [maintenance\_window\_node\_os](#input\_maintenance\_window\_node\_os) | - `day_of_month` -
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | -| [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `"10"` | no | +| [max\_surge](#input\_max\_surge) | The maximum number or percentage of nodes which will be added to the Default Node Pool size during an upgrade. | `string` | `null` | no | | [metrics](#input\_metrics) | n/a | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | | [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `false` | no | | [msd\_workspace\_id](#input\_msd\_workspace\_id) | Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to | `string` | `""` | no | | [node\_os\_channel\_upgrade](#input\_node\_os\_channel\_upgrade) | automatically upgrades the node image to the latest version available. | `string` | `"NodeImage"` | no | | [node\_pools](#input\_node\_pools) | Node pools to use | `list` | `[]` | no | | [node\_resource\_group](#input\_node\_resource\_group) | n/a | `any` | `null` | no | -| [not\_allowed](#input\_not\_allowed) | n/a | 
map(object({
    start = string
    end   = string
  }))
| `{}` | no | | [oidc\_issuer\_enabled](#input\_oidc\_issuer\_enabled) | n/a | `bool` | `false` | no | | [oms\_agent\_enable](#input\_oms\_agent\_enable) | Enable OMS Agent profile | `bool` | `true` | no | | [oms\_workspace\_id](#input\_oms\_workspace\_id) | Operations Management Suite Workspace ID | `string` | `""` | no | @@ -96,11 +95,7 @@ No modules. | [resource\_group\_location](#input\_resource\_group\_location) | Location of the RG the environment will run inside | `string` | `"West Europe"` | no | | [resource\_group\_name](#input\_resource\_group\_name) | Name of RG the environment will run inside | `any` | n/a | yes | | [ssh\_public\_key](#input\_ssh\_public\_key) | public key to add to admin\_user in VMs | `any` | n/a | yes | -| [start\_date](#input\_start\_date) | The date on which the maintenance window begins to take effect. | `string` | `null` | no | -| [start\_time](#input\_start\_time) | The time for maintenance to begin, based on the timezone determined by utc\_offset. | `string` | `null` | no | | [tags](#input\_tags) | # Metadata ## | `map` | `{}` | no | -| [utc\_offset](#input\_utc\_offset) | Used to determine the timezone for cluster maintenance. | `string` | `"+01:00"` | no | -| [week\_index](#input\_week\_index) | Specifies on which instance of the allowed days specified in day\_of\_week the maintenance occurs. | `string` | `null` | no | | [workload\_identity\_enabled](#input\_workload\_identity\_enabled) | n/a | `bool` | `false` | no | ## Outputs From d7cdf8ad6bb10931089f730095125824765dbbee Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Fri, 8 Mar 2024 08:36:09 +0100 Subject: [PATCH 28/38] Add missing description --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 3d7ffd7..bb84bf0 100644 --- a/variables.tf +++ b/variables.tf @@ -339,7 +339,7 @@ variable "maintenance_window_node_os" { }) default = null description = <<-EOT - - `day_of_month` - + - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive). - `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency. - `duration` - (Required) The duration of the window for maintenance to run in hours. - `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`. From 89ce2fd30cb1527ffdf1423f5cbdd0282af86dcb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 8 Mar 2024 07:36:32 +0000 Subject: [PATCH 29/38] terraform-docs: automated action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a999d6b..256d192 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ No modules. | [load\_balancer\_sku](#input\_load\_balancer\_sku) | Networking settings. | `string` | `"standard"` | no | | [log\_analytics](#input\_log\_analytics) | Diagnostics | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | | [maintenance\_window\_auto\_upgrade](#input\_maintenance\_window\_auto\_upgrade) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | -| [maintenance\_window\_node\_os](#input\_maintenance\_window\_node\_os) | - `day_of_month` -
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | +| [maintenance\_window\_node\_os](#input\_maintenance\_window\_node\_os) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | | [max\_surge](#input\_max\_surge) | The maximum number or percentage of nodes which will be added to the Default Node Pool size during an upgrade. | `string` | `null` | no | From 6db71b0748e8b97b631bd6f27ed15bc4f78505d3 Mon Sep 17 00:00:00 2001 From: Carsten Thiel Date: Fri, 8 Mar 2024 10:13:44 +0100 Subject: [PATCH 30/38] Chnaging formats --- aks.tf | 14 +++++++------- variables.tf | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/aks.tf b/aks.tf index 7c2d1d4..089539b 100644 --- a/aks.tf +++ b/aks.tf @@ -129,7 +129,7 @@ resource "azurerm_kubernetes_cluster" "k8s_cluster" { } automatic_channel_upgrade = var.automatic_channel_upgrade -dynamic "maintenance_window_auto_upgrade" { + dynamic "maintenance_window_auto_upgrade" { for_each = var.maintenance_window_auto_upgrade == null ? [] : [var.maintenance_window_auto_upgrade] content { duration = maintenance_window_auto_upgrade.value.duration @@ -143,7 +143,7 @@ dynamic "maintenance_window_auto_upgrade" { week_index = maintenance_window_auto_upgrade.value.week_index dynamic "not_allowed" { - for_each = maintenance_window_auto_upgrade.value.not_allowed == null ? [] : maintenance_window_auto_upgrade.value.not_allowed + for_each = maintenance_window_auto_upgrade.value.not_allowed == null ? {} : maintenance_window_auto_upgrade.value.not_allowed content { end = not_allowed.value.end start = not_allowed.value.start @@ -168,7 +168,7 @@ dynamic "maintenance_window_auto_upgrade" { week_index = maintenance_window_node_os.value.week_index dynamic "not_allowed" { - for_each = maintenance_window_node_os.value.not_allowed == null ? [] : maintenance_window_node_os.value.not_allowed + for_each = maintenance_window_node_os.value.not_allowed == null ? {} : maintenance_window_node_os.value.not_allowed content { end = not_allowed.value.end start = not_allowed.value.start @@ -210,12 +210,12 @@ dynamic "maintenance_window_auto_upgrade" { orchestrator_version = lookup(var.default_pool, "k8s_version", local.default_pool_settings.k8s_version) dynamic "upgrade_settings" { - for_each = var.max_surge == null ? [] : ["upgrade_settings"] + for_each = var.max_surge == null ? [] : ["upgrade_settings"] - content { - max_surge = var.max_surge - } + content { + max_surge = var.max_surge } + } } dynamic "service_principal" { diff --git a/variables.tf b/variables.tf index bb84bf0..8289361 100644 --- a/variables.tf +++ b/variables.tf @@ -297,7 +297,7 @@ variable "maintenance_window_auto_upgrade" { start_time = optional(string) utc_offset = optional(string) week_index = optional(string) - not_allowed = optional(set(object({ + not_allowed = optional(map(object({ end = string start = string }))) @@ -332,7 +332,7 @@ variable "maintenance_window_node_os" { start_time = optional(string) utc_offset = optional(string) week_index = optional(string) - not_allowed = optional(set(object({ + not_allowed = optional(map(object({ end = string start = string }))) From 34891d06dc283c36b0140a364960b2e04ed81992 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 8 Mar 2024 09:19:20 +0000 Subject: [PATCH 31/38] terraform-docs: automated action --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e81afb8..5b9fa24 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,12 @@ No modules. | [client\_secret](#input\_client\_secret) | azure client secret | `any` | `null` | no | | [cluster\_name](#input\_cluster\_name) | What the k8s cluster should be identified as | `any` | n/a | yes | | [create\_vnet](#input\_create\_vnet) | n/a | `bool` | `true` | no | +| [day\_of\_week](#input\_day\_of\_week) | The day of the week for the maintenance run. | `string` | `"Tuesday"` | no | | [default\_pool](#input\_default\_pool) | n/a | `map` | `{}` | no | | [dns\_prefix](#input\_dns\_prefix) | n/a | `any` | n/a | yes | +| [duration](#input\_duration) | The duration of the window for maintenance to run in hours. | `string` | `"5"` | no | | [enable\_diagnostics](#input\_enable\_diagnostics) | n/a | `bool` | `false` | no | +| [frequency](#input\_frequency) | Frequency of maintenance. | `string` | `"Weekly"` | no | | [identity\_ids](#input\_identity\_ids) | n/a | `list(string)` | `[]` | no | | [identity\_type](#input\_identity\_type) | n/a | `string` | `"SystemAssigned"` | no | | [ingress\_application\_gateway\_enable](#input\_ingress\_application\_gateway\_enable) | Ingress Application Gateway | `bool` | `false` | no | @@ -63,16 +66,17 @@ No modules. | [ingress\_application\_gateway\_subnet\_id](#input\_ingress\_application\_gateway\_subnet\_id) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_cidr](#input\_ingress\_application\_subnet\_cidr) | n/a | `string` | `null` | no | | [ingress\_application\_subnet\_id](#input\_ingress\_application\_subnet\_id) | n/a | `string` | `null` | no | +| [interval](#input\_interval) | The interval for maintenance runs. | `number` | `1` | no | | [k8s\_version](#input\_k8s\_version) | What version of k8s to request from provider | `any` | `null` | no | | [key\_vault\_secrets\_provider](#input\_key\_vault\_secrets\_provider) | n/a | 
map(object({
    secret_rotation_enabled  = string
    secret_rotation_interval = string
  }))
| `null` | no | | [kubelet\_identity](#input\_kubelet\_identity) | Identity / RBAC goes here | 
object({
    client_id                 = string
    object_id                 = string
    user_assigned_identity_id = string
  })
| `null` | no | | [load\_balancer\_sku](#input\_load\_balancer\_sku) | Networking settings. | `string` | `"standard"` | no | | [log\_analytics](#input\_log\_analytics) | Diagnostics | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | -| [maintenance\_window\_auto\_upgrade](#input\_maintenance\_window\_auto\_upgrade) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | -| [maintenance\_window\_node\_os](#input\_maintenance\_window\_node\_os) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | +| [maintenance\_window\_auto\_upgrade](#input\_maintenance\_window\_auto\_upgrade) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(map(object({
      end   = string
      start = string
    })))
  })
| `null` | no | +| [maintenance\_window\_node\_os](#input\_maintenance\_window\_node\_os) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(map(object({
      end   = string
      start = string
    })))
  })
| `null` | no | | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | -| [max\_surge](#input\_max\_surge) | The maximum number or percentage of nodes which will be added to the Default Node Pool size during an upgrade. | `string` | `null` | no | +| [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `"10"` | no | | [metrics](#input\_metrics) | n/a | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | | [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `false` | no | | [msd\_workspace\_id](#input\_msd\_workspace\_id) | Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to | `string` | `""` | no | From 6f1733600bbf99053c88942a9be1487838544b05 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Wed, 13 Mar 2024 09:29:35 +0100 Subject: [PATCH 32/38] Set the automatic_channel_upgrade to disabeld by default --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 24241e0..2e0e51c 100644 --- a/variables.tf +++ b/variables.tf @@ -271,7 +271,7 @@ variable "azure_policy_enable" { variable "automatic_channel_upgrade" { type = string - default = "stable" + default = null } variable "node_os_channel_upgrade" { From d93af579cee94d316b1ac35fce9c1454b88920b4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 Mar 2024 08:30:25 +0000 Subject: [PATCH 33/38] terraform-docs: automated action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b9fa24..f04d03d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ No modules. | [aks\_vnet\_subnet\_cidr](#input\_aks\_vnet\_subnet\_cidr) | n/a | `string` | `"10.200.0.0/24"` | no | | [aks\_vnet\_subnet\_id](#input\_aks\_vnet\_subnet\_id) | n/a | `string` | `""` | no | | [api\_server\_authorized\_ip\_ranges](#input\_api\_server\_authorized\_ip\_ranges) | List of IPs to whitelist for incoming to Kubernetes API | `list(string)` | `[]` | no | -| [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | n/a | `string` | `"stable"` | no | +| [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | n/a | `string` | `null` | no | | [azure\_policy\_enable](#input\_azure\_policy\_enable) | Turn on Azure Policy in cluster or not | `bool` | `false` | no | | [client\_id](#input\_client\_id) | azure client id | `any` | `null` | no | | [client\_secret](#input\_client\_secret) | azure client secret | `any` | `null` | no | From 032e7343bb9d328f45ef0035822585416b2dac53 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Wed, 13 Mar 2024 12:30:56 +0100 Subject: [PATCH 34/38] Default max_surge to 33% --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 2e0e51c..8aa3afe 100644 --- a/variables.tf +++ b/variables.tf @@ -282,7 +282,7 @@ variable "node_os_channel_upgrade" { variable "max_surge" { type = string - default = "10" + default = "33%" description = "The maximum percentage of nodes which will be added to the Node Pool size during an upgrade" } From e6df7bad3e106f6b0b99a33e9bf0ed6e4e638135 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 Mar 2024 11:31:36 +0000 Subject: [PATCH 35/38] terraform-docs: automated action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f04d03d..685c41f 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ No modules. | [maintenance\_window\_node\_os](#input\_maintenance\_window\_node\_os) | - `day_of_month` - (Optional) The day of the month for the maintenance run. Required in combination with RelativeMonthly frequency. Value between 0 and 31 (inclusive).
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(map(object({
      end   = string
      start = string
    })))
  })
| `null` | no | | [managed\_outbound\_ip\_count](#input\_managed\_outbound\_ip\_count) | n/a | `number` | `1` | no | | [max\_pods](#input\_max\_pods) | Max pods to support in this cluster pr node | `number` | `30` | no | -| [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `"10"` | no | +| [max\_surge](#input\_max\_surge) | The maximum percentage of nodes which will be added to the Node Pool size during an upgrade | `string` | `"33%"` | no | | [metrics](#input\_metrics) | n/a | 
map(object({
    enabled = bool
    retention = object({
      enabled = bool
      days    = number
    })
  }))
| `{}` | no | | [msd\_enable](#input\_msd\_enable) | Enable audit logs collected by Microsoft Defender | `bool` | `false` | no | | [msd\_workspace\_id](#input\_msd\_workspace\_id) | Specifies the ID of the Log Analytics Workspace where the audit logs collected by Microsoft Defender should be sent to | `string` | `""` | no | From fcec4cba19a029b6f42107a65336ef26b3eda3df Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Wed, 5 Jun 2024 12:07:48 +0200 Subject: [PATCH 36/38] Fix the upgrade_settings for cluster_node_pools and remove retention_policy in azurerm_monitor_diagnostic_setting --- aks.tf | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/aks.tf b/aks.tf index 089539b..bdd4805 100644 --- a/aks.tf +++ b/aks.tf @@ -370,6 +370,14 @@ resource "azurerm_kubernetes_cluster_node_pool" "aks-node" { eviction_policy = each.value.eviction_policy spot_max_price = each.value.spot_max_price + dynamic "upgrade_settings" { + for_each = var.max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.max_surge + } + } + lifecycle { ignore_changes = [ # Ignore changes to default_node_pools node_count , e.g. because it is managed by enable_auto_scaling @@ -390,11 +398,6 @@ resource "azurerm_monitor_diagnostic_setting" "aks-diagnostics" { content { category = log.key enabled = log.value.enabled - - retention_policy { - enabled = log.value.retention.enabled - days = log.value.retention.days - } } } dynamic "metric" { @@ -404,11 +407,6 @@ resource "azurerm_monitor_diagnostic_setting" "aks-diagnostics" { content { category = metric.key enabled = metric.value.enabled - - retention_policy { - enabled = metric.value.retention.enabled - days = metric.value.retention.days - } } } } From 70ad9fd96b8c7cd752941e8d7a2a3d6c67a62c87 Mon Sep 17 00:00:00 2001 From: Knut Ivar Alvestad Date: Wed, 5 Jun 2024 13:17:00 +0200 Subject: [PATCH 37/38] Add test for spot on upgrade_settings for nodepools --- aks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aks.tf b/aks.tf index bdd4805..c6d7e85 100644 --- a/aks.tf +++ b/aks.tf @@ -371,7 +371,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "aks-node" { spot_max_price = each.value.spot_max_price dynamic "upgrade_settings" { - for_each = var.max_surge == null ? [] : ["upgrade_settings"] + for_each = var.max_surge == null || each.value.priority == "Spot" ? [] : ["upgrade_settings"] content { max_surge = var.max_surge From 5bc3d32f747533a1542022942eaed34cdbc710fe Mon Sep 17 00:00:00 2001 From: Marcel Grygar Date: Thu, 4 Jul 2024 14:19:17 +0200 Subject: [PATCH 38/38] Add new param --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 8aa3afe..0be7b4a 100644 --- a/variables.tf +++ b/variables.tf @@ -276,7 +276,7 @@ variable "automatic_channel_upgrade" { variable "node_os_channel_upgrade" { type = string - default = "NodeImage" + default = "None" description = "automatically upgrades the node image to the latest version available." }