Merge pull request #136 from pluralsh/marcin/prod-4407-plural-up-aks-upgrades-dont-upgrade-node-pools

fix: Fix Azure node upgrades

Author: maciaszczykm

setup/stacks/mgmt.yaml

@@ -36,6 +36,8 @@ spec:
     labels:
       azure.workload.identity/use: "true"
     serviceAccount: "stacks"
+    nodeSelector:
+      platform.plural.sh/stack-runner: "true"
   [[ end ]]
   git:
     ref: main

terraform/clouds/azure/aks.tf

@@ -1,32 +1,38 @@
 locals {
   node_pool_add = {
     (local.active_node_group) = {
-      orchestrator_version = var.kubernetes_version,
-      node_taints = local.upgrading ? ["platform.plural.sh/draining=true:NoSchedule"] : [],
+      orchestrator_version = local.node_orchestrator_version,
+      node_taints          = local.upgrading ? ["platform.plural.sh/draining=true:NoSchedule"] : [],
+      node_labels = local.upgrading ? {} : {
+        "platform.plural.sh/stack-runner" = "true"
+      },
     },
-    (local.drain_node_group) =  {
-      orchestrator_version = var.next_kubernetes_version,
+    (local.drain_node_group) = {
+      orchestrator_version = local.next_kubernetes_version,
+      node_labels = local.upgrading ? {
+        "platform.plural.sh/stack-runner" = "true"
+      } : {},
     }
   }
 
-  full_node_pools = {for k, v in var.node_pools: k => merge(v, try(lookup(local.node_pool_add, k), {})) if k != local.drain_node_group || local.upgrading == true}
+  full_node_pools = { for k, v in var.node_pools : k => merge(v, try(lookup(local.node_pool_add, k), {})) if k != local.drain_node_group || local.upgrading == true }
 }
 
-
 module "aks" {
   source = "Azure/aks/azurerm"
   version = "9.2.0"
 
-  kubernetes_version   = var.next_kubernetes_version
+  kubernetes_version   = local.next_kubernetes_version
+  orchestrator_version = local.node_orchestrator_version
   cluster_name         = var.cluster_name
   resource_group_name  = local.resource_group.name
   prefix               = var.cluster_name
   os_disk_size_gb      = 60
   sku_tier             = "Standard"
   rbac_aad             = false
   vnet_subnet_id       = azurerm_subnet.network.id
-  node_pools           = {for name, pool in local.full_node_pools : name => merge(pool, {name = name, vnet_subnet_id = azurerm_subnet.network.id})}
-  
+  node_pools           = { for name, pool in local.full_node_pools : name => merge(pool, { name = name, vnet_subnet_id = azurerm_subnet.network.id }) }
+
   ebpf_data_plane     = "cilium"
   network_plugin_mode = "overlay"
   network_plugin      = "azure"
@@ -35,4 +41,4 @@ module "aks" {
 
   workload_identity_enabled = var.workload_identity_enabled
   oidc_issuer_enabled       = var.workload_identity_enabled
-}
+}

terraform/clouds/azure/locals.tf

@@ -7,9 +7,12 @@ locals {
   rg = var.create_resource_group ? azurerm_resource_group.main[0] : data.azurerm_resource_group.main[0]
   db_url = format("postgresql://console:%s@%s:5432/console", random_password.password.result, try(azurerm_postgresql_flexible_server.postgres[0].fqdn, ""))
 
-  upgrading = var.kubernetes_version != var.next_kubernetes_version
+  next_kubernetes_version = var.next_kubernetes_version != "" ? var.next_kubernetes_version : var.kubernetes_version
+  upgrading = var.kubernetes_version != local.next_kubernetes_version
   split_vsn = [ for i in split(".", var.kubernetes_version): tonumber(i) ]
   vsn_even = ((tonumber(local.split_vsn[0]) * 100 + tonumber(local.split_vsn[1])) % 2) == 0
   active_node_group = local.vsn_even ? "blue" : "green"
   drain_node_group = local.vsn_even ? "green" : "blue"
+  # AKS cannot upgrade CP and node pools in one apply when both versions change.
+  node_orchestrator_version = local.upgrading ? var.kubernetes_version : local.next_kubernetes_version
 }

terraform/clouds/azure/variables.tf

@@ -19,8 +19,9 @@ variable "kubernetes_version" {
 }
 
 variable "next_kubernetes_version" {
-  type    = string
-  default = "1.34"
+  type        = string
+  default     = ""
+  description = "AKS control plane target; leave empty to match kubernetes_version."
 }
 
 variable "create_resource_group" {

terraform/modules/clusters/azure/aks.tf

@@ -2,15 +2,22 @@ module "aks" {
   source = "Azure/aks/azurerm"
   version = "9.2.0"
 
-  kubernetes_version   = var.kubernetes_version
+  kubernetes_version   = local.next_kubernetes_version
+  orchestrator_version = local.node_orchestrator_version
   cluster_name         = var.cluster
   resource_group_name  = data.azurerm_resource_group.default.name
   prefix               = var.cluster
   os_disk_size_gb      = 60
   sku_tier             = "Standard"
   rbac_aad             = false
   vnet_subnet_id       = local.network.sn_subnet_id
-  node_pools           = {for name, pool in var.node_pools : name => merge(pool, {name = name, vnet_subnet_id = local.network.sn_subnet_id})}
+  node_pools = {
+    for name, pool in var.node_pools : name => merge(pool, {
+      name                   = name
+      vnet_subnet_id         = local.network.sn_subnet_id
+      orchestrator_version   = local.node_orchestrator_version
+    })
+  }
 
   ebpf_data_plane     = "cilium"
   network_plugin_mode = "overlay"

terraform/modules/clusters/azure/locals.tf

@@ -1,4 +1,9 @@
 locals {
-  identity = jsondecode(data.plural_service_context.identity.configuration)
-  network = jsondecode(data.plural_service_context.network.configuration)
+  identity  = jsondecode(data.plural_service_context.identity.configuration)
+  network   = jsondecode(data.plural_service_context.network.configuration)
+  # Empty next_kubernetes_version means in sync with kubernetes_version (safe before scaffolds passes both).
+  next_kubernetes_version = var.next_kubernetes_version != "" ? var.next_kubernetes_version : var.kubernetes_version
+  upgrading = var.kubernetes_version != local.next_kubernetes_version
+  # AKS upgrades control plane and node pools in separate applies; see clouds/azure/aks.tf.
+  node_orchestrator_version = local.upgrading ? var.kubernetes_version : local.next_kubernetes_version
 }

terraform/modules/clusters/azure/variables.tf

@@ -16,6 +16,12 @@ variable "kubernetes_version" {
   default = "1.34"
 }
 
+variable "next_kubernetes_version" {
+  type        = string
+  default     = ""
+  description = "AKS control plane target; leave empty to match kubernetes_version."
+}
+
 variable "resource_group_name" {
   type = string
   default = "plural"