start managing azure infra with terraform

infra/azure/terraform/root/README.md

@@ -0,0 +1 @@
+This layer holds the foundation resources for the Azure Tenant

infra/azure/terraform/root/data.tf

@@ -0,0 +1,22 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+locals {
+  subscriptions_id = {
+    "prod" = "46678f10-4bbb-447e-98e8-d2829589f2d8"
+    "ci"   = "59cb4516-507c-4c86-bb40-6f3572dcfaeb"
+  }
+}

infra/azure/terraform/root/main.tf

@@ -0,0 +1,89 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+// Azure RBAC
+
+module "role_assignments" {
+  source = "github.com/Azure/terraform-azurerm-avm-res-authorization-roleassignment?ref=v0.3.0"
+  #   version = "0.3.0"
+  groups_by_display_name = {
+    capz-admins = "capz-admins"
+    owners      = "owners"
+  }
+  app_registrations_by_display_name = {
+    datadog   = "Datadog"
+    terraform = "Terraform"
+  }
+  #   user_assigned_managed_identities_by_display_name = {}
+  role_definitions = {
+    owner = {
+      name = "Owner"
+    }
+    contributor = {
+      name = "Contributor"
+    }
+    reader = {
+      name = "Reader"
+    }
+    monitoring-reader = {
+      name = "Monitoring Reader"
+    }
+  }
+
+  entra_id_role_definitions = {
+    application-administrator = {
+      display_name = "Application Administrator"
+    }
+  }
+
+  role_assignments_for_management_groups = {
+    root = {
+      management_group_display_name = "Tenant Root Group"
+      role_assignments = {
+        owner = {
+          role_definition = "owner"
+          any_principals  = ["owners", "terraform"]
+        }
+        monitoring-reader = {
+          role_definition   = "monitoring-reader"
+          app_registrations = ["datadog"]
+        }
+      }
+    }
+  }
+
+  role_assignments_for_subscriptions = {
+    prod = {
+      subscription_id = local.subscriptions_id["prod"]
+      role_assignments = {
+        owner = {
+          role_definition = "owner"
+          groups          = ["capz-admins"]
+        }
+      }
+
+    }
+    ci = {
+      subscription_id = local.subscriptions_id["ci"]
+      role_assignments = {
+        owner = {
+          role_definition = "owner"
+          groups          = ["capz-admins"]
+        }
+      }
+    }
+  }
+
+}

infra/azure/terraform/root/providers.tf

@@ -0,0 +1,62 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+terraform {
+  required_version = "~> 1.11.4"
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = ">= 4.57, < 5.0"
+    }
+    azapi = {
+      source  = "azure/azapi"
+      version = ">= 2.8, < 3"
+    }
+    azuread = {
+      source  = "hashicorp/azuread"
+      version = ">= 3.7, < 4"
+    }
+  }
+
+  backend "azurerm" {
+    resource_group_name  = "k8s-infra-tf-states-rg"
+    storage_account_name = "k8sinfratfstateprow"
+    container_name       = "terraform-state"
+    key                  = "root.terraform.tfstate"
+  }
+}
+
+provider "azurerm" {
+  subscription_id = "46678f10-4bbb-447e-98e8-d2829589f2d8" # Prod Subscription
+  # Configuration options
+  features {}
+}
+
+provider "azurerm" {
+  subscription_id = "59cb4516-507c-4c86-bb40-6f3572dcfaeb" # CI Subscription
+  alias           = "ci"
+  # Configuration options
+  features {}
+}
+
+provider "azapi" {
+  subscription_id = "46678f10-4bbb-447e-98e8-d2829589f2d8"
+}
+
+provider "azuread" {
+  # Configuration options
+}
+

infra/azure/terraform/root/sp.tf

@@ -0,0 +1,101 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+locals {
+  apps = toset([
+    "Terraform",
+    "rg-cleanup",
+    "prow"
+  ])
+  build_cluster_issuers = {
+    aks = {
+      issuer = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f"
+    }
+    eks = {
+      issuer = "https://oidc.eks.us-east-2.amazonaws.com/id/F8B73554FE6FBAF9B19569183FB39762"
+    }
+    gke = {
+      issuer = "https://container.googleapis.com/v1/projects/k8s-infra-prow-build/locations/us-central1/clusters/prow-build"
+    }
+  }
+  graph_api_permissions = {
+    Terraform = {
+      roles = [
+        "62a82d76-70ea-41e2-9197-370581804d09", # Group.ReadWrite.All
+        "1bfefb4e-e0b5-418b-a88f-73c46d2cc8e9", # Application.ReadWrite.All
+        "df021288-bdef-4463-88db-98f22de89214", # User.Read.All
+        "9e3f62cf-ca93-4989-b6ce-bf83c28f9fe8", # RoleManagement.ReadWrite.Directory
+      ]
+    }
+  }
+
+}
+
+resource "azuread_application" "apps" {
+  for_each     = local.apps
+  display_name = each.key
+
+  dynamic "required_resource_access" {
+    for_each = try(local.graph_api_permissions[each.key], null) == null ? [] : [local.graph_api_permissions[each.key]]
+    content {
+      resource_app_id = "00000003-0000-0000-c000-000000000000" # Microsoft Graph
+
+      dynamic "resource_access" {
+        for_each = required_resource_access.value.roles
+        content {
+          id   = resource_access.value
+          type = "Role"
+        }
+      }
+    }
+  }
+}
+
+resource "azuread_service_principal" "service_principals" {
+  for_each  = local.apps
+  client_id = azuread_application.apps[each.key].client_id
+}
+
+resource "azuread_application_federated_identity_credential" "terraform" {
+  for_each = toset([
+    "system:serviceaccount:atlantis:atlantis",
+  ])
+  display_name   = reverse(split(":", each.key))[0]
+  audiences      = ["api://AzureADTokenExchange"]
+  issuer         = "https://container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/utility"
+  application_id = azuread_application.apps["Terraform"].id
+  subject        = each.key
+}
+
+resource "azuread_application_federated_identity_credential" "rg_cleanup" {
+  for_each = toset([
+    "system:serviceaccount:test-pods:rg-cleanup",
+  ])
+  display_name   = reverse(split(":", each.key))[0]
+  audiences      = ["api://AzureADTokenExchange"]
+  issuer         = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f"
+  application_id = azuread_application.apps["rg-cleanup"].id
+  subject        = each.key
+}
+
+resource "azuread_application_federated_identity_credential" "prow" {
+  for_each       = local.build_cluster_issuers
+  display_name   = each.key
+  audiences      = ["api://AzureADTokenExchange"]
+  issuer         = each.value.issuer
+  application_id = azuread_application.apps["prow"].id
+  subject        = "system:serviceaccount:test-pods:azure"
+}

kubernetes/aks-prow-build/prow/kustomization.yaml

@@ -4,3 +4,4 @@ namespace: test-pods
 
 resources:
   - kyverno.yaml
+  - serviceaccounts.yaml

kubernetes/aks-prow-build/prow/serviceaccounts.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: azure
+  annotations:
+    azure.workload.identity/client-id: 333bb18b-207b-4abd-9ed0-e7e3834378b1
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rg-cleanup
+  annotations:
+    azure.workload.identity/client-id: f23f8fcc-855b-40fd-a41b-b329ccdb95a1

kubernetes/gke-utility/atlantis/kustomization.yaml

@@ -36,6 +36,16 @@ patchesStrategicMerge:
                 value: /var/run/secrets/aws-iam-token/serviceaccount/token
               - name: AWS_REGION
                 value: us-east-2
+              - name: ARM_USE_AKS_WORKLOAD_IDENTITY
+                value: "true"
+              - name: ARM_SUBSCRIPTION_ID
+                value: 46678f10-4bbb-447e-98e8-d2829589f2d8
+              - name: AZURE_CLIENT_ID
+                value: 6fe87cee-6470-45d8-accc-57687193e504
+              - name: AZURE_FEDERATED_TOKEN_FILE
+                value: /var/run/secrets/azure-token/serviceaccount/token
+              - name: AZURE_TENANT_ID
+                value: d1aa7522-0959-442e-80ee-8c4f7fb4c184
               - name: ATLANTIS_CONFIG
                 value: /config/atlantis.yaml
               - name: ATLANTIS_GH_TOKEN
@@ -61,6 +71,9 @@ patchesStrategicMerge:
             - mountPath: /var/run/secrets/aws-iam-token/serviceaccount
               name: aws-iam-token
               readOnly: true
+            - name: azure-token
+              mountPath: /var/run/secrets/azure-token/serviceaccount
+              readOnly: true
           volumes:
           - name: config
             configMap:
@@ -73,3 +86,11 @@ patchesStrategicMerge:
                   audience: sts.amazonaws.com
                   expirationSeconds: 86400
                   path: token
+          - name: azure-token
+            projected:
+              defaultMode: 420
+              sources:
+                - serviceAccountToken:
+                    expirationSeconds: 86400
+                    path: token
+                    audience: api://AzureADTokenExchange