diff --git a/infra/azure/terraform/root/README.md b/infra/azure/terraform/root/README.md new file mode 100644 index 00000000000..0a62791e513 --- /dev/null +++ b/infra/azure/terraform/root/README.md @@ -0,0 +1 @@ +This layer holds the foundation resources for the Azure Tenant diff --git a/infra/azure/terraform/root/data.tf b/infra/azure/terraform/root/data.tf new file mode 100644 index 00000000000..0a4d96c34e5 --- /dev/null +++ b/infra/azure/terraform/root/data.tf @@ -0,0 +1,22 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +locals { + subscriptions_id = { + "prod" = "46678f10-4bbb-447e-98e8-d2829589f2d8" + "ci" = "59cb4516-507c-4c86-bb40-6f3572dcfaeb" + } +} diff --git a/infra/azure/terraform/root/groups.tf b/infra/azure/terraform/root/groups.tf new file mode 100644 index 00000000000..e69de29bb2d diff --git a/infra/azure/terraform/root/main.tf b/infra/azure/terraform/root/main.tf new file mode 100644 index 00000000000..06f17c48a19 --- /dev/null +++ b/infra/azure/terraform/root/main.tf @@ -0,0 +1,89 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Azure RBAC + +module "role_assignments" { + source = "github.com/Azure/terraform-azurerm-avm-res-authorization-roleassignment?ref=v0.3.0" + # version = "0.3.0" + groups_by_display_name = { + capz-admins = "capz-admins" + owners = "owners" + } + app_registrations_by_display_name = { + datadog = "Datadog" + terraform = "Terraform" + } + # user_assigned_managed_identities_by_display_name = {} + role_definitions = { + owner = { + name = "Owner" + } + contributor = { + name = "Contributor" + } + reader = { + name = "Reader" + } + monitoring-reader = { + name = "Monitoring Reader" + } + } + + entra_id_role_definitions = { + application-administrator = { + display_name = "Application Administrator" + } + } + + role_assignments_for_management_groups = { + root = { + management_group_display_name = "Tenant Root Group" + role_assignments = { + owner = { + role_definition = "owner" + any_principals = ["owners", "terraform"] + } + monitoring-reader = { + role_definition = "monitoring-reader" + app_registrations = ["datadog"] + } + } + } + } + + role_assignments_for_subscriptions = { + prod = { + subscription_id = local.subscriptions_id["prod"] + role_assignments = { + owner = { + role_definition = "owner" + groups = ["capz-admins"] + } + } + + } + ci = { + subscription_id = local.subscriptions_id["ci"] + role_assignments = { + owner = { + role_definition = "owner" + groups = ["capz-admins"] + } + } + } + } + +} diff --git a/infra/azure/terraform/root/providers.tf b/infra/azure/terraform/root/providers.tf new file mode 100644 index 00000000000..70af57d98c0 --- /dev/null +++ b/infra/azure/terraform/root/providers.tf @@ -0,0 +1,62 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +terraform { + required_version = "~> 1.11.4" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 4.57, < 5.0" + } + azapi = { + source = "azure/azapi" + version = ">= 2.8, < 3" + } + azuread = { + source = "hashicorp/azuread" + version = ">= 3.7, < 4" + } + } + + backend "azurerm" { + resource_group_name = "k8s-infra-tf-states-rg" + storage_account_name = "k8sinfratfstateprow" + container_name = "terraform-state" + key = "root.terraform.tfstate" + } +} + +provider "azurerm" { + subscription_id = "46678f10-4bbb-447e-98e8-d2829589f2d8" # Prod Subscription + # Configuration options + features {} +} + +provider "azurerm" { + subscription_id = "59cb4516-507c-4c86-bb40-6f3572dcfaeb" # CI Subscription + alias = "ci" + # Configuration options + features {} +} + +provider "azapi" { + subscription_id = "46678f10-4bbb-447e-98e8-d2829589f2d8" +} + +provider "azuread" { + # Configuration options +} + diff --git a/infra/azure/terraform/root/sp.tf b/infra/azure/terraform/root/sp.tf new file mode 100644 index 00000000000..788e98f9d24 --- /dev/null +++ b/infra/azure/terraform/root/sp.tf @@ -0,0 +1,101 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +locals { + apps = toset([ + "Terraform", + "rg-cleanup", + "prow" + ]) + build_cluster_issuers = { + aks = { + issuer = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f" + } + eks = { + issuer = "https://oidc.eks.us-east-2.amazonaws.com/id/F8B73554FE6FBAF9B19569183FB39762" + } + gke = { + issuer = "https://container.googleapis.com/v1/projects/k8s-infra-prow-build/locations/us-central1/clusters/prow-build" + } + } + graph_api_permissions = { + Terraform = { + roles = [ + "62a82d76-70ea-41e2-9197-370581804d09", # Group.ReadWrite.All + "1bfefb4e-e0b5-418b-a88f-73c46d2cc8e9", # Application.ReadWrite.All + "df021288-bdef-4463-88db-98f22de89214", # User.Read.All + "9e3f62cf-ca93-4989-b6ce-bf83c28f9fe8", # RoleManagement.ReadWrite.Directory + ] + } + } + +} + +resource "azuread_application" "apps" { + for_each = local.apps + display_name = each.key + + dynamic "required_resource_access" { + for_each = try(local.graph_api_permissions[each.key], null) == null ? [] : [local.graph_api_permissions[each.key]] + content { + resource_app_id = "00000003-0000-0000-c000-000000000000" # Microsoft Graph + + dynamic "resource_access" { + for_each = required_resource_access.value.roles + content { + id = resource_access.value + type = "Role" + } + } + } + } +} + +resource "azuread_service_principal" "service_principals" { + for_each = local.apps + client_id = azuread_application.apps[each.key].client_id +} + +resource "azuread_application_federated_identity_credential" "terraform" { + for_each = toset([ + "system:serviceaccount:atlantis:atlantis", + ]) + display_name = reverse(split(":", each.key))[0] + audiences = ["api://AzureADTokenExchange"] + issuer = "https://container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/utility" + application_id = azuread_application.apps["Terraform"].id + subject = each.key +} + +resource "azuread_application_federated_identity_credential" "rg_cleanup" { + for_each = toset([ + "system:serviceaccount:test-pods:rg-cleanup", + ]) + display_name = reverse(split(":", each.key))[0] + audiences = ["api://AzureADTokenExchange"] + issuer = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f" + application_id = azuread_application.apps["rg-cleanup"].id + subject = each.key +} + +resource "azuread_application_federated_identity_credential" "prow" { + for_each = local.build_cluster_issuers + display_name = each.key + audiences = ["api://AzureADTokenExchange"] + issuer = each.value.issuer + application_id = azuread_application.apps["prow"].id + subject = "system:serviceaccount:test-pods:azure" +} diff --git a/kubernetes/aks-prow-build/prow/kustomization.yaml b/kubernetes/aks-prow-build/prow/kustomization.yaml index 0956ff71900..84b28265f49 100644 --- a/kubernetes/aks-prow-build/prow/kustomization.yaml +++ b/kubernetes/aks-prow-build/prow/kustomization.yaml @@ -4,3 +4,4 @@ namespace: test-pods resources: - kyverno.yaml + - serviceaccounts.yaml diff --git a/kubernetes/aks-prow-build/prow/serviceaccounts.yaml b/kubernetes/aks-prow-build/prow/serviceaccounts.yaml new file mode 100644 index 00000000000..51f17c83c53 --- /dev/null +++ b/kubernetes/aks-prow-build/prow/serviceaccounts.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure + annotations: + azure.workload.identity/client-id: 333bb18b-207b-4abd-9ed0-e7e3834378b1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rg-cleanup + annotations: + azure.workload.identity/client-id: f23f8fcc-855b-40fd-a41b-b329ccdb95a1 diff --git a/kubernetes/gke-utility/atlantis/kustomization.yaml b/kubernetes/gke-utility/atlantis/kustomization.yaml index 734edb5ab87..73d3edef41b 100644 --- a/kubernetes/gke-utility/atlantis/kustomization.yaml +++ b/kubernetes/gke-utility/atlantis/kustomization.yaml @@ -36,6 +36,16 @@ patchesStrategicMerge: value: /var/run/secrets/aws-iam-token/serviceaccount/token - name: AWS_REGION value: us-east-2 + - name: ARM_USE_AKS_WORKLOAD_IDENTITY + value: "true" + - name: ARM_SUBSCRIPTION_ID + value: 46678f10-4bbb-447e-98e8-d2829589f2d8 + - name: AZURE_CLIENT_ID + value: 6fe87cee-6470-45d8-accc-57687193e504 + - name: AZURE_FEDERATED_TOKEN_FILE + value: /var/run/secrets/azure-token/serviceaccount/token + - name: AZURE_TENANT_ID + value: d1aa7522-0959-442e-80ee-8c4f7fb4c184 - name: ATLANTIS_CONFIG value: /config/atlantis.yaml - name: ATLANTIS_GH_TOKEN @@ -61,6 +71,9 @@ patchesStrategicMerge: - mountPath: /var/run/secrets/aws-iam-token/serviceaccount name: aws-iam-token readOnly: true + - name: azure-token + mountPath: /var/run/secrets/azure-token/serviceaccount + readOnly: true volumes: - name: config configMap: @@ -73,3 +86,11 @@ patchesStrategicMerge: audience: sts.amazonaws.com expirationSeconds: 86400 path: token + - name: azure-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + expirationSeconds: 86400 + path: token + audience: api://AzureADTokenExchange