Skip to content

Commit 6a013be

Browse files
upodroidupodroid
committed
start managing azure infra with terraform
1 parent f3efca9 commit 6a013be

File tree

9 files changed

+305
-4
lines changed

9 files changed

+305
-4
lines changed

‎infra/azure/terraform/README.md‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
11
# infra/azure/terraform
2-
Terraform scripts and documentation for Cluster API for Azure ([CAPZ](https://github.com/kubernetes-sigs/cluster-api-provider-azure/tree/main/templates)) infrastructure that the Kubernetes community runs on Azure.
3-
The terraform folder structure looks like this:
4-
- capz
5-
- cleanup-app
2+
Terraform code for the Kubernetes Infra on Azure
3+
4+
We have 2 subscriptions in Azure
5+
- Prod Sub - 46678f10-4bbb-447e-98e8-d2829589f2d8
6+
- CI Sub - 59cb4516-507c-4c86-bb40-6f3572dcfaeb
7+
8+
The prod sub has our build cluster and various production resources while the CI sub is used by prow to create ephemeral resource groups for CI testing.
9+
10+
Directory Structure:
11+
- root; this layer has rbac for privileged resources and Entra ID
12+
- k8s-infra-prow-build: this layer has the following resource groups
13+
- rg-k8s-infra-prow-build
14+
- capz; this layer has all the capz resources in the following resource groups
15+
- capz-ci
16+
- capz-monitoring
17+
- cluster-api-gallery

‎infra/azure/terraform/root/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This layer holds the foundation resources for the Azure Tenant

‎infra/azure/terraform/root/data.tf‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
locals {
18+
subscriptions_id = {
19+
"prod" = "46678f10-4bbb-447e-98e8-d2829589f2d8"
20+
"ci" = "59cb4516-507c-4c86-bb40-6f3572dcfaeb"
21+
}
22+
}

‎infra/azure/terraform/root/main.tf‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
// Azure RBAC
17+
18+
module "role_assignments" {
19+
source = "github.com/Azure/terraform-azurerm-avm-res-authorization-roleassignment?ref=v0.3.0"
20+
# version = "0.3.0"
21+
groups_by_display_name = {
22+
capz-admins = "capz-admins"
23+
owners = "owners"
24+
}
25+
app_registrations_by_display_name = {
26+
datadog = "Datadog"
27+
terraform = "Terraform"
28+
}
29+
# user_assigned_managed_identities_by_display_name = {}
30+
role_definitions = {
31+
owner = {
32+
name = "Owner"
33+
}
34+
contributor = {
35+
name = "Contributor"
36+
}
37+
reader = {
38+
name = "Reader"
39+
}
40+
monitoring-reader = {
41+
name = "Monitoring Reader"
42+
}
43+
}
44+
45+
entra_id_role_definitions = {
46+
application-administrator = {
47+
display_name = "Application Administrator"
48+
}
49+
}
50+
51+
role_assignments_for_management_groups = {
52+
root = {
53+
management_group_display_name = "Tenant Root Group"
54+
role_assignments = {
55+
owner = {
56+
role_definition = "owner"
57+
any_principals = ["owners", "terraform"]
58+
}
59+
monitoring-reader = {
60+
role_definition = "monitoring-reader"
61+
app_registrations = ["datadog"]
62+
}
63+
}
64+
}
65+
}
66+
67+
role_assignments_for_subscriptions = {
68+
prod = {
69+
subscription_id = local.subscriptions_id["prod"]
70+
role_assignments = {
71+
owner = {
72+
role_definition = "owner"
73+
groups = ["capz-admins"]
74+
}
75+
}
76+
77+
}
78+
ci = {
79+
subscription_id = local.subscriptions_id["ci"]
80+
role_assignments = {
81+
owner = {
82+
role_definition = "owner"
83+
groups = ["capz-admins"]
84+
}
85+
}
86+
}
87+
}
88+
89+
}

‎infra/azure/terraform/root/providers.tf‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
terraform {
18+
required_version = "~> 1.11.4"
19+
required_providers {
20+
azurerm = {
21+
source = "hashicorp/azurerm"
22+
version = ">= 4.57, < 5.0"
23+
}
24+
azapi = {
25+
source = "azure/azapi"
26+
version = ">= 2.8, < 3"
27+
}
28+
azuread = {
29+
source = "hashicorp/azuread"
30+
version = ">= 3.7, < 4"
31+
}
32+
}
33+
34+
backend "azurerm" {
35+
resource_group_name = "k8s-infra-tf-states-rg"
36+
storage_account_name = "k8sinfratfstateprow"
37+
container_name = "terraform-state"
38+
key = "root.terraform.tfstate"
39+
}
40+
}
41+
42+
provider "azurerm" {
43+
subscription_id = "46678f10-4bbb-447e-98e8-d2829589f2d8" # Prod Subscription
44+
# Configuration options
45+
features {}
46+
}
47+
48+
provider "azurerm" {
49+
subscription_id = "59cb4516-507c-4c86-bb40-6f3572dcfaeb" # CI Subscription
50+
alias = "ci"
51+
# Configuration options
52+
features {}
53+
}
54+
55+
provider "azuread" {
56+
# Configuration options
57+
}
58+

‎infra/azure/terraform/root/sp.tf‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
2+
locals {
3+
apps = toset([
4+
"Terraform",
5+
"rg-cleanup",
6+
"prow"
7+
])
8+
build_cluster_issuers = {
9+
aks = {
10+
issuer = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f"
11+
}
12+
eks = {
13+
issuer = "https://oidc.eks.us-east-2.amazonaws.com/id/F8B73554FE6FBAF9B19569183FB39762"
14+
}
15+
gke = {
16+
issuer = "https://container.googleapis.com/v1/projects/k8s-infra-prow-build/locations/us-central1/clusters/prow-build"
17+
}
18+
}
19+
graph_api_permissions = {
20+
Terraform = {
21+
roles = [
22+
"62a82d76-70ea-41e2-9197-370581804d09", # Group.ReadWrite.All
23+
"1bfefb4e-e0b5-418b-a88f-73c46d2cc8e9", # Application.ReadWrite.All
24+
"df021288-bdef-4463-88db-98f22de89214", # User.Read.All
25+
"9e3f62cf-ca93-4989-b6ce-bf83c28f9fe8", # RoleManagement.ReadWrite.Directory
26+
]
27+
}
28+
}
29+
30+
}
31+
32+
resource "azuread_application" "apps" {
33+
for_each = local.apps
34+
display_name = each.key
35+
36+
dynamic "required_resource_access" {
37+
for_each = try(local.graph_api_permissions[each.key], null) == null ? [] : [local.graph_api_permissions[each.key]]
38+
content {
39+
resource_app_id = "00000003-0000-0000-c000-000000000000" # Microsoft Graph
40+
41+
dynamic "resource_access" {
42+
for_each = required_resource_access.value.roles
43+
content {
44+
id = resource_access.value
45+
type = "Role"
46+
}
47+
}
48+
}
49+
}
50+
}
51+
52+
resource "azuread_service_principal" "service_principals" {
53+
for_each = local.apps
54+
client_id = azuread_application.apps[each.key].client_id
55+
}
56+
57+
resource "azuread_application_federated_identity_credential" "terraform" {
58+
for_each = toset([
59+
"system:serviceaccount:atlantis:atlantis",
60+
])
61+
display_name = reverse(split(":", each.key))[0]
62+
audiences = ["api://AzureADTokenExchange"]
63+
issuer = "https://container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/utility"
64+
application_id = azuread_application.apps["Terraform"].id
65+
subject = each.key
66+
}
67+
68+
resource "azuread_application_federated_identity_credential" "rg_cleanup" {
69+
for_each = toset([
70+
"system:serviceaccount:test-pods:rg-cleanup",
71+
])
72+
display_name = reverse(split(":", each.key))[0]
73+
audiences = ["api://AzureADTokenExchange"]
74+
issuer = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f"
75+
application_id = azuread_application.apps["rg-cleanup"].id
76+
subject = each.key
77+
}
78+
79+
resource "azuread_application_federated_identity_credential" "prow" {
80+
for_each = local.build_cluster_issuers
81+
display_name = each.key
82+
audiences = ["api://AzureADTokenExchange"]
83+
issuer = each.value.issuer
84+
application_id = azuread_application.apps["prow"].id
85+
subject = "system:serviceaccount:test-pods:azure"
86+
}

‎kubernetes/aks-prow-build/prow/kustomization.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ namespace: test-pods
44

55
resources:
66
- kyverno.yaml
7+
- serviceaccounts.yaml

‎kubernetes/aks-prow-build/prow/serviceaccounts.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: azure
5+
annotations:
6+
azure.workload.identity/client-id: 333bb18b-207b-4abd-9ed0-e7e3834378b1
7+
---
8+
apiVersion: v1
9+
kind: ServiceAccount
10+
metadata:
11+
name: rg-cleanup
12+
annotations:
13+
azure.workload.identity/client-id: f23f8fcc-855b-40fd-a41b-b329ccdb95a1

‎kubernetes/gke-utility/atlantis/kustomization.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,14 @@ patchesStrategicMerge:
3636
value: /var/run/secrets/aws-iam-token/serviceaccount/token
3737
- name: AWS_REGION
3838
value: us-east-2
39+
- name: ARM_USE_AKS_WORKLOAD_IDENTITY
40+
value: "true"
41+
- name: AZURE_CLIENT_ID
42+
value: 6fe87cee-6470-45d8-accc-57687193e504
43+
- name: AZURE_FEDERATED_TOKEN_FILE
44+
value: /var/run/secrets/azure-token/serviceaccount/token
45+
- name: AZURE_TENANT_ID
46+
value: d1aa7522-0959-442e-80ee-8c4f7fb4c184
3947
- name: ATLANTIS_CONFIG
4048
value: /config/atlantis.yaml
4149
- name: ATLANTIS_GH_TOKEN
@@ -61,6 +69,9 @@ patchesStrategicMerge:
6169
- mountPath: /var/run/secrets/aws-iam-token/serviceaccount
6270
name: aws-iam-token
6371
readOnly: true
72+
- name: azure-token
73+
mountPath: /var/run/secrets/azure-token/serviceaccount
74+
readOnly: true
6475
volumes:
6576
- name: config
6677
configMap:
@@ -73,3 +84,11 @@ patchesStrategicMerge:
7384
audience: sts.amazonaws.com
7485
expirationSeconds: 86400
7586
path: token
87+
- name: azure-token
88+
projected:
89+
defaultMode: 420
90+
sources:
91+
- serviceAccountToken:
92+
expirationSeconds: 86400
93+
path: token
94+
audience: api://AzureADTokenExchange

0 commit comments

Comments
 (0)