Skip to content

Commit ba08330

Browse files
upodroidupodroid
committed
start managing azure infra with terraform
1 parent f3efca9 commit ba08330

File tree

9 files changed

+310
-0
lines changed

9 files changed

+310
-0
lines changed

‎infra/azure/terraform/root/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This layer holds the foundation resources for the Azure Tenant

‎infra/azure/terraform/root/data.tf‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
locals {
18+
subscriptions_id = {
19+
"prod" = "46678f10-4bbb-447e-98e8-d2829589f2d8"
20+
"ci" = "59cb4516-507c-4c86-bb40-6f3572dcfaeb"
21+
}
22+
}

‎infra/azure/terraform/root/groups.tf‎

Whitespace-only changes.

‎infra/azure/terraform/root/main.tf‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
// Azure RBAC
17+
18+
module "role_assignments" {
19+
source = "github.com/Azure/terraform-azurerm-avm-res-authorization-roleassignment?ref=v0.3.0"
20+
# version = "0.3.0"
21+
groups_by_display_name = {
22+
capz-admins = "capz-admins"
23+
owners = "owners"
24+
}
25+
app_registrations_by_display_name = {
26+
datadog = "Datadog"
27+
terraform = "Terraform"
28+
}
29+
# user_assigned_managed_identities_by_display_name = {}
30+
role_definitions = {
31+
owner = {
32+
name = "Owner"
33+
}
34+
contributor = {
35+
name = "Contributor"
36+
}
37+
reader = {
38+
name = "Reader"
39+
}
40+
monitoring-reader = {
41+
name = "Monitoring Reader"
42+
}
43+
}
44+
45+
entra_id_role_definitions = {
46+
application-administrator = {
47+
display_name = "Application Administrator"
48+
}
49+
}
50+
51+
role_assignments_for_management_groups = {
52+
root = {
53+
management_group_display_name = "Tenant Root Group"
54+
role_assignments = {
55+
owner = {
56+
role_definition = "owner"
57+
any_principals = ["owners", "terraform"]
58+
}
59+
monitoring-reader = {
60+
role_definition = "monitoring-reader"
61+
app_registrations = ["datadog"]
62+
}
63+
}
64+
}
65+
}
66+
67+
role_assignments_for_subscriptions = {
68+
prod = {
69+
subscription_id = local.subscriptions_id["prod"]
70+
role_assignments = {
71+
owner = {
72+
role_definition = "owner"
73+
groups = ["capz-admins"]
74+
}
75+
}
76+
77+
}
78+
ci = {
79+
subscription_id = local.subscriptions_id["ci"]
80+
role_assignments = {
81+
owner = {
82+
role_definition = "owner"
83+
groups = ["capz-admins"]
84+
}
85+
}
86+
}
87+
}
88+
89+
}

‎infra/azure/terraform/root/providers.tf‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
terraform {
18+
required_version = "~> 1.11.4"
19+
required_providers {
20+
azurerm = {
21+
source = "hashicorp/azurerm"
22+
version = ">= 4.57, < 5.0"
23+
}
24+
azapi = {
25+
source = "azure/azapi"
26+
version = ">= 2.8, < 3"
27+
}
28+
azuread = {
29+
source = "hashicorp/azuread"
30+
version = ">= 3.7, < 4"
31+
}
32+
}
33+
34+
backend "azurerm" {
35+
resource_group_name = "k8s-infra-tf-states-rg"
36+
storage_account_name = "k8sinfratfstateprow"
37+
container_name = "terraform-state"
38+
key = "root.terraform.tfstate"
39+
}
40+
}
41+
42+
provider "azurerm" {
43+
subscription_id = "46678f10-4bbb-447e-98e8-d2829589f2d8" # Prod Subscription
44+
# Configuration options
45+
features {}
46+
}
47+
48+
provider "azurerm" {
49+
subscription_id = "59cb4516-507c-4c86-bb40-6f3572dcfaeb" # CI Subscription
50+
alias = "ci"
51+
# Configuration options
52+
features {}
53+
}
54+
55+
provider "azapi" {
56+
subscription_id = "46678f10-4bbb-447e-98e8-d2829589f2d8"
57+
}
58+
59+
provider "azuread" {
60+
# Configuration options
61+
}
62+

‎infra/azure/terraform/root/sp.tf‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
locals {
18+
apps = toset([
19+
"Terraform",
20+
"rg-cleanup",
21+
"prow"
22+
])
23+
build_cluster_issuers = {
24+
aks = {
25+
issuer = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f"
26+
}
27+
eks = {
28+
issuer = "https://oidc.eks.us-east-2.amazonaws.com/id/F8B73554FE6FBAF9B19569183FB39762"
29+
}
30+
gke = {
31+
issuer = "https://container.googleapis.com/v1/projects/k8s-infra-prow-build/locations/us-central1/clusters/prow-build"
32+
}
33+
}
34+
graph_api_permissions = {
35+
Terraform = {
36+
roles = [
37+
"62a82d76-70ea-41e2-9197-370581804d09", # Group.ReadWrite.All
38+
"1bfefb4e-e0b5-418b-a88f-73c46d2cc8e9", # Application.ReadWrite.All
39+
"df021288-bdef-4463-88db-98f22de89214", # User.Read.All
40+
"9e3f62cf-ca93-4989-b6ce-bf83c28f9fe8", # RoleManagement.ReadWrite.Directory
41+
]
42+
}
43+
}
44+
45+
}
46+
47+
resource "azuread_application" "apps" {
48+
for_each = local.apps
49+
display_name = each.key
50+
51+
dynamic "required_resource_access" {
52+
for_each = try(local.graph_api_permissions[each.key], null) == null ? [] : [local.graph_api_permissions[each.key]]
53+
content {
54+
resource_app_id = "00000003-0000-0000-c000-000000000000" # Microsoft Graph
55+
56+
dynamic "resource_access" {
57+
for_each = required_resource_access.value.roles
58+
content {
59+
id = resource_access.value
60+
type = "Role"
61+
}
62+
}
63+
}
64+
}
65+
}
66+
67+
resource "azuread_service_principal" "service_principals" {
68+
for_each = local.apps
69+
client_id = azuread_application.apps[each.key].client_id
70+
}
71+
72+
resource "azuread_application_federated_identity_credential" "terraform" {
73+
for_each = toset([
74+
"system:serviceaccount:atlantis:atlantis",
75+
])
76+
display_name = reverse(split(":", each.key))[0]
77+
audiences = ["api://AzureADTokenExchange"]
78+
issuer = "https://container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/utility"
79+
application_id = azuread_application.apps["Terraform"].id
80+
subject = each.key
81+
}
82+
83+
resource "azuread_application_federated_identity_credential" "rg_cleanup" {
84+
for_each = toset([
85+
"system:serviceaccount:test-pods:rg-cleanup",
86+
])
87+
display_name = reverse(split(":", each.key))[0]
88+
audiences = ["api://AzureADTokenExchange"]
89+
issuer = "https://eastus2.oic.prod-aks.azure.com/d1aa7522-0959-442e-80ee-8c4f7fb4c184/85d5aa19-bc3c-4cdb-bc17-0cf8703cfa3f"
90+
application_id = azuread_application.apps["rg-cleanup"].id
91+
subject = each.key
92+
}
93+
94+
resource "azuread_application_federated_identity_credential" "prow" {
95+
for_each = local.build_cluster_issuers
96+
display_name = each.key
97+
audiences = ["api://AzureADTokenExchange"]
98+
issuer = each.value.issuer
99+
application_id = azuread_application.apps["prow"].id
100+
subject = "system:serviceaccount:test-pods:azure"
101+
}

‎kubernetes/aks-prow-build/prow/kustomization.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ namespace: test-pods
44

55
resources:
66
- kyverno.yaml
7+
- serviceaccounts.yaml

‎kubernetes/aks-prow-build/prow/serviceaccounts.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: azure
5+
annotations:
6+
azure.workload.identity/client-id: 333bb18b-207b-4abd-9ed0-e7e3834378b1
7+
---
8+
apiVersion: v1
9+
kind: ServiceAccount
10+
metadata:
11+
name: rg-cleanup
12+
annotations:
13+
azure.workload.identity/client-id: f23f8fcc-855b-40fd-a41b-b329ccdb95a1

‎kubernetes/gke-utility/atlantis/kustomization.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@ patchesStrategicMerge:
3636
value: /var/run/secrets/aws-iam-token/serviceaccount/token
3737
- name: AWS_REGION
3838
value: us-east-2
39+
- name: ARM_USE_AKS_WORKLOAD_IDENTITY
40+
value: "true"
41+
- name: ARM_SUBSCRIPTION_ID
42+
value: 46678f10-4bbb-447e-98e8-d2829589f2d8
43+
- name: AZURE_CLIENT_ID
44+
value: 6fe87cee-6470-45d8-accc-57687193e504
45+
- name: AZURE_FEDERATED_TOKEN_FILE
46+
value: /var/run/secrets/azure-token/serviceaccount/token
47+
- name: AZURE_TENANT_ID
48+
value: d1aa7522-0959-442e-80ee-8c4f7fb4c184
3949
- name: ATLANTIS_CONFIG
4050
value: /config/atlantis.yaml
4151
- name: ATLANTIS_GH_TOKEN
@@ -61,6 +71,9 @@ patchesStrategicMerge:
6171
- mountPath: /var/run/secrets/aws-iam-token/serviceaccount
6272
name: aws-iam-token
6373
readOnly: true
74+
- name: azure-token
75+
mountPath: /var/run/secrets/azure-token/serviceaccount
76+
readOnly: true
6477
volumes:
6578
- name: config
6679
configMap:
@@ -73,3 +86,11 @@ patchesStrategicMerge:
7386
audience: sts.amazonaws.com
7487
expirationSeconds: 86400
7588
path: token
89+
- name: azure-token
90+
projected:
91+
defaultMode: 420
92+
sources:
93+
- serviceAccountToken:
94+
expirationSeconds: 86400
95+
path: token
96+
audience: api://AzureADTokenExchange

0 commit comments

Comments
 (0)