Skip to content

Commit 949e7bc

Browse files
Copilotbcho
andcommitted
Address review feedback: Bicep module, kubeadm cert-rotation RBAC, direct apply invocation
Co-authored-by: bcho <1975118+bcho@users.noreply.github.com>
1 parent 5dd892a commit 949e7bc

3 files changed

Lines changed: 193 additions & 254 deletions

File tree

hack/e2e/infra/main.bicep

Lines changed: 49 additions & 232 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55
// - AKS cluster (1-node control plane)
66
// - VM with system-assigned managed identity (MSI auth mode)
77
// - VM without managed identity (bootstrap token auth mode)
8+
// - VM without managed identity (kubeadm apply -f auth mode)
89
//
9-
// Both VMs run Ubuntu 22.04 LTS, have public IPs, and allow SSH ingress.
10+
// All flex-node VMs run Ubuntu 22.04 LTS, have public IPs, and allow SSH
11+
// ingress. VM creation is delegated to the reusable modules/vm.bicep module.
1012
// =============================================================================
1113

1214
@description('Azure region for all resources.')
@@ -137,233 +139,47 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = {
137139
}
138140

139141
// ---------------------------------------------------------------------------
140-
// Public IPs for VMs
142+
// Flex-node VMs (via reusable module)
141143
// ---------------------------------------------------------------------------
142-
resource pipMsi 'Microsoft.Network/publicIPAddresses@2023-11-01' = {
143-
name: '${msiVmName}-pip'
144-
location: location
145-
tags: tags
146-
sku: { name: 'Standard' }
147-
properties: {
148-
publicIPAllocationMethod: 'Static'
149-
}
150-
}
151-
152-
resource pipToken 'Microsoft.Network/publicIPAddresses@2023-11-01' = {
153-
name: '${tokenVmName}-pip'
154-
location: location
155-
tags: tags
156-
sku: { name: 'Standard' }
157-
properties: {
158-
publicIPAllocationMethod: 'Static'
159-
}
160-
}
161-
162-
resource pipKubeadm 'Microsoft.Network/publicIPAddresses@2023-11-01' = {
163-
name: '${kubeadmVmName}-pip'
164-
location: location
165-
tags: tags
166-
sku: { name: 'Standard' }
167-
properties: {
168-
publicIPAllocationMethod: 'Static'
169-
}
170-
}
171-
172-
// ---------------------------------------------------------------------------
173-
// NICs
174-
// ---------------------------------------------------------------------------
175-
resource nicMsi 'Microsoft.Network/networkInterfaces@2023-11-01' = {
176-
name: '${msiVmName}-nic'
177-
location: location
178-
tags: tags
179-
properties: {
180-
ipConfigurations: [
181-
{
182-
name: 'ipconfig1'
183-
properties: {
184-
subnet: {
185-
id: vnet.properties.subnets[1].id
186-
}
187-
publicIPAddress: {
188-
id: pipMsi.id
189-
}
190-
privateIPAllocationMethod: 'Dynamic'
191-
}
192-
}
193-
]
194-
}
195-
}
196-
197-
resource nicToken 'Microsoft.Network/networkInterfaces@2023-11-01' = {
198-
name: '${tokenVmName}-nic'
199-
location: location
200-
tags: tags
201-
properties: {
202-
ipConfigurations: [
203-
{
204-
name: 'ipconfig1'
205-
properties: {
206-
subnet: {
207-
id: vnet.properties.subnets[1].id
208-
}
209-
publicIPAddress: {
210-
id: pipToken.id
211-
}
212-
privateIPAllocationMethod: 'Dynamic'
213-
}
214-
}
215-
]
216-
}
217-
}
218-
219-
resource nicKubeadm 'Microsoft.Network/networkInterfaces@2023-11-01' = {
220-
name: '${kubeadmVmName}-nic'
221-
location: location
222-
tags: tags
223-
properties: {
224-
ipConfigurations: [
225-
{
226-
name: 'ipconfig1'
227-
properties: {
228-
subnet: {
229-
id: vnet.properties.subnets[1].id
230-
}
231-
publicIPAddress: {
232-
id: pipKubeadm.id
233-
}
234-
privateIPAllocationMethod: 'Dynamic'
235-
}
236-
}
237-
]
144+
module vmMsi 'modules/vm.bicep' = {
145+
name: 'deploy-vm-msi'
146+
params: {
147+
location: location
148+
vmName: msiVmName
149+
vmSize: vmSize
150+
adminUsername: adminUsername
151+
sshPublicKey: sshPublicKey
152+
subnetId: vnet.properties.subnets[1].id
153+
assignManagedIdentity: true
154+
tags: tags
238155
}
239156
}
240157

241-
// ---------------------------------------------------------------------------
242-
// VM: MSI (system-assigned managed identity)
243-
// ---------------------------------------------------------------------------
244-
resource vmMsi 'Microsoft.Compute/virtualMachines@2024-03-01' = {
245-
name: msiVmName
246-
location: location
247-
tags: tags
248-
identity: {
249-
type: 'SystemAssigned'
250-
}
251-
properties: {
252-
hardwareProfile: { vmSize: vmSize }
253-
osProfile: {
254-
computerName: msiVmName
255-
adminUsername: adminUsername
256-
linuxConfiguration: {
257-
disablePasswordAuthentication: true
258-
ssh: {
259-
publicKeys: [
260-
{
261-
path: '/home/${adminUsername}/.ssh/authorized_keys'
262-
keyData: sshPublicKey
263-
}
264-
]
265-
}
266-
}
267-
}
268-
storageProfile: {
269-
imageReference: {
270-
publisher: 'Canonical'
271-
offer: '0001-com-ubuntu-server-jammy'
272-
sku: '22_04-lts-gen2'
273-
version: 'latest'
274-
}
275-
osDisk: {
276-
createOption: 'FromImage'
277-
managedDisk: { storageAccountType: 'StandardSSD_LRS' }
278-
}
279-
}
280-
networkProfile: {
281-
networkInterfaces: [ { id: nicMsi.id } ]
282-
}
158+
module vmToken 'modules/vm.bicep' = {
159+
name: 'deploy-vm-token'
160+
params: {
161+
location: location
162+
vmName: tokenVmName
163+
vmSize: vmSize
164+
adminUsername: adminUsername
165+
sshPublicKey: sshPublicKey
166+
subnetId: vnet.properties.subnets[1].id
167+
assignManagedIdentity: false
168+
tags: tags
283169
}
284170
}
285171

286-
// ---------------------------------------------------------------------------
287-
// VM: Token (no managed identity)
288-
// ---------------------------------------------------------------------------
289-
resource vmToken 'Microsoft.Compute/virtualMachines@2024-03-01' = {
290-
name: tokenVmName
291-
location: location
292-
tags: tags
293-
properties: {
294-
hardwareProfile: { vmSize: vmSize }
295-
osProfile: {
296-
computerName: tokenVmName
297-
adminUsername: adminUsername
298-
linuxConfiguration: {
299-
disablePasswordAuthentication: true
300-
ssh: {
301-
publicKeys: [
302-
{
303-
path: '/home/${adminUsername}/.ssh/authorized_keys'
304-
keyData: sshPublicKey
305-
}
306-
]
307-
}
308-
}
309-
}
310-
storageProfile: {
311-
imageReference: {
312-
publisher: 'Canonical'
313-
offer: '0001-com-ubuntu-server-jammy'
314-
sku: '22_04-lts-gen2'
315-
version: 'latest'
316-
}
317-
osDisk: {
318-
createOption: 'FromImage'
319-
managedDisk: { storageAccountType: 'StandardSSD_LRS' }
320-
}
321-
}
322-
networkProfile: {
323-
networkInterfaces: [ { id: nicToken.id } ]
324-
}
325-
}
326-
}
327-
328-
// ---------------------------------------------------------------------------
329-
// VM: Kubeadm (no managed identity - uses apply -f with kubeadm join flow)
330-
// ---------------------------------------------------------------------------
331-
resource vmKubeadm 'Microsoft.Compute/virtualMachines@2024-03-01' = {
332-
name: kubeadmVmName
333-
location: location
334-
tags: tags
335-
properties: {
336-
hardwareProfile: { vmSize: vmSize }
337-
osProfile: {
338-
computerName: kubeadmVmName
339-
adminUsername: adminUsername
340-
linuxConfiguration: {
341-
disablePasswordAuthentication: true
342-
ssh: {
343-
publicKeys: [
344-
{
345-
path: '/home/${adminUsername}/.ssh/authorized_keys'
346-
keyData: sshPublicKey
347-
}
348-
]
349-
}
350-
}
351-
}
352-
storageProfile: {
353-
imageReference: {
354-
publisher: 'Canonical'
355-
offer: '0001-com-ubuntu-server-jammy'
356-
sku: '22_04-lts-gen2'
357-
version: 'latest'
358-
}
359-
osDisk: {
360-
createOption: 'FromImage'
361-
managedDisk: { storageAccountType: 'StandardSSD_LRS' }
362-
}
363-
}
364-
networkProfile: {
365-
networkInterfaces: [ { id: nicKubeadm.id } ]
366-
}
172+
module vmKubeadm 'modules/vm.bicep' = {
173+
name: 'deploy-vm-kubeadm'
174+
params: {
175+
location: location
176+
vmName: kubeadmVmName
177+
vmSize: vmSize
178+
adminUsername: adminUsername
179+
sshPublicKey: sshPublicKey
180+
subnetId: vnet.properties.subnets[1].id
181+
assignManagedIdentity: false
182+
tags: tags
367183
}
368184
}
369185

@@ -372,21 +188,21 @@ resource vmKubeadm 'Microsoft.Compute/virtualMachines@2024-03-01' = {
372188
// ---------------------------------------------------------------------------
373189
// Azure Kubernetes Service Cluster Admin Role
374190
resource roleClusterAdmin 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
375-
name: guid(aksCluster.id, vmMsi.id, 'aks-cluster-admin')
191+
name: guid(aksCluster.id, vmMsi.outputs.principalId, 'aks-cluster-admin')
376192
scope: aksCluster
377193
properties: {
378-
principalId: vmMsi.identity.principalId
194+
principalId: vmMsi.outputs.principalId
379195
principalType: 'ServicePrincipal'
380196
roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8')
381197
}
382198
}
383199

384200
// Azure Kubernetes Service RBAC Cluster Admin
385201
resource roleRbacAdmin 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
386-
name: guid(aksCluster.id, vmMsi.id, 'aks-rbac-cluster-admin')
202+
name: guid(aksCluster.id, vmMsi.outputs.principalId, 'aks-rbac-cluster-admin')
387203
scope: aksCluster
388204
properties: {
389-
principalId: vmMsi.identity.principalId
205+
principalId: vmMsi.outputs.principalId
390206
principalType: 'ServicePrincipal'
391207
roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b')
392208
}
@@ -399,14 +215,15 @@ output clusterName string = aksCluster.name
399215
output clusterId string = aksCluster.id
400216
output clusterFqdn string = aksCluster.properties.fqdn
401217

402-
output msiVmName string = vmMsi.name
403-
output msiVmIp string = pipMsi.properties.ipAddress
404-
output msiVmPrincipalId string = vmMsi.identity.principalId
218+
output msiVmName string = vmMsi.outputs.vmName
219+
output msiVmIp string = vmMsi.outputs.publicIpAddress
220+
output msiVmPrincipalId string = vmMsi.outputs.principalId
405221

406-
output tokenVmName string = vmToken.name
407-
output tokenVmIp string = pipToken.properties.ipAddress
222+
output tokenVmName string = vmToken.outputs.vmName
223+
output tokenVmIp string = vmToken.outputs.publicIpAddress
408224

409-
output kubeadmVmName string = vmKubeadm.name
410-
output kubeadmVmIp string = pipKubeadm.properties.ipAddress
225+
output kubeadmVmName string = vmKubeadm.outputs.vmName
226+
output kubeadmVmIp string = vmKubeadm.outputs.publicIpAddress
411227

412228
output adminUsername string = adminUsername
229+

0 commit comments

Comments
 (0)