Skip to content

Commit 5acac0e

Browse files
fseldowCopilot
andauthored
feat: windows network isolated cluster support anonymous-disabled bootstrap acr (#8039)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 7b57ab2 commit 5acac0e

File tree

3 files changed

+397
-5
lines changed

3 files changed

+397
-5
lines changed

e2e/scenario_win_test.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/Azure/agentbaker/e2e/components"
1010
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
11+
"github.com/Masterminds/semver"
1112
"github.com/stretchr/testify/require"
1213

1314
"github.com/Azure/agentbaker/e2e/config"
@@ -507,18 +508,35 @@ func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
507508
Description: "Tests that Windows nodes in network isolated clusters configure containerd to use the bootstrap profile container registry for MCR images",
508509
Tags: Tags{
509510
NetworkIsolated: true,
510-
NonAnonymousACR: false,
511+
NonAnonymousACR: true,
511512
},
512513
Config: Config{
513514
Cluster: ClusterAzureBootstrapProfileCache,
514515
VHD: config.VHDWindows2025Gen2,
515516
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
517+
Windows2025BootstrapConfigMutator(t, nbc)
516518
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
517519
PrivateEgress: &datamodel.PrivateEgress{
518520
Enabled: true,
519-
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRName(config.Config.DefaultLocation)),
521+
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
520522
},
521523
}
524+
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true
525+
nbc.AgentPoolProfile.KubernetesConfig.UseManagedIdentity = true
526+
nbc.KubeletConfig["--image-credential-provider-config"] = "c:\\k\\credential-provider-config.yaml"
527+
nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
528+
orchestratorVersion, _ := semver.NewVersion(nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
529+
if orchestratorVersion.LessThan(semver.MustParse("1.32.0")) {
530+
nbc.K8sComponents.WindowsCredentialProviderURL = fmt.Sprintf(
531+
"https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-windows-amd64-v%s.tar.gz",
532+
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion,
533+
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
534+
} else {
535+
nbc.K8sComponents.WindowsCredentialProviderURL = fmt.Sprintf(
536+
"https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/%s/windows/amd64/azure-acr-credential-provider_%s-1_amd64.zip",
537+
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion,
538+
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
539+
}
522540
},
523541
Validator: func(ctx context.Context, s *Scenario) {
524542
// Verify mcr.microsoft.com host config exist

staging/cse/windows/networkisolatedclusterfunc.ps1

Lines changed: 223 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# Initialize-Oras will install oras and login the registry if anonymous access is disabled. This is required for network isolated cluster to pull windowszip from private container registry.
44
function Initialize-Oras {
55
Install-Oras
6-
# reserve for Invoke-OrasLogin to avoid frequent code changes in parts/windows/
6+
Invoke-OrasLogin -Acr_Url $(Get-BootstrapRegistryDomainName) -ClientID $UserAssignedClientID -TenantID $global:TenantId
77
}
88

99
# unpackage and install oras from cache
@@ -143,3 +143,225 @@ function Set-PodInfraContainerImage {
143143
Remove-Item -Path $podInfraContainerImageDownloadDir -Recurse -Force -ErrorAction SilentlyContinue
144144
Remove-Item -Path $podInfraContainerImageTar -Force -ErrorAction SilentlyContinue
145145
}
146+
147+
function Invoke-OrasLogin {
148+
param(
149+
[Parameter(Mandatory = $true)][string]
150+
$Acr_Url,
151+
[Parameter(Mandatory = $true)][string]
152+
$ClientID,
153+
[Parameter(Mandatory = $true)][string]
154+
$TenantID
155+
)
156+
157+
# Check for required variables
158+
if ([string]::IsNullOrWhiteSpace($ClientID) -or [string]::IsNullOrWhiteSpace($TenantID)) {
159+
Write-Host "ClientID or TenantID are not set. Oras login is not possible, proceeding with anonymous pull"
160+
return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
161+
}
162+
163+
# Attempt anonymous pull check (assumes helper function exists)
164+
$retCode = Assert-AnonymousAcrAccess 10 5 $Acr_Url
165+
if ($retCode -eq 0) {
166+
Write-Host "anonymous pull is allowed for acr '$Acr_Url', proceeding with anonymous pull"
167+
return
168+
}
169+
elseif ($retCode -ne 1) {
170+
Write-Host "failed with an error other than unauthorized, exiting.."
171+
Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT -ErrorMessage "failed with an error other than unauthorized, exiting"
172+
}
173+
174+
# Get AAD Access Token using Managed Identity Metadata Service
175+
$accessUrl = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$ClientID"
176+
try {
177+
$requestArgs = @{
178+
Uri = $accessUrl
179+
Method = "Get"
180+
Headers = @{ Metadata = "true" }
181+
TimeoutSec = 10
182+
}
183+
$rawAccessTokenResponse = Retry-Command -Command "Invoke-RestMethod" -Args $requestArgs -Retries 10 -RetryDelaySeconds 5
184+
$accessToken = $rawAccessTokenResponse.access_token
185+
}
186+
catch {
187+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT -ErrorMessage "failed to retrieve AAD access token: $($_.Exception.Message)"
188+
}
189+
190+
if ([string]::IsNullOrWhiteSpace($accessToken)) {
191+
Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to parse imds access token"
192+
}
193+
194+
# Exchange AAD Access Token for ACR Refresh Token
195+
try {
196+
$exchangeUrl = "https://$Acr_Url/oauth2/exchange"
197+
$body = @{
198+
grant_type = "access_token"
199+
service = $Acr_Url
200+
tenant = $TenantID
201+
access_token = $accessToken
202+
}
203+
$requestArgs = @{
204+
Uri = $exchangeUrl
205+
Method = "Post"
206+
ContentType = "application/x-www-form-urlencoded"
207+
Body = $body
208+
TimeoutSec = 10
209+
}
210+
$rawRefreshTokenResponse = Retry-Command -Command "Invoke-RestMethod" -Args $requestArgs -Retries 10 -RetryDelaySeconds 5
211+
$refreshToken = $rawRefreshTokenResponse.refresh_token
212+
}
213+
catch {
214+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to retrieve refresh token: $($_.Exception.Message)"
215+
}
216+
217+
if ([string]::IsNullOrWhiteSpace($refreshToken)) {
218+
Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to parse refresh token"
219+
}
220+
221+
# Pre-validate refresh token permissions
222+
$retCode = Assert-RefreshToken -RefreshToken $refreshToken -RequiredActions @("read")
223+
if ($retCode -ne 0) {
224+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to validate refresh token permissions"
225+
}
226+
227+
# Perform Oras Login (pipe refresh token to stdin for --identity-token-stdin)
228+
$loginSuccess = $false
229+
for ($i = 1; $i -le 3; $i++) {
230+
try {
231+
Write-Log "Retry $i : oras login $Acr_Url"
232+
$loginOutput = $refreshToken | & $global:OrasPath login $Acr_Url --identity-token-stdin --registry-config $global:OrasRegistryConfigFile 2>&1
233+
if ($LASTEXITCODE -eq 0) {
234+
$loginSuccess = $true
235+
break
236+
}
237+
Write-Log "oras login attempt $i failed (exit code $LASTEXITCODE): $loginOutput"
238+
}
239+
catch {
240+
Write-Log "oras login attempt $i exception: $($_.Exception.Message)"
241+
}
242+
if ($i -lt 3) {
243+
Start-Sleep -Seconds 5
244+
}
245+
}
246+
if (-Not $loginSuccess) {
247+
Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to login to acr '$Acr_Url' with identity token"
248+
}
249+
250+
# Clean up sensitive data
251+
Remove-Variable accessToken, refreshToken -ErrorAction SilentlyContinue
252+
253+
Write-Host "successfully logged in to acr '$Acr_Url' with identity token"
254+
}
255+
256+
function Assert-AnonymousAcrAccess {
257+
Param(
258+
[Parameter(Mandatory = $true)][int]$Retries,
259+
[Parameter(Mandatory = $true)][int]$WaitSleep,
260+
[Parameter(Mandatory = $true)][string]$AcrUrl
261+
)
262+
263+
for ($i = 1; $i -le $Retries; $i++) {
264+
# Logout first to ensure insufficient ABAC token won't affect anonymous judging
265+
try { & $global:OrasPath logout $AcrUrl --registry-config $global:OrasRegistryConfigFile 2>$null } catch { }
266+
267+
$output = $null
268+
try {
269+
$output = & $global:OrasPath repo ls $AcrUrl --registry-config $global:OrasRegistryConfigFile 2>&1
270+
}
271+
catch {
272+
$output = $_.Exception.Message
273+
# Ensure we do not rely on a stale success exit code when repo ls throws
274+
$LASTEXITCODE = 1
275+
}
276+
277+
if ($LASTEXITCODE -eq 0) {
278+
Write-Host "acr is anonymously reachable"
279+
return 0
280+
}
281+
282+
if ($output -and ($output -like "*unauthorized: authentication required*")) {
283+
Write-Host "ACR is not anonymously reachable: $output"
284+
return 1
285+
}
286+
287+
Start-Sleep -Seconds $WaitSleep
288+
}
289+
290+
Write-Host "unexpected response from acr: $output"
291+
return $global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT
292+
}
293+
294+
function Assert-RefreshToken {
295+
Param(
296+
[Parameter(Mandatory = $true)][string]$RefreshToken,
297+
[Parameter(Mandatory = $true)][string[]]$RequiredActions
298+
)
299+
300+
# Decode the refresh token (JWT format: header.payload.signature)
301+
# Extract the payload (second part) and decode from base64
302+
$tokenParts = $RefreshToken.Split('.')
303+
if ($tokenParts.Length -lt 2) {
304+
Write-Host "Invalid JWT token format"
305+
return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
306+
}
307+
308+
$tokenPayload = $tokenParts[1]
309+
# Add padding if needed for base64url decoding
310+
switch ($tokenPayload.Length % 4) {
311+
2 { $tokenPayload += "==" }
312+
3 { $tokenPayload += "=" }
313+
}
314+
# Replace base64url characters with standard base64
315+
$tokenPayload = $tokenPayload -replace '-', '+' -replace '_', '/'
316+
317+
try {
318+
$decodedBytes = [System.Convert]::FromBase64String($tokenPayload)
319+
$decodedToken = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
320+
}
321+
catch {
322+
Write-Host "Failed to decode token payload: $($_.Exception.Message)"
323+
return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
324+
}
325+
326+
if (-Not [string]::IsNullOrWhiteSpace($decodedToken)) {
327+
try {
328+
$tokenObj = $decodedToken | ConvertFrom-Json
329+
}
330+
catch {
331+
Write-Host "Failed to parse token JSON: $($_.Exception.Message)"
332+
return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
333+
}
334+
335+
# Check if permissions field exists (RBAC token vs ABAC token)
336+
if ($null -ne $tokenObj.permissions) {
337+
Write-Host "RBAC token detected, validating permissions"
338+
339+
$tokenActions = @()
340+
if ($null -ne $tokenObj.permissions.actions) {
341+
$tokenActions = @($tokenObj.permissions.actions)
342+
}
343+
344+
foreach ($action in $RequiredActions) {
345+
if ($tokenActions -notcontains $action) {
346+
Write-Host "Required action '$action' not found in token permissions"
347+
return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
348+
}
349+
}
350+
Write-Host "Token validation passed: all required actions present"
351+
}
352+
else {
353+
Write-Host "No permissions field found in token. Assuming ABAC token, skipping permission validation"
354+
}
355+
}
356+
357+
return 0
358+
}
359+
360+
function Get-BootstrapRegistryDomainName {
361+
$registryDomainName = if ($global:MCRRepositoryBase) { $global:MCRRepositoryBase } else { "mcr.microsoft.com" } # default to mcr
362+
$registryDomainName = $registryDomainName.TrimEnd("/")
363+
if ($global:BootstrapProfileContainerRegistryServer) {
364+
$registryDomainName = $global:BootstrapProfileContainerRegistryServer.Split("/")[0]
365+
}
366+
return $registryDomainName
367+
}

0 commit comments

Comments
 (0)