Skip to content

Commit 7b57ab2

Browse files
fseldowCopilot
fseldow
and
Copilot
authored
feat: oras pull pause image if not cached for windows network isolated cluster (#8038)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent a600784 commit 7b57ab2

File tree

4 files changed

+213
-2
lines changed

4 files changed

+213
-2
lines changed

‎e2e/scenario_win_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
507507
Description: "Tests that Windows nodes in network isolated clusters configure containerd to use the bootstrap profile container registry for MCR images",
508508
Tags: Tags{
509509
NetworkIsolated: true,
510-
NonAnonymousACR: true,
510+
NonAnonymousACR: false,
511511
},
512512
Config: Config{
513513
Cluster: ClusterAzureBootstrapProfileCache,
@@ -516,7 +516,7 @@ func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
516516
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
517517
PrivateEgress: &datamodel.PrivateEgress{
518518
Enabled: true,
519-
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
519+
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRName(config.Config.DefaultLocation)),
520520
},
521521
}
522522
},

‎staging/cse/windows/containerdfunc.ps1‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,5 +330,13 @@ function Install-Containerd {
330330
-CNIConfDir $CNIConfDir
331331

332332
RegisterContainerDService -KubeDir $KubeDir
333+
if (-not [string]::IsNullOrEmpty($global:BootstrapProfileContainerRegistryServer)) {
334+
if (Get-Command -Name Set-PodInfraContainerImage -ErrorAction SilentlyContinue) {
335+
Set-PodInfraContainerImage
336+
}
337+
else {
338+
Write-Log "Set-PodInfraContainerImage command not found; skipping pod infra container image configuration."
339+
}
340+
}
333341
Enable-Logging
334342
}

‎staging/cse/windows/networkisolatedclusterfunc.ps1‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,81 @@ function Install-Oras {
6565

6666
Write-Log "Oras installed successfully at $($global:OrasPath)"
6767
}
68+
69+
# For network isolated cluster, containerd has issue to pull pause image via credential provider, thus it will not be able to pull pause image during runtime. We have to pull pause image and set pinned label to avoid gc.
70+
function Set-PodInfraContainerImage {
71+
$podInfraContainerImageDownloadDir = "C:\k\pod-infra-container-image\downloads"
72+
$podInfraContainerImageTar = "C:\k\pod-infra-container-image\pod-infra-container-image.tar"
73+
74+
$clusterConfig = ConvertFrom-Json ((Get-Content $global:KubeClusterConfigPath -ErrorAction Stop) | Out-String)
75+
$podInfraContainerImage = $clusterConfig.Cri.Images.Pause
76+
if ([string]::IsNullOrWhiteSpace($podInfraContainerImage)) {
77+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "Failed to recognize pod infra container image"
78+
}
79+
80+
Write-Log "Checking if '$podInfraContainerImage' already exists locally..."
81+
$images = & ctr.exe -n k8s.io images list -q 2>$null
82+
if (($LASTEXITCODE -eq 0) -and ($images -contains $podInfraContainerImage)) {
83+
Write-Log "Image '$podInfraContainerImage' already exists locally, skipping pull"
84+
return
85+
}
86+
87+
$baseName = $podInfraContainerImage -replace ':[^:]+$'
88+
$tag = "local"
89+
90+
$image = $podInfraContainerImage
91+
if (-not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) {
92+
$image = $podInfraContainerImage.Replace("mcr.microsoft.com", $global:BootstrapProfileContainerRegistryServer)
93+
}
94+
95+
if (-not (Test-Path -Path $podInfraContainerImageDownloadDir)) {
96+
New-Item -ItemType Directory -Path $podInfraContainerImageDownloadDir -Force | Out-Null
97+
}
98+
99+
Write-Log "Pulling via oras for '$image'"
100+
$orasCopySucceeded = $false
101+
$orasDestination = '{0}:{1}' -f $podInfraContainerImageDownloadDir, $tag
102+
for ($i = 1; $i -le 10; $i++) {
103+
if ($i -gt 1) {
104+
Start-Sleep -Seconds 5
105+
}
106+
107+
Write-Log "Try $i : oras cp '$image' to '$orasDestination'"
108+
$orasOutput = & $global:OrasPath cp $image $orasDestination --to-oci-layout --from-registry-config $global:OrasRegistryConfigFile 2>&1
109+
if ($LASTEXITCODE -eq 0) {
110+
Write-Log ("Successfully pulled '$image' via oras on attempt $i")
111+
$orasCopySucceeded = $true
112+
break
113+
}
114+
115+
Write-Log ('oras cp attempt {0} failed (exit code {1}): {2}' -f $i, $LASTEXITCODE, $orasOutput)
116+
}
117+
118+
if (-not $orasCopySucceeded) {
119+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "Failed to pull '$image'"
120+
}
121+
tar -cf $podInfraContainerImageTar -C $podInfraContainerImageDownloadDir .
122+
if ($LASTEXITCODE -ne 0) {
123+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "failed to create tar for pod infra image from '$podInfraContainerImageDownloadDir'"
124+
}
125+
126+
$importOutput = $(ctr.exe -n k8s.io image import --base-name $baseName $podInfraContainerImageTar 2>&1)
127+
if ($LASTEXITCODE -ne 0) {
128+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage ('failed to import ''{0}'': {1}' -f $podInfraContainerImage, $importOutput)
129+
}
130+
131+
$finalImage = '{0}:{1}' -f $baseName, $tag
132+
$tagOutput = $(ctr.exe -n k8s.io image tag ${finalImage} $podInfraContainerImage 2>&1)
133+
if ($LASTEXITCODE -ne 0) {
134+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "failed to tag pod infra image: {0}" -f $tagOutput
135+
}
136+
137+
$labelOutput = $(ctr.exe -n k8s.io images label $podInfraContainerImage io.cri-containerd.pinned=pinned 2>&1)
138+
if ($LASTEXITCODE -ne 0) {
139+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage ('failed to label pod infra image ''{0}'' as pinned: {1}' -f $podInfraContainerImage, $labelOutput)
140+
}
141+
Write-Log "Successfully imported '$podInfraContainerImage'"
142+
143+
Remove-Item -Path $podInfraContainerImageDownloadDir -Recurse -Force -ErrorAction SilentlyContinue
144+
Remove-Item -Path $podInfraContainerImageTar -Force -ErrorAction SilentlyContinue
145+
}

‎staging/cse/windows/networkisolatedclusterfunc.tests.ps1‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,128 @@ Describe "Install-Oras" {
105105
} | Should -Throw "*Set-ExitCode:$($global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND):Failed to extract oras archive*"
106106
}
107107
}
108+
109+
Describe "Set-PodInfraContainerImage" {
110+
BeforeEach {
111+
$global:KubeClusterConfigPath = "C:\k\kubeclusterconfig.json"
112+
$global:BootstrapProfileContainerRegistryServer = "myacr.azurecr.io/aks-managed-repository"
113+
$global:OrasRegistryConfigFile = "C:\aks-tools\oras\config.json"
114+
$global:OrasPath = "Mock-OrasCli"
115+
$global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER = 82
116+
117+
Mock Start-Sleep
118+
Mock New-Item
119+
Mock Remove-Item
120+
Mock tar -MockWith { $global:LASTEXITCODE = 0 }
121+
$script:CtrExeInvocations = @()
122+
$script:CtrExeMock = {
123+
param($Args)
124+
return "ok"
125+
}
126+
function global:ctr.exe {
127+
param([Parameter(ValueFromRemainingArguments = $true)]$Args)
128+
$script:CtrExeInvocations += , @($Args)
129+
$global:LASTEXITCODE = 0
130+
return & $script:CtrExeMock $Args
131+
}
132+
Mock Test-Path -MockWith { $false }
133+
Mock Set-ExitCode -MockWith {
134+
param(
135+
[Parameter(Mandatory = $true)][int]$ExitCode,
136+
[Parameter(Mandatory = $true)][string]$ErrorMessage
137+
)
138+
throw "Set-ExitCode:${ExitCode}:${ErrorMessage}"
139+
}
140+
141+
Mock Get-Content -MockWith {
142+
@'
143+
{
144+
"Cri": {
145+
"Images": {
146+
"Pause": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1"
147+
}
148+
}
149+
}
150+
'@
151+
}
152+
}
153+
154+
AfterEach {
155+
Remove-Item Function:\global:ctr.exe -ErrorAction SilentlyContinue
156+
}
157+
158+
It "fails when pod infra image is empty" {
159+
Mock Get-Content -MockWith {
160+
@'
161+
{
162+
"Cri": {
163+
"Images": {
164+
"Pause": ""
165+
}
166+
}
167+
}
168+
'@
169+
}
170+
171+
{
172+
Set-PodInfraContainerImage
173+
} | Should -Throw "*Set-ExitCode:82:Failed to recognize pod infra container image*"
174+
}
175+
176+
It "returns early when image already exists locally" {
177+
$script:CtrExeMock = {
178+
param($Args)
179+
return @("mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1")
180+
}
181+
182+
function global:Mock-OrasCli {
183+
param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Args)
184+
$global:LASTEXITCODE = 0
185+
return "ok"
186+
}
187+
188+
{ Set-PodInfraContainerImage } | Should -Not -Throw
189+
Assert-MockCalled -CommandName 'tar' -Times 0
190+
$script:CtrExeInvocations.Count | Should -Be 1
191+
}
192+
193+
It "pulls via oras and imports image when not found locally" {
194+
$script:CtrExeMock = {
195+
param($Args)
196+
if ($Args -contains 'list') {
197+
return @()
198+
}
199+
return "ok"
200+
}
201+
202+
function global:Mock-OrasCli {
203+
param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Args)
204+
$global:LASTEXITCODE = 0
205+
return "oras ok"
206+
}
207+
208+
{ Set-PodInfraContainerImage } | Should -Not -Throw
209+
Assert-MockCalled -CommandName 'tar' -Times 1
210+
$script:CtrExeInvocations.Count | Should -Be 4
211+
Assert-MockCalled -CommandName 'Remove-Item' -Times 2
212+
}
213+
214+
It "fails after oras retry exhaustion" {
215+
$script:CtrExeMock = {
216+
param($Args)
217+
return @()
218+
}
219+
220+
function global:Mock-OrasCli {
221+
param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Args)
222+
$global:LASTEXITCODE = 1
223+
return "oras failed"
224+
}
225+
226+
{
227+
Set-PodInfraContainerImage
228+
} | Should -Throw "*Set-ExitCode:82:Failed to pull*"
229+
Assert-MockCalled -CommandName 'Start-Sleep' -Times 9
230+
$script:CtrExeInvocations.Count | Should -Be 1
231+
}
232+
}

0 commit comments

Comments
 (0)