[main] Update common Docker engineering infrastructure with latest

eng/common/templates/1es-official.yml

@@ -2,7 +2,7 @@
 # do the following:
 #
 # - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines
-# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans
+# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans
 # - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans
 #
 # If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in
@@ -57,8 +57,7 @@ extends:
           enabled: true
         sourceRepositoriesToScan:
           exclude:
-          - repository: InternalVersionsRepo
-          - repository: PublicVersionsRepo
+          - repository: VersionsRepo
         sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
         tsa:
           enabled: true

eng/common/templates/jobs/build-images.yml

@@ -49,7 +49,7 @@ jobs:
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
       $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
-      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") {
+      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
         $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push"
       }
 
@@ -70,6 +70,11 @@ jobs:
         id: $(build.serviceConnection.id)
         tenantId: $(build.serviceConnection.tenantId)
         clientId: $(build.serviceConnection.clientId)
+      - ${{ if eq(parameters.isInternalServicingValidation, true) }}:
+        - name: storage
+          id: $(dotnetstaging.serviceConnection.id)
+          tenantId: $(dotnetstaging.serviceConnection.tenantId)
+          clientId: $(dotnetstaging.serviceConnection.clientId)
       internalProjectName: ${{ parameters.internalProjectName }}
       dockerClientOS: ${{ parameters.dockerClientOS }}
       args: >-
@@ -92,7 +97,7 @@ jobs:
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
     # The following task depends on the SBOM Manifest Generator task installed on the agent.
     # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves.
     - powershell: |
@@ -144,11 +149,11 @@ jobs:
         }
       displayName: Generate SBOMs
       condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
-  - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}:
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
     - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/templates/steps/publish-artifact.yml@self
       parameters:
         path: $(sbomDirectory)

eng/common/templates/jobs/publish.yml

@@ -5,6 +5,8 @@ parameters:
   customPublishVariables: []
   sourceBuildPipelineDefinitionId: ""
   sourceBuildPipelineRunId: ""
+  versionsRepoRef: null
+  versionsRepoPath: ""
 
 jobs:
 - job: Publish
@@ -34,13 +36,20 @@ jobs:
   - ${{ parameters.customPublishVariables }}
 
   steps:
+  - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
+    parameters:
+      cloneVersionsRepo: ${{ variables.publishImageInfo }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+
   - template: /eng/common/templates/steps/retain-build.yml@self
 
   - template: /eng/common/templates/steps/init-docker-linux.yml@self
 
   - pwsh: |
       $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
       echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName"
+      $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}"
+      echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot"
     displayName: Set Publish Variables
 
   - ${{ parameters.customInitSteps }}
@@ -138,13 +147,16 @@ jobs:
   - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
     displayName: Create EOL Annotation Data Directory
 
-  - powershell: >-
-      $(engCommonPath)/Invoke-WithRetry.ps1
-      "curl -fSL
-      --output $(imageInfoHostDir)/full-image-info-orig.json
-      https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)"
+  - script: |-
+      cd $(versionsRepoRoot)
+      git pull origin $(gitHubVersionsRepoInfo.branch)
+    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
+    displayName: Pull Latest Changes from Versions Repo
+
+  - script: >-
+      cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
-    displayName: Download Latest Image Info
+    displayName: Copy Latest Image Info from Versions Repo
 
   - script: >
       $(runImageBuilderCmd) mergeImageInfo

eng/common/templates/stages/build-and-test.yml

@@ -22,8 +22,7 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
-  internalVersionsRepoRef: null
-  publicVersionsRepoRef: null
+  versionsRepoRef: ""
 
   isInternalServicingValidation: false
 
@@ -51,6 +50,7 @@ stages:
   condition: and(succeeded(), contains(variables['stages'], 'build'))
   dependsOn: []
   jobs:
+
   - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
     parameters:
       name: PreBuildValidation
@@ -69,12 +69,14 @@ stages:
             echo "##vso[task.setvariable variable=osVersions]"
             echo "##vso[task.setvariable variable=architecture]"
           displayName: Initialize Test Variables
+
   - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self
     parameters:
       name: CopyBaseImages
       pool: ${{ parameters.linuxAmd64Pool }}
       additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)"
       customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
+
   - template: /eng/common/templates/jobs/generate-matrix.yml@self
     parameters:
       matrixType: ${{ parameters.buildMatrixType }}
@@ -85,12 +87,10 @@ stages:
       noCache: ${{ parameters.noCache }}
       customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
+
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Linux_amd64
@@ -99,12 +99,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -118,12 +115,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -137,12 +131,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -156,12 +147,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -175,12 +163,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -194,18 +179,14 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
       isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
@@ -215,12 +196,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -243,7 +221,7 @@ stages:
 ################################################################################
 # Test Images
 ################################################################################
-- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
+- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
   - stage: Test
     dependsOn: Post_Build
     condition: "
@@ -270,11 +248,9 @@ stages:
         customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
         commonInitStepsForMatrixAndBuild:
-        - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
           parameters:
-            noCache: ${{ parameters.noCache }}
-            internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-            publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+            versionsRepoRef: ${{ parameters.versionsRepoRef }}
     - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
       parameters:
         name: Linux_amd64

eng/common/templates/stages/dotnet/build-and-test.yml

@@ -30,6 +30,8 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
+  versionsRepoRef: null
+
 stages:
 - template: /eng/common/templates/stages/build-and-test.yml@self
   parameters:
@@ -51,8 +53,9 @@ stages:
     testMatrixType: ${{ parameters.testMatrixType }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
 
-    internalVersionsRepoRef: InternalVersionsRepo
-    publicVersionsRepoRef: PublicVersionsRepo
+    # Only clone versions repo if we need to reference it during the build in order to cache images.
+    ${{ if eq(parameters.noCache, false) }}:
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
     # Linux AMD64
     linuxAmd64Pool:

eng/common/templates/stages/dotnet/build-test-publish-repo.yml

@@ -32,6 +32,7 @@ parameters:
   # Other common parameters
   internalProjectName: null
   publicProjectName: null
+  versionsRepoRef: null
 
 
 stages:
@@ -61,6 +62,7 @@ stages:
     # Other
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
 - template: /eng/common/templates/stages/dotnet/publish.yml@self
   parameters:
@@ -70,3 +72,4 @@ stages:
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}

eng/common/templates/stages/dotnet/publish.yml

@@ -10,6 +10,7 @@ parameters:
   customPublishInitSteps: []
   sourceBuildPipelineDefinitionId: ''
   sourceBuildPipelineRunId: ''
+  versionsRepoRef: null
 
 stages:
 - template: /eng/common/templates/stages/publish.yml@self
@@ -20,6 +21,7 @@ stages:
     isStandalonePublish: ${{ parameters.isStandalonePublish }}
     sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
     customPublishInitSteps:
     - pwsh: |

eng/common/templates/stages/publish.yml

@@ -14,13 +14,18 @@ parameters:
   sourceBuildPipelineDefinitionId: ''
   sourceBuildPipelineRunId: ''
 
+  versionsRepoRef: null
+  versionsRepoPath: "versions"
+
 ################################################################################
 # Publish Images
 ################################################################################
 stages:
 - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}:
   - stage: Publish
-    ${{ if not(parameters.isStandalonePublish) }}:
+    ${{ if eq(parameters.isStandalonePublish, true) }}:
+      dependsOn: []
+    ${{ else }}:
       ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
         dependsOn: Test
       ${{ else }}:
@@ -63,3 +68,5 @@ stages:
         customInitSteps: ${{ parameters.customPublishInitSteps }}
         sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+        versionsRepoRef: ${{ parameters.versionsRepoRef }}
+        versionsRepoPath: ${{ parameters.versionsRepoPath }}