From a65def077858c07d0262491fb49ec1cc036daa3d Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:37:59 -0700 Subject: [PATCH 1/5] Update common Docker engineering infrastructure with latest --- eng/common/templates/1es-official.yml | 5 +- eng/common/templates/jobs/build-images.yml | 13 +++- eng/common/templates/jobs/publish.yml | 24 ++++-- .../templates/stages/build-and-test.yml | 74 ++++++------------ .../stages/dotnet/build-and-test.yml | 7 +- .../stages/dotnet/build-test-publish-repo.yml | 3 + .../templates/stages/dotnet/publish.yml | 2 + eng/common/templates/stages/publish.yml | 9 ++- .../steps/init-matrix-build-publish.yml | 77 +++++++++++++++++++ .../templates/variables/docker-images.yml | 2 +- 10 files changed, 150 insertions(+), 66 deletions(-) create mode 100644 eng/common/templates/steps/init-matrix-build-publish.yml diff --git a/eng/common/templates/1es-official.yml b/eng/common/templates/1es-official.yml index 357ab811d..2cb7a9430 100644 --- a/eng/common/templates/1es-official.yml +++ b/eng/common/templates/1es-official.yml @@ -2,7 +2,7 @@ # do the following: # # - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines -# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans # - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans # # If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in @@ -57,8 +57,7 @@ extends: enabled: true sourceRepositoriesToScan: exclude: - - repository: InternalVersionsRepo - - repository: PublicVersionsRepo + - repository: VersionsRepo sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} tsa: enabled: true diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 6d2482e9b..420120d25 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -49,7 +49,7 @@ jobs: # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates # the environment variable for us. $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" - if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { + if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push" } @@ -70,6 +70,11 @@ jobs: id: $(build.serviceConnection.id) tenantId: $(build.serviceConnection.tenantId) clientId: $(build.serviceConnection.clientId) + - ${{ if eq(parameters.isInternalServicingValidation, true) }}: + - name: storage + id: $(dotnetstaging.serviceConnection.id) + tenantId: $(dotnetstaging.serviceConnection.tenantId) + clientId: $(dotnetstaging.serviceConnection.clientId) internalProjectName: ${{ parameters.internalProjectName }} dockerClientOS: ${{ parameters.dockerClientOS }} args: >- @@ -92,7 +97,7 @@ jobs: displayName: Publish Image Info File Artifact internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: # The following task depends on the SBOM Manifest Generator task installed on the agent. # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves. - powershell: | @@ -144,11 +149,11 @@ jobs: } displayName: Generate SBOMs condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) - - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}: + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self parameters: condition: ne(variables.testScriptPath, '') - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: path: $(sbomDirectory) diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml index 52a851446..6e8703ea0 100644 --- a/eng/common/templates/jobs/publish.yml +++ b/eng/common/templates/jobs/publish.yml @@ -5,6 +5,8 @@ parameters: customPublishVariables: [] sourceBuildPipelineDefinitionId: "" sourceBuildPipelineRunId: "" + versionsRepoRef: null + versionsRepoPath: "" jobs: - job: Publish @@ -34,6 +36,11 @@ jobs: - ${{ parameters.customPublishVariables }} steps: + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self + parameters: + cloneVersionsRepo: ${{ variables.publishImageInfo }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + - template: /eng/common/templates/steps/retain-build.yml@self - template: /eng/common/templates/steps/init-docker-linux.yml@self @@ -41,6 +48,8 @@ jobs: - pwsh: | $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName" + $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}" + echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot" displayName: Set Publish Variables - ${{ parameters.customInitSteps }} @@ -138,13 +147,16 @@ jobs: - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data displayName: Create EOL Annotation Data Directory - - powershell: >- - $(engCommonPath)/Invoke-WithRetry.ps1 - "curl -fSL - --output $(imageInfoHostDir)/full-image-info-orig.json - https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)" + - script: |- + cd $(versionsRepoRoot) + git pull origin $(gitHubVersionsRepoInfo.branch) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Pull Latest Changes from Versions Repo + + - script: >- + cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Download Latest Image Info + displayName: Copy Latest Image Info from Versions Repo - script: > $(runImageBuilderCmd) mergeImageInfo diff --git a/eng/common/templates/stages/build-and-test.yml b/eng/common/templates/stages/build-and-test.yml index 1b5d4a8f7..d1930a2b5 100644 --- a/eng/common/templates/stages/build-and-test.yml +++ b/eng/common/templates/stages/build-and-test.yml @@ -22,8 +22,7 @@ parameters: internalProjectName: null publicProjectName: null - internalVersionsRepoRef: null - publicVersionsRepoRef: null + versionsRepoRef: "" isInternalServicingValidation: false @@ -51,6 +50,7 @@ stages: condition: and(succeeded(), contains(variables['stages'], 'build')) dependsOn: [] jobs: + - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: PreBuildValidation @@ -69,12 +69,14 @@ stages: echo "##vso[task.setvariable variable=osVersions]" echo "##vso[task.setvariable variable=architecture]" displayName: Initialize Test Variables + - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self parameters: name: CopyBaseImages pool: ${{ parameters.linuxAmd64Pool }} additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + - template: /eng/common/templates/jobs/generate-matrix.yml@self parameters: matrixType: ${{ parameters.buildMatrixType }} @@ -85,12 +87,10 @@ stages: noCache: ${{ parameters.noCache }} customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_amd64 @@ -99,12 +99,9 @@ stages: dockerClientOS: linux buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} @@ -118,12 +115,9 @@ stages: dockerClientOS: linux buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} @@ -137,12 +131,9 @@ stages: dockerClientOS: linux buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} @@ -156,12 +147,9 @@ stages: dockerClientOS: windows buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} @@ -175,12 +163,9 @@ stages: dockerClientOS: windows buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} @@ -194,18 +179,14 @@ stages: dockerClientOS: windows buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: @@ -215,12 +196,9 @@ stages: dockerClientOS: windows buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} @@ -243,7 +221,7 @@ stages: ################################################################################ # Test Images ################################################################################ -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - stage: Test dependsOn: Post_Build condition: " @@ -270,11 +248,9 @@ stages: customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self + - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_amd64 diff --git a/eng/common/templates/stages/dotnet/build-and-test.yml b/eng/common/templates/stages/dotnet/build-and-test.yml index 03bcaf372..3e6c80d89 100644 --- a/eng/common/templates/stages/dotnet/build-and-test.yml +++ b/eng/common/templates/stages/dotnet/build-and-test.yml @@ -30,6 +30,8 @@ parameters: internalProjectName: null publicProjectName: null + versionsRepoRef: null + stages: - template: /eng/common/templates/stages/build-and-test.yml@self parameters: @@ -51,8 +53,9 @@ stages: testMatrixType: ${{ parameters.testMatrixType }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - internalVersionsRepoRef: InternalVersionsRepo - publicVersionsRepoRef: PublicVersionsRepo + # Only clone versions repo if we need to reference it during the build in order to cache images. + ${{ if eq(parameters.noCache, false) }}: + versionsRepoRef: ${{ parameters.versionsRepoRef }} # Linux AMD64 linuxAmd64Pool: diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml index 65bc54584..f7f54fce8 100644 --- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml @@ -32,6 +32,7 @@ parameters: # Other common parameters internalProjectName: null publicProjectName: null + versionsRepoRef: null stages: @@ -61,6 +62,7 @@ stages: # Other internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} - template: /eng/common/templates/stages/dotnet/publish.yml@self parameters: @@ -70,3 +72,4 @@ stages: internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/common/templates/stages/dotnet/publish.yml b/eng/common/templates/stages/dotnet/publish.yml index a8a9170a6..80d2f886b 100644 --- a/eng/common/templates/stages/dotnet/publish.yml +++ b/eng/common/templates/stages/dotnet/publish.yml @@ -10,6 +10,7 @@ parameters: customPublishInitSteps: [] sourceBuildPipelineDefinitionId: '' sourceBuildPipelineRunId: '' + versionsRepoRef: null stages: - template: /eng/common/templates/stages/publish.yml@self @@ -20,6 +21,7 @@ stages: isStandalonePublish: ${{ parameters.isStandalonePublish }} sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} customPublishInitSteps: - pwsh: | diff --git a/eng/common/templates/stages/publish.yml b/eng/common/templates/stages/publish.yml index e1a3ec9ca..816eb4b99 100644 --- a/eng/common/templates/stages/publish.yml +++ b/eng/common/templates/stages/publish.yml @@ -14,13 +14,18 @@ parameters: sourceBuildPipelineDefinitionId: '' sourceBuildPipelineRunId: '' + versionsRepoRef: null + versionsRepoPath: "versions" + ################################################################################ # Publish Images ################################################################################ stages: - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - stage: Publish - ${{ if not(parameters.isStandalonePublish) }}: + ${{ if eq(parameters.isStandalonePublish, true) }}: + dependsOn: [] + ${{ else }}: ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: dependsOn: Test ${{ else }}: @@ -63,3 +68,5 @@ stages: customInitSteps: ${{ parameters.customPublishInitSteps }} sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + versionsRepoPath: ${{ parameters.versionsRepoPath }} diff --git a/eng/common/templates/steps/init-matrix-build-publish.yml b/eng/common/templates/steps/init-matrix-build-publish.yml new file mode 100644 index 000000000..95408c66d --- /dev/null +++ b/eng/common/templates/steps/init-matrix-build-publish.yml @@ -0,0 +1,77 @@ +# Initialize common variables used in +# - Generating build matrix +# - Building images +# - Running tests +# - Publishing images + +parameters: + versionsRepoRef: "" + versionsRepoPath: "versions" + +steps: +- checkout: self +- ${{ if ne(parameters.versionsRepoRef, '') }}: + - checkout: ${{ parameters.versionsRepoRef }} + path: s/${{ parameters.versionsRepoPath }} + persistCredentials: true + fetchDepth: 1 + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) +- powershell: | + $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" + if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix $(mirrorRepoPrefix) --registry-override $(acr-staging.server)" + } + + if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" + } + + if ("${{ parameters.versionsRepoRef }}" -ne "") { + $versionsBasePath = "${{ parameters.versionsRepoPath }}/" + $pipelineDisabledCache = "false" + + $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") + if ($pathSeparatorIndex -ge 0) { + $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) + } + else { + $buildRepoName = "$(Build.Repository.Name)" + } + + $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)" + + $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" + $manifest = "$buildRepoName/$(manifest)" + $testResultsDirectory = "$buildRepoName/$testResultsDirectory" + + if ("$(testScriptPath)") { + $testScriptPath = "$buildRepoName/$(testScriptPath)" + } + + echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" + echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath" + echo "##vso[task.setvariable variable=manifest]$manifest" + echo "##vso[task.setvariable variable=engPath]$engPath" + echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" + echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" + } + else { + $versionsBasePath = "" + $pipelineDisabledCache = "true" + } + + echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" + echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" + echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" + displayName: Set Common Variables for Matrix, Build, and Publish + +- ${{ if ne(parameters.versionsRepoRef, '') }}: + # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT. + # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the + # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path. + - task: CopyFiles@2 + displayName: Copy TSA Config + inputs: + SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' + Contents: '.config/tsaoptions.json' + TargetFolder: '$(Build.SourcesDirectory)' diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 83141ef47..400e7ad73 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2744543 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2749906 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner From 2e7545d5269504c8ca1d7be134cfef9916dba921 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Mon, 21 Jul 2025 14:07:18 -0500 Subject: [PATCH 2/5] update repo ref --- eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml | 4 +++- eng/pipelines/dotnet-buildtools-prereqs-all.yml | 6 ++++-- eng/pipelines/dotnet-buildtools-prereqs-eng.yml | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml b/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml index ba314b718..5d8db82c6 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml @@ -9,10 +9,11 @@ pr: resources: repositories: - - repository: PublicVersionsRepo + - repository: VersionsRepo type: github endpoint: public name: dotnet/versions + ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} variables: - template: variables/common.yml @@ -22,6 +23,7 @@ stages: parameters: internalProjectName: ${{ variables.internalProjectName }} publicProjectName: ${{ variables.publicProjectName }} + versionsRepoRef: VersionsRepo linuxAmdBuildJobTimeout: 480 linuxArmBuildJobTimeout: 300 linuxAmd64Pool: diff --git a/eng/pipelines/dotnet-buildtools-prereqs-all.yml b/eng/pipelines/dotnet-buildtools-prereqs-all.yml index 2bc4493a5..690c80a34 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-all.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-all.yml @@ -32,10 +32,11 @@ schedules: resources: repositories: - - repository: InternalVersionsRepo + - repository: VersionsRepo type: github - endpoint: dotnet + endpoint: public name: dotnet/versions + ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} variables: - template: /eng/pipelines/variables/common.yml@self @@ -58,6 +59,7 @@ extends: parameters: internalProjectName: ${{ variables.internalProjectName }} publicProjectName: ${{ variables.publicProjectName }} + versionsRepoRef: VersionsRepo linuxAmdBuildJobTimeout: 480 linuxArmBuildJobTimeout: 300 linuxAmd64Pool: diff --git a/eng/pipelines/dotnet-buildtools-prereqs-eng.yml b/eng/pipelines/dotnet-buildtools-prereqs-eng.yml index 4864191c8..650bb721c 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-eng.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-eng.yml @@ -13,10 +13,11 @@ pr: resources: repositories: - - repository: PublicVersionsRepo + - repository: VersionsRepo type: github endpoint: public name: dotnet/versions + ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} variables: - template: /eng/pipelines/variables/common.yml@self @@ -32,6 +33,7 @@ stages: parameters: internalProjectName: ${{ variables.internalProjectName }} publicProjectName: ${{ variables.publicProjectName }} + versionsRepoRef: VersionsRepo linuxAmdBuildJobTimeout: 360 linuxArmBuildJobTimeout: 300 customBuildInitSteps: From 9e2a36a13c0b90c4cd2bd55699eea4f35a5e43b0 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Mon, 21 Jul 2025 14:13:19 -0500 Subject: [PATCH 3/5] move position --- .../dotnet-buildtools-prereqs-all-pr.yml | 6 +++--- eng/pipelines/dotnet-buildtools-prereqs-all.yml | 16 ++++++++-------- eng/pipelines/dotnet-buildtools-prereqs-eng.yml | 16 ++++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml b/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml index 5d8db82c6..a2cd49c63 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml @@ -7,6 +7,9 @@ pr: include: - src/* +variables: +- template: variables/common.yml + resources: repositories: - repository: VersionsRepo @@ -15,9 +18,6 @@ resources: name: dotnet/versions ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} -variables: -- template: variables/common.yml - stages: - template: stages/build-test-publish-repo.yml parameters: diff --git a/eng/pipelines/dotnet-buildtools-prereqs-all.yml b/eng/pipelines/dotnet-buildtools-prereqs-all.yml index 690c80a34..d495fcf0e 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-all.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-all.yml @@ -30,14 +30,6 @@ schedules: - main always: true -resources: - repositories: - - repository: VersionsRepo - type: github - endpoint: public - name: dotnet/versions - ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} - variables: - template: /eng/pipelines/variables/common.yml@self parameters: @@ -46,6 +38,14 @@ variables: - name: publishEolAnnotations value: true +resources: + repositories: + - repository: VersionsRepo + type: github + endpoint: public + name: dotnet/versions + ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} + extends: template: /eng/common/templates/1es-official.yml@self parameters: diff --git a/eng/pipelines/dotnet-buildtools-prereqs-eng.yml b/eng/pipelines/dotnet-buildtools-prereqs-eng.yml index 650bb721c..8a1109877 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-eng.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-eng.yml @@ -11,14 +11,6 @@ pr: - CODEOWNERS - LICENSE -resources: - repositories: - - repository: VersionsRepo - type: github - endpoint: public - name: dotnet/versions - ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} - variables: - template: /eng/pipelines/variables/common.yml@self - name: imageBuilder.pathArgs @@ -28,6 +20,14 @@ variables: - name: trimCachedImagesForMatrix value: false +resources: + repositories: + - repository: VersionsRepo + type: github + endpoint: public + name: dotnet/versions + ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} + stages: - template: /eng/pipelines/stages/build-test-publish-repo.yml@self parameters: From 845a7aab64904311811772baf6f7e453fd43026a Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Mon, 21 Jul 2025 14:41:30 -0500 Subject: [PATCH 4/5] Delete unused files --- .../stages/build-test-publish-repo.yml | 378 ------------------ .../common-init-for-matrix-and-build.yml | 74 ---- 2 files changed, 452 deletions(-) delete mode 100644 eng/common/templates/stages/build-test-publish-repo.yml delete mode 100644 eng/common/templates/steps/common-init-for-matrix-and-build.yml diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml deleted file mode 100644 index aa29b2ba1..000000000 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ /dev/null @@ -1,378 +0,0 @@ -parameters: - buildMatrixType: platformDependencyGraph - testMatrixType: platformVersionedOs - buildMatrixCustomBuildLegGroupArgs: "" - testMatrixCustomBuildLegGroupArgs: "" - customCopyBaseImagesInitSteps: [] - customGenerateMatrixInitSteps: [] - customBuildInitSteps: [] - customTestInitSteps: [] - customPublishInitSteps: [] - customPublishVariables: [] - - linuxAmdBuildJobTimeout: 60 - linuxArmBuildJobTimeout: 60 - windowsAmdBuildJobTimeout: 60 - - linuxAmdTestJobTimeout: 60 - linuxArmTestJobTimeout: 60 - windowsAmdTestJobTimeout: 60 - - noCache: false - - internalProjectName: null - publicProjectName: null - - internalVersionsRepoRef: null - publicVersionsRepoRef: null - - isInternalServicingValidation: false - - linuxAmd64Pool: - vmImage: $(defaultLinuxAmd64PoolImage) - linuxArm32Pool: - vmImage: $(defaultLinuxArm32PoolImage) - linuxArm64Pool: - vmImage: $(defaultLinuxArm64PoolImage) - windows2016Pool: - vmImage: $(defaultWindows2016PoolImage) - windows1809Pool: - vmImage: $(defaultWindows1809PoolImage) - windows2022Pool: - vmImage: $(defaultWindows2022PoolImage) - windows2025Pool: - vmImage: $(defaultWindows2025PoolImage) - -stages: - -################################################################################ -# Build Images -################################################################################ -- stage: Build - condition: and(succeeded(), contains(variables['stages'], 'build')) - jobs: - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: PreBuildValidation - pool: ${{ parameters.linuxAmd64Pool }} - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - preBuildValidation: true - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables - - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self - parameters: - name: CopyBaseImages - pool: ${{ parameters.linuxAmd64Pool }} - additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" - customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} - - template: /eng/common/templates/jobs/generate-matrix.yml@self - parameters: - matrixType: ${{ parameters.buildMatrixType }} - name: GenerateBuildMatrix - pool: ${{ parameters.linuxAmd64Pool }} - customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} - internalProjectName: ${{ parameters.internalProjectName }} - noCache: ${{ parameters.noCache }} - customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_amd64 - pool: ${{ parameters.linuxAmd64Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_arm64 - pool: ${{ parameters.linuxArm64Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_arm32 - pool: ${{ parameters.linuxArm32Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows1809_amd64 - pool: ${{ parameters.windows1809Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows2022_amd64 - pool: ${{ parameters.windows2022Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows2025_amd64 - pool: ${{ parameters.windows2025Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: WindowsLtsc2016_amd64 - pool: ${{ parameters.windows2016Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - -################################################################################ -# Post-Build -################################################################################ -- stage: Post_Build - dependsOn: Build - condition: and(succeeded(), contains(variables['stages'], 'build')) - jobs: - - template: /eng/common/templates/jobs/post-build.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - -################################################################################ -# Test Images -################################################################################ -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - - stage: Test - dependsOn: Post_Build - condition: " - and( - ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), - ne(variables['testScriptPath'], ''), - and( - contains(variables['stages'], 'test'), - or( - and( - succeeded(), - contains(variables['stages'], 'build')), - not(contains(variables['stages'], 'build')))))" - jobs: - - template: /eng/common/templates/jobs/generate-matrix.yml@self - parameters: - matrixType: ${{ parameters.testMatrixType }} - name: GenerateTestMatrix - pool: ${{ parameters.linuxAmd64Pool }} - customBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} - isTestStage: true - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_amd64 - pool: ${{ parameters.linuxAmd64Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_arm64 - pool: ${{ parameters.linuxArm64Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] - testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_arm32 - pool: ${{ parameters.linuxArm32Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] - testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows1809_amd64 - pool: ${{ parameters.windows1809Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows2022_amd64 - pool: ${{ parameters.windows2022Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows2025_amd64 - pool: ${{ parameters.windows2025Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: WindowsLtsc2016_amd64 - pool: ${{ parameters.windows2016Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - -################################################################################ -# Publish Images -################################################################################ -- ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - - stage: Publish - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build - condition: " - and( - not(canceled()), - and( - contains(variables['stages'], 'publish'), - or( - or( - and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - or( - and( - not(contains(variables['stages'], 'build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - and( - not(contains(variables['stages'], 'test')), - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" - jobs: - - template: /eng/common/templates/jobs/publish.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - internalProjectName: ${{ parameters.internalProjectName }} - customPublishVariables: ${{ parameters.customPublishVariables }} - customInitSteps: ${{ parameters.customPublishInitSteps }} diff --git a/eng/common/templates/steps/common-init-for-matrix-and-build.yml b/eng/common/templates/steps/common-init-for-matrix-and-build.yml deleted file mode 100644 index ce34eaf26..000000000 --- a/eng/common/templates/steps/common-init-for-matrix-and-build.yml +++ /dev/null @@ -1,74 +0,0 @@ -parameters: - noCache: false - internalVersionsRepoRef: null - publicVersionsRepoRef: null - versionsRepoPath: versions - isInternalServicingValidation: false - -steps: -- checkout: self -- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(parameters.noCache, false)) }}: - - checkout: ${{ parameters.publicVersionsRepoRef }} - path: s/${{ parameters.versionsRepoPath }} -- ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.noCache, false)) }}: - - checkout: ${{ parameters.internalVersionsRepoRef }} - path: s/${{ parameters.versionsRepoPath }} -- powershell: | - $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" - if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix $(mirrorRepoPrefix) --registry-override $(acr-staging.server)" - } - - if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" - } - - if ("${{ parameters.noCache }}" -eq "false") { - $versionsBasePath = "${{ parameters.versionsRepoPath }}/" - $pipelineDisabledCache = "false" - - $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") - if ($pathSeparatorIndex -ge 0) { - $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) - } - else { - $buildRepoName = "$(Build.Repository.Name)" - } - - $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)" - - $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" - $manifest = "$buildRepoName/$(manifest)" - $testResultsDirectory = "$buildRepoName/$testResultsDirectory" - - if ("$(testScriptPath)") { - $testScriptPath = "$buildRepoName/$(testScriptPath)" - } - - echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" - echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath" - echo "##vso[task.setvariable variable=manifest]$manifest" - echo "##vso[task.setvariable variable=engPath]$engPath" - echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" - echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" - } - else { - $versionsBasePath = "" - $pipelineDisabledCache = "true" - } - - echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" - echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" - echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" - displayName: Set Common Variables for Matrix and Build - -- ${{ if eq(parameters.noCache, false) }}: - # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT. - # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the - # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path. - - task: CopyFiles@2 - displayName: Copy TSA Config - inputs: - SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' - Contents: '.config/tsaoptions.json' - TargetFolder: '$(Build.SourcesDirectory)' From b6e17a2bab96473705c71711ccec1b3976f639d6 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Mon, 21 Jul 2025 14:43:10 -0500 Subject: [PATCH 5/5] add repo ref param --- eng/pipelines/stages/build-test-publish-repo.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/stages/build-test-publish-repo.yml b/eng/pipelines/stages/build-test-publish-repo.yml index 0dc49b450..fe78c377d 100644 --- a/eng/pipelines/stages/build-test-publish-repo.yml +++ b/eng/pipelines/stages/build-test-publish-repo.yml @@ -7,6 +7,7 @@ parameters: customBuildInitSteps: [] customCopyBaseImagesInitSteps: [] linuxAmd64Pool: '' + versionsRepoRef: null stages: - template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self @@ -14,6 +15,7 @@ stages: noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} ${{ if and(eq(variables['System.TeamProject'], parameters.publicProjectName), eq(parameters.linuxAmd64Pool, ''))}}: linuxAmd64Pool: