diff --git a/NuGet.config b/NuGet.config
index e2087a016db2..1c2f27eb90ce 100644
--- a/NuGet.config
+++ b/NuGet.config
@@ -6,14 +6,8 @@
-
-
-
-
-
-
@@ -34,14 +28,8 @@
-
-
-
-
-
-
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index ee11ee5e72e9..ebd02201a1bf 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -376,26 +376,26 @@
https://github.com/dotnet/winforms
abda8e3bfa78319363526b5a5f86863ec979940e
-
+
https://github.com/dotnet/arcade
- c7a36e53f91e41943746f97a8c183549364c5092
+ 4246a31e5de9de87f760218c4f588cebf4661f45
-
+
https://github.com/dotnet/arcade
- c7a36e53f91e41943746f97a8c183549364c5092
+ 4246a31e5de9de87f760218c4f588cebf4661f45
-
+
https://github.com/dotnet/arcade
- c7a36e53f91e41943746f97a8c183549364c5092
+ 4246a31e5de9de87f760218c4f588cebf4661f45
-
+
https://github.com/dotnet/arcade
- c7a36e53f91e41943746f97a8c183549364c5092
+ 4246a31e5de9de87f760218c4f588cebf4661f45
-
+
https://github.com/dotnet/arcade
- c7a36e53f91e41943746f97a8c183549364c5092
+ 4246a31e5de9de87f760218c4f588cebf4661f45
https://github.com/dotnet/extensions
diff --git a/eng/Versions.props b/eng/Versions.props
index dbfe6df117bd..c7c52acfe13c 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -162,9 +162,9 @@
6.2.4
6.2.4
- 8.0.0-beta.25208.7
- 8.0.0-beta.25208.7
- 8.0.0-beta.25208.7
+ 8.0.0-beta.25230.1
+ 8.0.0-beta.25230.1
+ 8.0.0-beta.25230.1
8.0.0-alpha.1.25202.2
diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml
index f983033bb028..4217d6d8b148 100644
--- a/eng/common/templates-official/job/source-build.yml
+++ b/eng/common/templates-official/job/source-build.yml
@@ -54,7 +54,7 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
- demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
+ demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml
index 60dfb6b2d1c0..fb632b71a250 100644
--- a/eng/common/templates-official/job/source-index-stage1.yml
+++ b/eng/common/templates-official/job/source-index-stage1.yml
@@ -1,7 +1,7 @@
parameters:
runAsPublic: false
- sourceIndexUploadPackageVersion: 2.0.0-20240502.12
- sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
+ sourceIndexUploadPackageVersion: 2.0.0-20250425.2
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml
index 5cf6a269c0b6..b9247be1547b 100644
--- a/eng/common/templates-official/jobs/source-build.yml
+++ b/eng/common/templates-official/jobs/source-build.yml
@@ -14,7 +14,7 @@ parameters:
# This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
- container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'. If no platforms are specified,
diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml
index 503164fa7f22..829f17c34d11 100644
--- a/eng/common/templates-official/steps/source-build.yml
+++ b/eng/common/templates-official/steps/source-build.yml
@@ -44,7 +44,7 @@ steps:
# in the default public locations.
internalRuntimeDownloadArgs=
if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then
- internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
+ internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
fi
buildConfig=Release
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
index c0ff472b697b..c48f95d93d91 100644
--- a/eng/common/templates/job/source-build.yml
+++ b/eng/common/templates/job/source-build.yml
@@ -54,11 +54,11 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
- demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
+ demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
- demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
+ demands: ImageOverride -equals Build.Ubuntu.2204.Amd64
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
index 0b6bb89dc78a..8538f44bab28 100644
--- a/eng/common/templates/job/source-index-stage1.yml
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -1,7 +1,7 @@
parameters:
runAsPublic: false
- sourceIndexUploadPackageVersion: 2.0.0-20240502.12
- sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
+ sourceIndexUploadPackageVersion: 2.0.0-20250425.2
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
index 5f46bfa895c1..3ec997108107 100644
--- a/eng/common/templates/jobs/source-build.yml
+++ b/eng/common/templates/jobs/source-build.yml
@@ -14,7 +14,7 @@ parameters:
# This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
- container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'. If no platforms are specified,
diff --git a/global.json b/global.json
index a7cd171707cd..2d7bd87b2e5f 100644
--- a/global.json
+++ b/global.json
@@ -25,8 +25,8 @@
},
"msbuild-sdks": {
"Yarn.MSBuild": "1.22.19",
- "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25208.7",
- "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25208.7"
+ "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25230.1",
+ "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25230.1"
},
"native-tools": {
"jdk": "latest"
diff --git a/src/arcade/eng/common/BuildConfiguration/build-configuration.json b/src/arcade/eng/common/BuildConfiguration/build-configuration.json
new file mode 100644
index 000000000000..3d1cc89894c9
--- /dev/null
+++ b/src/arcade/eng/common/BuildConfiguration/build-configuration.json
@@ -0,0 +1,4 @@
+{
+ "RetryCountLimit": 1,
+ "RetryByAnyError": false
+}
diff --git a/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1
new file mode 100644
index 000000000000..4c1ea7c98ea4
--- /dev/null
+++ b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1
@@ -0,0 +1,11 @@
+@{
+ IncludeRules=@('PSAvoidUsingCmdletAliases',
+ 'PSAvoidUsingWMICmdlet',
+ 'PSAvoidUsingPositionalParameters',
+ 'PSAvoidUsingInvokeExpression',
+ 'PSUseDeclaredVarsMoreThanAssignments',
+ 'PSUseCmdletCorrectly',
+ 'PSStandardDSCFunctionsInResource',
+ 'PSUseIdenticalMandatoryParametersForDSC',
+ 'PSUseIdenticalParametersForDSC')
+}
\ No newline at end of file
diff --git a/src/arcade/eng/common/README.md b/src/arcade/eng/common/README.md
new file mode 100644
index 000000000000..ff49c371527a
--- /dev/null
+++ b/src/arcade/eng/common/README.md
@@ -0,0 +1,28 @@
+# Don't touch this folder
+
+ uuuuuuuuuuuuuuuuuuuu
+ u" uuuuuuuuuuuuuuuuuu "u
+ u" u$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+ $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+ $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+ $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $
+ $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $
+ $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $
+ $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $
+ $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $
+ $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$$$$$" u"
+ "u "$$$$$$$$$$$$$$$$$$$$" u"
+ "u """""""""""""""""" u"
+ """"""""""""""""""""
+
+!!! Changes made in this directory are subject to being overwritten by automation !!!
+
+The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first.
diff --git a/src/arcade/eng/common/cross/arm/tizen/tizen.patch b/src/arcade/eng/common/cross/arm/tizen/tizen.patch
new file mode 100644
index 000000000000..fb12ade7250a
--- /dev/null
+++ b/src/arcade/eng/common/cross/arm/tizen/tizen.patch
@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+ Use the shared library, but some functions are only in
+ the static library, so try that secondarily. */
+ OUTPUT_FORMAT(elf32-littlearm)
+-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) )
++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) )
diff --git a/src/arcade/eng/common/cross/armel/tizen/tizen.patch b/src/arcade/eng/common/cross/armel/tizen/tizen.patch
new file mode 100644
index 000000000000..ca7c7c1ff751
--- /dev/null
+++ b/src/arcade/eng/common/cross/armel/tizen/tizen.patch
@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+ Use the shared library, but some functions are only in
+ the static library, so try that secondarily. */
+ OUTPUT_FORMAT(elf32-littlearm)
+-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) )
++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) )
diff --git a/src/arcade/eng/common/cross/x64/tizen/tizen.patch b/src/arcade/eng/common/cross/x64/tizen/tizen.patch
new file mode 100644
index 000000000000..56fbc881095b
--- /dev/null
+++ b/src/arcade/eng/common/cross/x64/tizen/tizen.patch
@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib64/libc.so b/usr/lib64/libc.so
+--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+ Use the shared library, but some functions are only in
+ the static library, so try that secondarily. */
+ OUTPUT_FORMAT(elf64-x86-64)
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) )
++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-x86-64.so.2 ) )
diff --git a/src/arcade/eng/common/cross/x86/tizen/tizen.patch b/src/arcade/eng/common/cross/x86/tizen/tizen.patch
new file mode 100644
index 000000000000..f4fe8838ad66
--- /dev/null
+++ b/src/arcade/eng/common/cross/x86/tizen/tizen.patch
@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+ Use the shared library, but some functions are only in
+ the static library, so try that secondarily. */
+ OUTPUT_FORMAT(elf32-i386)
+-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.2 ) )
++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.2 ) )
diff --git a/src/arcade/eng/common/dotnet-install.cmd b/src/arcade/eng/common/dotnet-install.cmd
new file mode 100644
index 000000000000..b1c2642e76f7
--- /dev/null
+++ b/src/arcade/eng/common/dotnet-install.cmd
@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*"
\ No newline at end of file
diff --git a/src/arcade/eng/common/dotnet-install.ps1 b/src/arcade/eng/common/dotnet-install.ps1
new file mode 100644
index 000000000000..811f0f717f73
--- /dev/null
+++ b/src/arcade/eng/common/dotnet-install.ps1
@@ -0,0 +1,28 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string] $verbosity = 'minimal',
+ [string] $architecture = '',
+ [string] $version = 'Latest',
+ [string] $runtime = 'dotnet',
+ [string] $RuntimeSourceFeed = '',
+ [string] $RuntimeSourceFeedKey = ''
+)
+
+. $PSScriptRoot\tools.ps1
+
+$dotnetRoot = Join-Path $RepoRoot '.dotnet'
+
+$installdir = $dotnetRoot
+try {
+ if ($architecture -and $architecture.Trim() -eq 'x86') {
+ $installdir = Join-Path $installdir 'x86'
+ }
+ InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ ExitWithExitCode 1
+}
+
+ExitWithExitCode 0
diff --git a/src/arcade/eng/common/enable-cross-org-publishing.ps1 b/src/arcade/eng/common/enable-cross-org-publishing.ps1
new file mode 100644
index 000000000000..da09da4f1fc4
--- /dev/null
+++ b/src/arcade/eng/common/enable-cross-org-publishing.ps1
@@ -0,0 +1,13 @@
+param(
+ [string] $token
+)
+
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+# Write-PipelineSetVariable will no-op if a variable named $ci is not defined
+# Since this script is only ever called in AzDO builds, just universally set it
+$ci = $true
+
+Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false
diff --git a/src/arcade/eng/common/generate-locproject.ps1 b/src/arcade/eng/common/generate-locproject.ps1
new file mode 100644
index 000000000000..524aaa57f2b7
--- /dev/null
+++ b/src/arcade/eng/common/generate-locproject.ps1
@@ -0,0 +1,189 @@
+Param(
+ [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here
+ [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json
+ [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
+ [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally
+)
+
+# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
+# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
+
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
+$exclusions = @{ Exclusions = @() }
+if (Test-Path -Path $exclusionsFilePath)
+{
+ $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
+}
+
+Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
+
+# Template files
+$jsonFiles = @()
+$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
+$jsonTemplateFiles | ForEach-Object {
+ $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
+
+ $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
+ $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+
+$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
+if (-not $wxlFiles) {
+ $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files
+ if ($wxlEnFiles) {
+ $wxlFiles = @()
+ $wxlEnFiles | ForEach-Object {
+ $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+ $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+ }
+ }
+}
+
+$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files
+$macosHtmlFiles = @()
+if ($macosHtmlEnFiles) {
+ $macosHtmlEnFiles | ForEach-Object {
+ $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+ $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+ }
+}
+
+$xlfFiles = @()
+
+$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
+$langXlfFiles = @()
+if ($allXlfFiles) {
+ $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
+ $firstLangCode = $Matches.1
+ $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
+}
+$langXlfFiles | ForEach-Object {
+ $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
+
+ $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
+ $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles
+
+$locJson = @{
+ Projects = @(
+ @{
+ LanguageSet = $LanguageSet
+ LocItems = @(
+ $locFiles | ForEach-Object {
+ $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
+ $continue = $true
+ foreach ($exclusion in $exclusions.Exclusions) {
+ if ($_.FullName.Contains($exclusion))
+ {
+ $continue = $false
+ }
+ }
+ $sourceFile = ($_.FullName | Resolve-Path -Relative)
+ if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
+ Remove-Item -Path $sourceFile
+ }
+ if ($continue)
+ {
+ if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnPath"
+ OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
+ }
+ } else {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnName"
+ OutputPath = $outputPath
+ }
+ }
+ }
+ }
+ )
+ },
+ @{
+ LanguageSet = $LanguageSet
+ CloneLanguageSet = "WiX_CloneLanguages"
+ LssFiles = @( "wxl_loc.lss" )
+ LocItems = @(
+ $wxlFiles | ForEach-Object {
+ $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+ $continue = $true
+ foreach ($exclusion in $exclusions.Exclusions) {
+ if ($_.FullName.Contains($exclusion)) {
+ $continue = $false
+ }
+ }
+ $sourceFile = ($_.FullName | Resolve-Path -Relative)
+ if ($continue)
+ {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnPath"
+ OutputPath = $outputPath
+ }
+ }
+ }
+ )
+ },
+ @{
+ LanguageSet = $LanguageSet
+ CloneLanguageSet = "VS_macOS_CloneLanguages"
+ LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" )
+ LocItems = @(
+ $macosHtmlFiles | ForEach-Object {
+ $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+ $continue = $true
+ foreach ($exclusion in $exclusions.Exclusions) {
+ if ($_.FullName.Contains($exclusion)) {
+ $continue = $false
+ }
+ }
+ $sourceFile = ($_.FullName | Resolve-Path -Relative)
+ $lciFile = $sourceFile + ".lci"
+ if ($continue) {
+ $result = @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnPath"
+ OutputPath = $outputPath
+ }
+ if (Test-Path $lciFile -PathType Leaf) {
+ $result["LciFile"] = $lciFile
+ }
+ return $result
+ }
+ }
+ )
+ }
+ )
+}
+
+$json = ConvertTo-Json $locJson -Depth 5
+Write-Host "LocProject.json generated:`n`n$json`n`n"
+Pop-Location
+
+if (!$UseCheckedInLocProjectJson) {
+ New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+ Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json
+}
+else {
+ New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+ Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json
+
+ if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
+ Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
+
+ exit 1
+ }
+ else {
+ Write-Host "Generated LocProject.json and current LocProject.json are identical."
+ }
+}
diff --git a/src/arcade/eng/common/generate-sbom-prep.ps1 b/src/arcade/eng/common/generate-sbom-prep.ps1
new file mode 100644
index 000000000000..a0c7d792a76f
--- /dev/null
+++ b/src/arcade/eng/common/generate-sbom-prep.ps1
@@ -0,0 +1,29 @@
+Param(
+ [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed
+)
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM"
+$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_'
+$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName
+
+Write-Host "Artifact name before : $ArtifactName"
+Write-Host "Artifact name after : $SafeArtifactName"
+
+Write-Host "Creating dir $ManifestDirPath"
+
+# create directory for sbom manifest to be placed
+if (!(Test-Path -path $SbomGenerationDir))
+{
+ New-Item -ItemType Directory -path $SbomGenerationDir
+ Write-Host "Successfully created directory $SbomGenerationDir"
+}
+else{
+ Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
+}
+
+Write-Host "Updating artifact name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName"
diff --git a/src/arcade/eng/common/init-tools-native.ps1 b/src/arcade/eng/common/init-tools-native.ps1
new file mode 100644
index 000000000000..27ccdb9ecc95
--- /dev/null
+++ b/src/arcade/eng/common/init-tools-native.ps1
@@ -0,0 +1,203 @@
+<#
+.SYNOPSIS
+Entry point script for installing native tools
+
+.DESCRIPTION
+Reads $RepoRoot\global.json file to determine native assets to install
+and executes installers for those tools
+
+.PARAMETER BaseUri
+Base file directory or Url from which to acquire tool archives
+
+.PARAMETER InstallDirectory
+Directory to install native toolset. This is a command-line override for the default
+Install directory precedence order:
+- InstallDirectory command-line override
+- NETCOREENG_INSTALL_DIRECTORY environment variable
+- (default) %USERPROFILE%/.netcoreeng/native
+
+.PARAMETER Clean
+Switch specifying to not install anything, but cleanup native asset folders
+
+.PARAMETER Force
+Clean and then install tools
+
+.PARAMETER DownloadRetries
+Total number of retry attempts
+
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds
+
+.PARAMETER GlobalJsonFile
+File path to global.json file
+
+.PARAMETER PathPromotion
+Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines)
+or break the build if a native tool is not found on the path (on a local dev machine)
+
+.NOTES
+#>
+[CmdletBinding(PositionalBinding=$false)]
+Param (
+ [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external',
+ [string] $InstallDirectory,
+ [switch] $Clean = $False,
+ [switch] $Force = $False,
+ [int] $DownloadRetries = 5,
+ [int] $RetryWaitTimeInSeconds = 30,
+ [string] $GlobalJsonFile,
+ [switch] $PathPromotion
+)
+
+if (!$GlobalJsonFile) {
+ $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json'
+}
+
+Set-StrictMode -version 2.0
+$ErrorActionPreference='Stop'
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+try {
+ # Define verbose switch if undefined
+ $Verbose = $VerbosePreference -Eq 'Continue'
+
+ $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\'
+ $NativeBaseDir = $InstallDirectory
+ if (!$NativeBaseDir) {
+ $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory
+ }
+ $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir
+ $InstallBin = Join-Path $NativeBaseDir 'bin'
+ $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1'
+
+ # Process tools list
+ Write-Host "Processing $GlobalJsonFile"
+ If (-Not (Test-Path $GlobalJsonFile)) {
+ Write-Host "Unable to find '$GlobalJsonFile'"
+ exit 0
+ }
+ $NativeTools = Get-Content($GlobalJsonFile) -Raw |
+ ConvertFrom-Json |
+ Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
+ if ($NativeTools) {
+ if ($PathPromotion -eq $True) {
+ $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools"
+ if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine
+ $NativeTools.PSObject.Properties | ForEach-Object {
+ $ToolName = $_.Name
+ $ToolVersion = $_.Value
+ $InstalledTools = @{}
+
+ if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
+ if ($ToolVersion -eq "latest") {
+ $ToolVersion = ""
+ }
+ $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)
+ if ($ToolDirectories -eq $null) {
+ Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image."
+ exit 1
+ }
+ $ToolDirectory = $ToolDirectories[0]
+ $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt"
+ if (-not (Test-Path -Path "$BinPathFile")) {
+ Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool."
+ exit 1
+ }
+ $BinPath = Get-Content "$BinPathFile"
+ $ToolPath = Convert-Path -Path $BinPath
+ Write-Host "Adding $ToolName to the path ($ToolPath)..."
+ Write-Host "##vso[task.prependpath]$ToolPath"
+ $env:PATH = "$ToolPath;$env:PATH"
+ $InstalledTools += @{ $ToolName = $ToolDirectory.FullName }
+ }
+ }
+ return $InstalledTools
+ } else {
+ $NativeTools.PSObject.Properties | ForEach-Object {
+ $ToolName = $_.Name
+ $ToolVersion = $_.Value
+
+ if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
+ Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding."
+ Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image."
+ }
+ }
+ exit 0
+ }
+ } else {
+ $NativeTools.PSObject.Properties | ForEach-Object {
+ $ToolName = $_.Name
+ $ToolVersion = $_.Value
+ $LocalInstallerArguments = @{ ToolName = "$ToolName" }
+ $LocalInstallerArguments += @{ InstallPath = "$InstallBin" }
+ $LocalInstallerArguments += @{ BaseUri = "$BaseUri" }
+ $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" }
+ $LocalInstallerArguments += @{ Version = "$ToolVersion" }
+
+ if ($Verbose) {
+ $LocalInstallerArguments += @{ Verbose = $True }
+ }
+ if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
+ if($Force) {
+ $LocalInstallerArguments += @{ Force = $True }
+ }
+ }
+ if ($Clean) {
+ $LocalInstallerArguments += @{ Clean = $True }
+ }
+
+ Write-Verbose "Installing $ToolName version $ToolVersion"
+ Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
+ & $InstallerPath @LocalInstallerArguments
+ if ($LASTEXITCODE -Ne "0") {
+ $errMsg = "$ToolName installation failed"
+ if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
+ $showNativeToolsWarning = $true
+ if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+ $showNativeToolsWarning = $false
+ }
+ if ($showNativeToolsWarning) {
+ Write-Warning $errMsg
+ }
+ $toolInstallationFailure = $true
+ } else {
+ # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+ Write-Host $errMsg
+ exit 1
+ }
+ }
+ }
+
+ if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
+ # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+ Write-Host 'Native tools bootstrap failed'
+ exit 1
+ }
+ }
+ }
+ else {
+ Write-Host 'No native tools defined in global.json'
+ exit 0
+ }
+
+ if ($Clean) {
+ exit 0
+ }
+ if (Test-Path $InstallBin) {
+ Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin)
+ Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
+ return $InstallBin
+ }
+ elseif (-not ($PathPromotion)) {
+ Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed'
+ exit 1
+ }
+ exit 0
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/src/arcade/eng/common/init-tools-native.sh b/src/arcade/eng/common/init-tools-native.sh
new file mode 100644
index 000000000000..3e6a8d6acf2f
--- /dev/null
+++ b/src/arcade/eng/common/init-tools-native.sh
@@ -0,0 +1,238 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
+install_directory=''
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
+declare -a native_assets
+
+. $scriptroot/pipeline-logging-functions.sh
+. $scriptroot/native/common-library.sh
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --baseuri)
+ base_uri=$2
+ shift 2
+ ;;
+ --installdirectory)
+ install_directory=$2
+ shift 2
+ ;;
+ --clean)
+ clean=true
+ shift 1
+ ;;
+ --force)
+ force=true
+ shift 1
+ ;;
+ --donotabortonfailure)
+ donotabortonfailure=true
+ shift 1
+ ;;
+ --donotdisplaywarnings)
+ donotdisplaywarnings=true
+ shift 1
+ ;;
+ --downloadretries)
+ download_retries=$2
+ shift 2
+ ;;
+ --retrywaittimeseconds)
+ retry_wait_time_seconds=$2
+ shift 2
+ ;;
+ --help)
+ echo "Common settings:"
+ echo " --installdirectory Directory to install native toolset."
+ echo " This is a command-line override for the default"
+ echo " Install directory precedence order:"
+ echo " - InstallDirectory command-line override"
+ echo " - NETCOREENG_INSTALL_DIRECTORY environment variable"
+ echo " - (default) %USERPROFILE%/.netcoreeng/native"
+ echo ""
+ echo " --clean Switch specifying not to install anything, but cleanup native asset folders"
+ echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure"
+ echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure"
+ echo " --force Clean and then install tools"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Advanced settings:"
+ echo " --baseuri Base URI for where to download native tools from"
+ echo " --downloadretries Number of times a download should be attempted"
+ echo " --retrywaittimeseconds Wait time between download attempts"
+ echo ""
+ exit 0
+ ;;
+ esac
+done
+
+function ReadGlobalJsonNativeTools {
+ # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+ if command -v jq &> /dev/null; then
+
+ # jq: read each key/value pair under "native-tools" entry and emit:
+ # KEY="" VALUE=""
+ # followed by a null byte.
+ #
+ # bash: read line with null byte delimeter and push to array (for later `eval`uation).
+
+ while IFS= read -rd '' line; do
+ native_assets+=("$line")
+ done < <(jq -r '. |
+ select(has("native-tools")) |
+ ."native-tools" |
+ keys[] as $k |
+ @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+ return
+ fi
+
+ # Warning: falling back to manually parsing JSON, which is not recommended.
+
+ # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+ # It has been tested with several weird strings with escaped characters in entries (key and value)
+ # and results were compared with the output of jq(1) in binary representation using xxd(1);
+ # just before the assignment to 'native_assets' array (above and below).
+
+ # try to capture the section under "native-tools".
+ if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+ return
+ fi
+
+ section="${BASH_REMATCH[1]}"
+
+ parseStarted=0
+ possibleEnd=0
+ escaping=0
+ escaped=0
+ isKey=1
+
+ for (( i=0; i<${#section}; i++ )); do
+ char="${section:$i:1}"
+ if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+ if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+ escaping=1
+ elif ((escaping)) && ! ((escaped)); then
+ escaped=1
+ fi
+
+ if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+ parseStarted=1
+ possibleEnd=0
+ elif [[ "$char" == "'" ]]; then
+ token="$token'\\\''"
+ possibleEnd=0
+ elif ((escaping)) || [[ "$char" != "\"" ]]; then
+ token="$token$char"
+ possibleEnd=1
+ fi
+
+ if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+ # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+ # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+ printf -v token "'$token'"
+
+ if ((isKey)); then
+ KEY="$token"
+ isKey=0
+ else
+ line="KEY=$KEY VALUE=$token"
+ native_assets+=("$line")
+ isKey=1
+ fi
+
+ # reset for next token
+ parseStarted=0
+ token=
+ elif ((escaping)) && ((escaped)); then
+ escaping=0
+ escaped=0
+ fi
+ done
+}
+
+native_base_dir=$install_directory
+if [[ -z $install_directory ]]; then
+ native_base_dir=$(GetNativeInstallDirectory)
+fi
+
+install_bin="${native_base_dir}/bin"
+installed_any=false
+
+ReadGlobalJsonNativeTools
+
+if [[ ${#native_assets[@]} -eq 0 ]]; then
+ echo "No native tools defined in global.json"
+ exit 0;
+else
+ native_installer_dir="$scriptroot/native"
+ for index in "${!native_assets[@]}"; do
+ eval "${native_assets["$index"]}"
+
+ installer_path="$native_installer_dir/install-$KEY.sh"
+ installer_command="$installer_path"
+ installer_command+=" --baseuri $base_uri"
+ installer_command+=" --installpath $install_bin"
+ installer_command+=" --version $VALUE"
+ echo $installer_command
+
+ if [[ $force = true ]]; then
+ installer_command+=" --force"
+ fi
+
+ if [[ $clean = true ]]; then
+ installer_command+=" --clean"
+ fi
+
+ if [[ -a $installer_path ]]; then
+ $installer_command
+ if [[ $? != 0 ]]; then
+ if [[ $donotabortonfailure = true ]]; then
+ if [[ $donotdisplaywarnings != true ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
+ fi
+ else
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
+ exit 1
+ fi
+ else
+ $installed_any = true
+ fi
+ else
+ if [[ $donotabortonfailure == true ]]; then
+ if [[ $donotdisplaywarnings != true ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
+ fi
+ else
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
+ exit 1
+ fi
+ fi
+ done
+fi
+
+if [[ $clean = true ]]; then
+ exit 0
+fi
+
+if [[ -d $install_bin ]]; then
+ echo "Native tools are available from $install_bin"
+ echo "##vso[task.prependpath]$install_bin"
+else
+ if [[ $installed_any = true ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed"
+ exit 1
+ fi
+fi
+
+exit 0
diff --git a/src/arcade/eng/common/internal-feed-operations.ps1 b/src/arcade/eng/common/internal-feed-operations.ps1
new file mode 100644
index 000000000000..92b77347d990
--- /dev/null
+++ b/src/arcade/eng/common/internal-feed-operations.ps1
@@ -0,0 +1,132 @@
+param(
+ [Parameter(Mandatory=$true)][string] $Operation,
+ [string] $AuthToken,
+ [string] $CommitSha,
+ [string] $RepoName,
+ [switch] $IsFeedPrivate
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+. $PSScriptRoot\tools.ps1
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified
+# internal builds
+function SetupCredProvider {
+ param(
+ [string] $AuthToken
+ )
+
+ # Install the Cred Provider NuGet plugin
+ Write-Host 'Setting up Cred Provider NuGet plugin in the agent...'
+ Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+ $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
+
+ Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
+ Invoke-WebRequest $url -OutFile installcredprovider.ps1
+
+ Write-Host 'Installing plugin...'
+ .\installcredprovider.ps1 -Force
+
+ Write-Host "Deleting local copy of 'installcredprovider.ps1'..."
+ Remove-Item .\installcredprovider.ps1
+
+ if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) {
+ Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!'
+ ExitWithExitCode 1
+ }
+ else {
+ Write-Host 'CredProvider plugin was installed correctly!'
+ }
+
+ # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
+ # feeds successfully
+
+ $nugetConfigPath = Join-Path $RepoRoot "NuGet.config"
+
+ if (-Not (Test-Path -Path $nugetConfigPath)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
+ ExitWithExitCode 1
+ }
+
+ $endpoints = New-Object System.Collections.ArrayList
+ $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value}
+
+ if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) {
+ foreach ($stableRestoreResource in $nugetConfigPackageSources) {
+ $trimmedResource = ([string]$stableRestoreResource).Trim()
+ [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"})
+ }
+ }
+
+ if (($endpoints | Measure-Object).Count -gt 0) {
+ $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
+
+ # Create the environment variables the AzDo way
+ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{
+ 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS'
+ 'issecret' = 'false'
+ }
+
+ # We don't want sessions cached since we will be updating the endpoints quite frequently
+ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{
+ 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED'
+ 'issecret' = 'false'
+ }
+ }
+ else
+ {
+ Write-Host 'No internal endpoints found in NuGet.config'
+ }
+}
+
+#Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+ $dotnetTempDir = Join-Path $RepoRoot "dotnet"
+ $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+ $dotnet = "$dotnetTempDir\dotnet.exe"
+ $restoreProjPath = "$PSScriptRoot\restore.proj"
+
+ Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+ InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+
+ '' | Out-File "$restoreProjPath"
+
+ & $dotnet restore $restoreProjPath
+
+ Write-Host 'Arcade SDK restored!'
+
+ if (Test-Path -Path $restoreProjPath) {
+ Remove-Item $restoreProjPath
+ }
+
+ if (Test-Path -Path $dotnetTempDir) {
+ Remove-Item $dotnetTempDir -Recurse
+ }
+}
+
+try {
+ Push-Location $PSScriptRoot
+
+ if ($Operation -like 'setup') {
+ SetupCredProvider $AuthToken
+ }
+ elseif ($Operation -like 'install-restore') {
+ InstallDotNetSdkAndRestoreArcade
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!"
+ ExitWithExitCode 1
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Arcade' -Message $_
+ ExitWithExitCode 1
+}
+finally {
+ Pop-Location
+}
diff --git a/src/arcade/eng/common/internal-feed-operations.sh b/src/arcade/eng/common/internal-feed-operations.sh
new file mode 100644
index 000000000000..9378223ba095
--- /dev/null
+++ b/src/arcade/eng/common/internal-feed-operations.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables.
+# This should ONLY be called from identified internal builds
+function SetupCredProvider {
+ local authToken=$1
+
+ # Install the Cred Provider NuGet plugin
+ echo "Setting up Cred Provider NuGet plugin in the agent..."...
+ echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+ local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh"
+
+ echo "Writing the contents of 'installcredprovider.ps1' locally..."
+ local installcredproviderPath="installcredprovider.sh"
+ if command -v curl > /dev/null; then
+ curl $url > "$installcredproviderPath"
+ else
+ wget -q -O "$installcredproviderPath" "$url"
+ fi
+
+ echo "Installing plugin..."
+ . "$installcredproviderPath"
+
+ echo "Deleting local copy of 'installcredprovider.sh'..."
+ rm installcredprovider.sh
+
+ if [ ! -d "$HOME/.nuget/plugins" ]; then
+ Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!'
+ ExitWithExitCode 1
+ else
+ echo "CredProvider plugin was installed correctly!"
+ fi
+
+ # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
+ # feeds successfully
+
+ local nugetConfigPath="{$repo_root}NuGet.config"
+
+ if [ ! "$nugetConfigPath" ]; then
+ Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"
+ ExitWithExitCode 1
+ fi
+
+ local endpoints='['
+ local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"`
+ local pattern="value=\"(.*)\""
+
+ for value in $nugetConfigPackageValues
+ do
+ if [[ $value =~ $pattern ]]; then
+ local endpoint="${BASH_REMATCH[1]}"
+ endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"},"
+ fi
+ done
+
+ endpoints=${endpoints%?}
+ endpoints+=']'
+
+ if [ ${#endpoints} -gt 2 ]; then
+ local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
+
+ echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
+ echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False"
+ else
+ echo "No internal endpoints found in NuGet.config"
+ fi
+}
+
+# Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+ local dotnetTempDir="$repo_root/dotnet"
+ local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+ local restoreProjPath="$repo_root/eng/common/restore.proj"
+
+ echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+ echo "" > "$restoreProjPath"
+
+ InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+
+ local res=`$dotnetTempDir/dotnet restore $restoreProjPath`
+ echo "Arcade SDK restored!"
+
+ # Cleanup
+ if [ "$restoreProjPath" ]; then
+ rm "$restoreProjPath"
+ fi
+
+ if [ "$dotnetTempDir" ]; then
+ rm -r $dotnetTempDir
+ fi
+}
+
+source="${BASH_SOURCE[0]}"
+operation=''
+authToken=''
+repoName=''
+
+while [[ $# > 0 ]]; do
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ --operation)
+ operation=$2
+ shift
+ ;;
+ --authtoken)
+ authToken=$2
+ shift
+ ;;
+ *)
+ echo "Invalid argument: $1"
+ usage
+ exit 1
+ ;;
+ esac
+
+ shift
+done
+
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ "$operation" = "setup" ]; then
+ SetupCredProvider $authToken
+elif [ "$operation" = "install-restore" ]; then
+ InstallDotNetSdkAndRestoreArcade
+else
+ echo "Unknown operation '$operation'!"
+fi
diff --git a/src/arcade/eng/common/internal/NuGet.config b/src/arcade/eng/common/internal/NuGet.config
new file mode 100644
index 000000000000..19d3d311b166
--- /dev/null
+++ b/src/arcade/eng/common/internal/NuGet.config
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
diff --git a/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss
new file mode 100644
index 000000000000..5d892d619398
--- /dev/null
+++ b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss
@@ -0,0 +1,29 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/arcade/eng/common/msbuild.ps1 b/src/arcade/eng/common/msbuild.ps1
new file mode 100644
index 000000000000..f041e5ddd958
--- /dev/null
+++ b/src/arcade/eng/common/msbuild.ps1
@@ -0,0 +1,28 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string] $verbosity = 'minimal',
+ [bool] $warnAsError = $true,
+ [bool] $nodeReuse = $true,
+ [switch] $ci,
+ [switch] $prepareMachine,
+ [switch] $excludePrereleaseVS,
+ [string] $msbuildEngine = $null,
+ [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
+)
+
+. $PSScriptRoot\tools.ps1
+
+try {
+ if ($ci) {
+ $nodeReuse = $false
+ }
+
+ MSBuild @extraArgs
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Build' -Message $_
+ ExitWithExitCode 1
+}
+
+ExitWithExitCode 0
\ No newline at end of file
diff --git a/src/arcade/eng/common/msbuild.sh b/src/arcade/eng/common/msbuild.sh
new file mode 100644
index 000000000000..20d3dad54352
--- /dev/null
+++ b/src/arcade/eng/common/msbuild.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+verbosity='minimal'
+warn_as_error=true
+node_reuse=true
+prepare_machine=false
+extra_args=''
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --verbosity)
+ verbosity=$2
+ shift 2
+ ;;
+ --warnaserror)
+ warn_as_error=$2
+ shift 2
+ ;;
+ --nodereuse)
+ node_reuse=$2
+ shift 2
+ ;;
+ --ci)
+ ci=true
+ shift 1
+ ;;
+ --preparemachine)
+ prepare_machine=true
+ shift 1
+ ;;
+ *)
+ extra_args="$extra_args $1"
+ shift 1
+ ;;
+ esac
+done
+
+. "$scriptroot/tools.sh"
+
+if [[ "$ci" == true ]]; then
+ node_reuse=false
+fi
+
+MSBuild $extra_args
+ExitWithExitCode 0
diff --git a/src/arcade/eng/common/native/common-library.sh b/src/arcade/eng/common/native/common-library.sh
new file mode 100644
index 000000000000..080c2c283ae4
--- /dev/null
+++ b/src/arcade/eng/common/native/common-library.sh
@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+
+function GetNativeInstallDirectory {
+ local install_dir
+
+ if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then
+ install_dir=$HOME/.netcoreeng/native/
+ else
+ install_dir=$NETCOREENG_INSTALL_DIRECTORY
+ fi
+
+ echo $install_dir
+ return 0
+}
+
+function GetTempDirectory {
+
+ echo $(GetNativeInstallDirectory)temp/
+ return 0
+}
+
+function ExpandZip {
+ local zip_path=$1
+ local output_directory=$2
+ local force=${3:-false}
+
+ echo "Extracting $zip_path to $output_directory"
+ if [[ -d $output_directory ]] && [[ $force = false ]]; then
+ echo "Directory '$output_directory' already exists, skipping extract"
+ return 0
+ fi
+
+ if [[ -d $output_directory ]]; then
+ echo "'Force flag enabled, but '$output_directory' exists. Removing directory"
+ rm -rf $output_directory
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'"
+ return 1
+ fi
+ fi
+
+ echo "Creating directory: '$output_directory'"
+ mkdir -p $output_directory
+
+ echo "Extracting archive"
+ tar -xf $zip_path -C $output_directory
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'"
+ return 1
+ fi
+
+ return 0
+}
+
+function GetCurrentOS {
+ local unameOut="$(uname -s)"
+ case $unameOut in
+ Linux*) echo "Linux";;
+ Darwin*) echo "MacOS";;
+ esac
+ return 0
+}
+
+function GetFile {
+ local uri=$1
+ local path=$2
+ local force=${3:-false}
+ local download_retries=${4:-5}
+ local retry_wait_time_seconds=${5:-30}
+
+ if [[ -f $path ]]; then
+ if [[ $force = false ]]; then
+ echo "File '$path' already exists. Skipping download"
+ return 0
+ else
+ rm -rf $path
+ fi
+ fi
+
+ if [[ -f $uri ]]; then
+ echo "'$uri' is a file path, copying file to '$path'"
+ cp $uri $path
+ return $?
+ fi
+
+ echo "Downloading $uri"
+ # Use curl if available, otherwise use wget
+ if command -v curl > /dev/null; then
+ curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail
+ else
+ wget -q -O "$path" "$uri" --tries="$download_retries"
+ fi
+
+ return $?
+}
+
+function GetTempPathFileName {
+ local path=$1
+
+ local temp_dir=$(GetTempDirectory)
+ local temp_file_name=$(basename $path)
+ echo $temp_dir$temp_file_name
+ return 0
+}
+
+function DownloadAndExtract {
+ local uri=$1
+ local installDir=$2
+ local force=${3:-false}
+ local download_retries=${4:-5}
+ local retry_wait_time_seconds=${5:-30}
+
+ local temp_tool_path=$(GetTempPathFileName $uri)
+
+ echo "downloading to: $temp_tool_path"
+
+ # Download file
+ GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'."
+ return 1
+ fi
+
+ # Extract File
+ echo "extracting from $temp_tool_path to $installDir"
+ ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds
+ if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'."
+ return 1
+ fi
+
+ return 0
+}
+
+function NewScriptShim {
+ local shimpath=$1
+ local tool_file_path=$2
+ local force=${3:-false}
+
+ echo "Generating '$shimpath' shim"
+ if [[ -f $shimpath ]]; then
+ if [[ $force = false ]]; then
+ echo "File '$shimpath' already exists." >&2
+ return 1
+ else
+ rm -rf $shimpath
+ fi
+ fi
+
+ if [[ ! -f $tool_file_path ]]; then
+ # try to see if the path is lower cased
+ tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")"
+ if [[ ! -f $tool_file_path ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
+ return 1
+ fi
+ fi
+
+ local shim_contents=$'#!/usr/bin/env bash\n'
+ shim_contents+="SHIMARGS="$'$1\n'
+ shim_contents+="$tool_file_path"$' $SHIMARGS\n'
+
+ # Write shim file
+ echo "$shim_contents" > $shimpath
+
+ chmod +x $shimpath
+
+ echo "Finished generating shim '$shimpath'"
+
+ return $?
+}
+
diff --git a/src/arcade/eng/common/native/install-cmake-test.sh b/src/arcade/eng/common/native/install-cmake-test.sh
new file mode 100644
index 000000000000..8a5e7cf0db5a
--- /dev/null
+++ b/src/arcade/eng/common/native/install-cmake-test.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --baseuri)
+ base_uri=$2
+ shift 2
+ ;;
+ --installpath)
+ install_path=$2
+ shift 2
+ ;;
+ --version)
+ version=$2
+ shift 2
+ ;;
+ --clean)
+ clean=true
+ shift 1
+ ;;
+ --force)
+ force=true
+ shift 1
+ ;;
+ --downloadretries)
+ download_retries=$2
+ shift 2
+ ;;
+ --retrywaittimeseconds)
+ retry_wait_time_seconds=$2
+ shift 2
+ ;;
+ --help)
+ echo "Common settings:"
+ echo " --baseuri Base file directory or Url wrom which to acquire tool archives"
+ echo " --installpath Base directory to install native tool to"
+ echo " --clean Don't install the tool, just clean up the current install of the tool"
+ echo " --force Force install of tools even if they previously exist"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Advanced settings:"
+ echo " --downloadretries Total number of retry attempts"
+ echo " --retrywaittimeseconds Wait time between retry attempts in seconds"
+ echo ""
+ exit 0
+ ;;
+ esac
+done
+
+tool_name="cmake-test"
+tool_os=$(GetCurrentOS)
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+ echo "Cleaning $tool_install_directory"
+ if [[ -d $tool_install_directory ]]; then
+ rm -rf $tool_install_directory
+ fi
+
+ echo "Cleaning $shim_path"
+ if [[ -f $shim_path ]]; then
+ rm -rf $shim_path
+ fi
+
+ tool_temp_path=$(GetTempPathFileName $uri)
+ echo "Cleaning $tool_temp_path"
+ if [[ -f $tool_temp_path ]]; then
+ rm -rf $tool_temp_path
+ fi
+
+ exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+ echo "$tool_name ($version) already exists, skipping install"
+ exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
+ exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
+ exit 1
+fi
+
+exit 0
diff --git a/src/arcade/eng/common/native/install-cmake.sh b/src/arcade/eng/common/native/install-cmake.sh
new file mode 100644
index 000000000000..de496beebc5a
--- /dev/null
+++ b/src/arcade/eng/common/native/install-cmake.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --baseuri)
+ base_uri=$2
+ shift 2
+ ;;
+ --installpath)
+ install_path=$2
+ shift 2
+ ;;
+ --version)
+ version=$2
+ shift 2
+ ;;
+ --clean)
+ clean=true
+ shift 1
+ ;;
+ --force)
+ force=true
+ shift 1
+ ;;
+ --downloadretries)
+ download_retries=$2
+ shift 2
+ ;;
+ --retrywaittimeseconds)
+ retry_wait_time_seconds=$2
+ shift 2
+ ;;
+ --help)
+ echo "Common settings:"
+ echo " --baseuri Base file directory or Url wrom which to acquire tool archives"
+ echo " --installpath Base directory to install native tool to"
+ echo " --clean Don't install the tool, just clean up the current install of the tool"
+ echo " --force Force install of tools even if they previously exist"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Advanced settings:"
+ echo " --downloadretries Total number of retry attempts"
+ echo " --retrywaittimeseconds Wait time between retry attempts in seconds"
+ echo ""
+ exit 0
+ ;;
+ esac
+done
+
+tool_name="cmake"
+tool_os=$(GetCurrentOS)
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+ echo "Cleaning $tool_install_directory"
+ if [[ -d $tool_install_directory ]]; then
+ rm -rf $tool_install_directory
+ fi
+
+ echo "Cleaning $shim_path"
+ if [[ -f $shim_path ]]; then
+ rm -rf $shim_path
+ fi
+
+ tool_temp_path=$(GetTempPathFileName $uri)
+ echo "Cleaning $tool_temp_path"
+ if [[ -f $tool_temp_path ]]; then
+ rm -rf $tool_temp_path
+ fi
+
+ exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+ echo "$tool_name ($version) already exists, skipping install"
+ exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
+ exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+ Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
+ exit 1
+fi
+
+exit 0
diff --git a/src/arcade/eng/common/native/install-tool.ps1 b/src/arcade/eng/common/native/install-tool.ps1
new file mode 100644
index 000000000000..78f2d84a4e4b
--- /dev/null
+++ b/src/arcade/eng/common/native/install-tool.ps1
@@ -0,0 +1,132 @@
+<#
+.SYNOPSIS
+Install native tool
+
+.DESCRIPTION
+Install cmake native tool from Azure blob storage
+
+.PARAMETER InstallPath
+Base directory to install native tool to
+
+.PARAMETER BaseUri
+Base file directory or Url from which to acquire tool archives
+
+.PARAMETER CommonLibraryDirectory
+Path to folder containing common library modules
+
+.PARAMETER Force
+Force install of tools even if they previously exist
+
+.PARAMETER Clean
+Don't install the tool, just clean up the current install of the tool
+
+.PARAMETER DownloadRetries
+Total number of retry attempts
+
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds
+
+.NOTES
+Returns 0 if install succeeds, 1 otherwise
+#>
+[CmdletBinding(PositionalBinding=$false)]
+Param (
+ [Parameter(Mandatory=$True)]
+ [string] $ToolName,
+ [Parameter(Mandatory=$True)]
+ [string] $InstallPath,
+ [Parameter(Mandatory=$True)]
+ [string] $BaseUri,
+ [Parameter(Mandatory=$True)]
+ [string] $Version,
+ [string] $CommonLibraryDirectory = $PSScriptRoot,
+ [switch] $Force = $False,
+ [switch] $Clean = $False,
+ [int] $DownloadRetries = 5,
+ [int] $RetryWaitTimeInSeconds = 30
+)
+
+. $PSScriptRoot\..\pipeline-logging-functions.ps1
+
+# Import common library modules
+Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1")
+
+try {
+ # Define verbose switch if undefined
+ $Verbose = $VerbosePreference -Eq "Continue"
+
+ $Arch = CommonLibrary\Get-MachineArchitecture
+ $ToolOs = "win64"
+ if($Arch -Eq "x32") {
+ $ToolOs = "win32"
+ }
+ $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch"
+ $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\"
+ $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip"
+ $ShimPath = Join-Path $InstallPath "$ToolName.exe"
+
+ if ($Clean) {
+ Write-Host "Cleaning $ToolInstallDirectory"
+ if (Test-Path $ToolInstallDirectory) {
+ Remove-Item $ToolInstallDirectory -Force -Recurse
+ }
+ Write-Host "Cleaning $ShimPath"
+ if (Test-Path $ShimPath) {
+ Remove-Item $ShimPath -Force
+ }
+ $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri
+ Write-Host "Cleaning $ToolTempPath"
+ if (Test-Path $ToolTempPath) {
+ Remove-Item $ToolTempPath -Force
+ }
+ exit 0
+ }
+
+ # Install tool
+ if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) {
+ Write-Verbose "$ToolName ($Version) already exists, skipping install"
+ }
+ else {
+ $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri `
+ -InstallDirectory $ToolInstallDirectory `
+ -Force:$Force `
+ -DownloadRetries $DownloadRetries `
+ -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+ -Verbose:$Verbose
+
+ if ($InstallStatus -Eq $False) {
+ Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping"
+ exit 1
+ }
+ }
+
+ $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName }
+ if (@($ToolFilePath).Length -Gt 1) {
+ Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
+ exit 1
+ } elseif (@($ToolFilePath).Length -Lt 1) {
+ Write-Host "$ToolName was not found in $ToolInstallDirectory."
+ exit 1
+ }
+
+ # Generate shim
+ # Always rewrite shims so that we are referencing the expected version
+ $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName `
+ -ShimDirectory $InstallPath `
+ -ToolFilePath "$ToolFilePath" `
+ -BaseUri $BaseUri `
+ -Force:$Force `
+ -Verbose:$Verbose
+
+ if ($GenerateShimStatus -Eq $False) {
+ Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping"
+ return 1
+ }
+
+ exit 0
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_
+ exit 1
+}
diff --git a/src/arcade/eng/common/pipeline-logging-functions.ps1 b/src/arcade/eng/common/pipeline-logging-functions.ps1
new file mode 100644
index 000000000000..8e422c561e4b
--- /dev/null
+++ b/src/arcade/eng/common/pipeline-logging-functions.ps1
@@ -0,0 +1,260 @@
+# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified.
+
+# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1
+
+$script:loggingCommandPrefix = '##vso['
+$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"?
+ New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' }
+ New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' }
+ New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' }
+ New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' }
+)
+# TODO: BUG: Escape % ???
+# TODO: Add test to verify don't need to escape "=".
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTelemetryError {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Category,
+ [Parameter(Mandatory = $true)]
+ [string]$Message,
+ [Parameter(Mandatory = $false)]
+ [string]$Type = 'error',
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput,
+ [switch]$Force)
+
+ $PSBoundParameters.Remove('Category') | Out-Null
+
+ if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
+ $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
+ }
+ $PSBoundParameters.Remove('Message') | Out-Null
+ $PSBoundParameters.Add('Message', $Message)
+ Write-PipelineTaskError @PSBoundParameters
+}
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTaskError {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Message,
+ [Parameter(Mandatory = $false)]
+ [string]$Type = 'error',
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput,
+ [switch]$Force
+ )
+
+ if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
+ if ($Type -eq 'error') {
+ Write-Host $Message -ForegroundColor Red
+ return
+ }
+ elseif ($Type -eq 'warning') {
+ Write-Host $Message -ForegroundColor Yellow
+ return
+ }
+ }
+
+ if (($Type -ne 'error') -and ($Type -ne 'warning')) {
+ Write-Host $Message
+ return
+ }
+ $PSBoundParameters.Remove('Force') | Out-Null
+ if (-not $PSBoundParameters.ContainsKey('Type')) {
+ $PSBoundParameters.Add('Type', 'error')
+ }
+ Write-LogIssue @PSBoundParameters
+}
+
+function Write-PipelineSetVariable {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Name,
+ [string]$Value,
+ [switch]$Secret,
+ [switch]$AsOutput,
+ [bool]$IsMultiJobVariable = $true)
+
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
+ 'variable' = $Name
+ 'isSecret' = $Secret
+ 'isOutput' = $IsMultiJobVariable
+ } -AsOutput:$AsOutput
+ }
+}
+
+function Write-PipelinePrependPath {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Path,
+ [switch]$AsOutput)
+
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
+ }
+}
+
+function Write-PipelineSetResult {
+ [CmdletBinding()]
+ param(
+ [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
+ [Parameter(Mandatory = $true)]
+ [string]$Result,
+ [string]$Message)
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
+ 'result' = $Result
+ }
+ }
+}
+
+<########################################
+# Private functions.
+########################################>
+function Format-LoggingCommandData {
+ [CmdletBinding()]
+ param([string]$Value, [switch]$Reverse)
+
+ if (!$Value) {
+ return ''
+ }
+
+ if (!$Reverse) {
+ foreach ($mapping in $script:loggingCommandEscapeMappings) {
+ $Value = $Value.Replace($mapping.Token, $mapping.Replacement)
+ }
+ }
+ else {
+ for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
+ $mapping = $script:loggingCommandEscapeMappings[$i]
+ $Value = $Value.Replace($mapping.Replacement, $mapping.Token)
+ }
+ }
+
+ return $Value
+}
+
+function Format-LoggingCommand {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Area,
+ [Parameter(Mandatory = $true)]
+ [string]$Event,
+ [string]$Data,
+ [hashtable]$Properties)
+
+ # Append the preamble.
+ [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder
+ $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event)
+
+ # Append the properties.
+ if ($Properties) {
+ $first = $true
+ foreach ($key in $Properties.Keys) {
+ [string]$value = Format-LoggingCommandData $Properties[$key]
+ if ($value) {
+ if ($first) {
+ $null = $sb.Append(' ')
+ $first = $false
+ }
+ else {
+ $null = $sb.Append(';')
+ }
+
+ $null = $sb.Append("$key=$value")
+ }
+ }
+ }
+
+ # Append the tail and output the value.
+ $Data = Format-LoggingCommandData $Data
+ $sb.Append(']').Append($Data).ToString()
+}
+
+function Write-LoggingCommand {
+ [CmdletBinding(DefaultParameterSetName = 'Parameters')]
+ param(
+ [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+ [string]$Area,
+ [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+ [string]$Event,
+ [Parameter(ParameterSetName = 'Parameters')]
+ [string]$Data,
+ [Parameter(ParameterSetName = 'Parameters')]
+ [hashtable]$Properties,
+ [Parameter(Mandatory = $true, ParameterSetName = 'Object')]
+ $Command,
+ [switch]$AsOutput)
+
+ if ($PSCmdlet.ParameterSetName -eq 'Object') {
+ Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput
+ return
+ }
+
+ $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
+ if ($AsOutput) {
+ $command
+ }
+ else {
+ Write-Host $command
+ }
+}
+
+function Write-LogIssue {
+ [CmdletBinding()]
+ param(
+ [ValidateSet('warning', 'error')]
+ [Parameter(Mandatory = $true)]
+ [string]$Type,
+ [string]$Message,
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput)
+
+ $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
+ 'type' = $Type
+ 'code' = $ErrCode
+ 'sourcepath' = $SourcePath
+ 'linenumber' = $LineNumber
+ 'columnnumber' = $ColumnNumber
+ }
+ if ($AsOutput) {
+ return $command
+ }
+
+ if ($Type -eq 'error') {
+ $foregroundColor = $host.PrivateData.ErrorForegroundColor
+ $backgroundColor = $host.PrivateData.ErrorBackgroundColor
+ if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+ $foregroundColor = [System.ConsoleColor]::Red
+ $backgroundColor = [System.ConsoleColor]::Black
+ }
+ }
+ else {
+ $foregroundColor = $host.PrivateData.WarningForegroundColor
+ $backgroundColor = $host.PrivateData.WarningBackgroundColor
+ if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+ $foregroundColor = [System.ConsoleColor]::Yellow
+ $backgroundColor = [System.ConsoleColor]::Black
+ }
+ }
+
+ Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor
+}
diff --git a/src/arcade/eng/common/pipeline-logging-functions.sh b/src/arcade/eng/common/pipeline-logging-functions.sh
new file mode 100644
index 000000000000..6a0b2255e911
--- /dev/null
+++ b/src/arcade/eng/common/pipeline-logging-functions.sh
@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+
+function Write-PipelineTelemetryError {
+ local telemetry_category=''
+ local force=false
+ local function_args=()
+ local message=''
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -category|-c)
+ telemetry_category=$2
+ shift
+ ;;
+ -force|-f)
+ force=true
+ ;;
+ -*)
+ function_args+=("$1 $2")
+ shift
+ ;;
+ *)
+ message=$*
+ ;;
+ esac
+ shift
+ done
+
+ if [[ $force != true ]] && [[ "$ci" != true ]]; then
+ echo "$message" >&2
+ return
+ fi
+
+ if [[ $force == true ]]; then
+ function_args+=("-force")
+ fi
+ message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message"
+ function_args+=("$message")
+ Write-PipelineTaskError ${function_args[@]}
+}
+
+function Write-PipelineTaskError {
+ local message_type="error"
+ local sourcepath=''
+ local linenumber=''
+ local columnnumber=''
+ local error_code=''
+ local force=false
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -type|-t)
+ message_type=$2
+ shift
+ ;;
+ -sourcepath|-s)
+ sourcepath=$2
+ shift
+ ;;
+ -linenumber|-ln)
+ linenumber=$2
+ shift
+ ;;
+ -columnnumber|-cn)
+ columnnumber=$2
+ shift
+ ;;
+ -errcode|-e)
+ error_code=$2
+ shift
+ ;;
+ -force|-f)
+ force=true
+ ;;
+ *)
+ break
+ ;;
+ esac
+
+ shift
+ done
+
+ if [[ $force != true ]] && [[ "$ci" != true ]]; then
+ echo "$@" >&2
+ return
+ fi
+
+ local message="##vso[task.logissue"
+
+ message="$message type=$message_type"
+
+ if [ -n "$sourcepath" ]; then
+ message="$message;sourcepath=$sourcepath"
+ fi
+
+ if [ -n "$linenumber" ]; then
+ message="$message;linenumber=$linenumber"
+ fi
+
+ if [ -n "$columnnumber" ]; then
+ message="$message;columnnumber=$columnnumber"
+ fi
+
+ if [ -n "$error_code" ]; then
+ message="$message;code=$error_code"
+ fi
+
+ message="$message]$*"
+ echo "$message"
+}
+
+function Write-PipelineSetVariable {
+ if [[ "$ci" != true ]]; then
+ return
+ fi
+
+ local name=''
+ local value=''
+ local secret=false
+ local as_output=false
+ local is_multi_job_variable=true
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -name|-n)
+ name=$2
+ shift
+ ;;
+ -value|-v)
+ value=$2
+ shift
+ ;;
+ -secret|-s)
+ secret=true
+ ;;
+ -as_output|-a)
+ as_output=true
+ ;;
+ -is_multi_job_variable|-i)
+ is_multi_job_variable=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ value=${value/;/%3B}
+ value=${value/\\r/%0D}
+ value=${value/\\n/%0A}
+ value=${value/]/%5D}
+
+ local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value"
+
+ if [[ "$as_output" == true ]]; then
+ $message
+ else
+ echo "$message"
+ fi
+}
+
+function Write-PipelinePrependPath {
+ local prepend_path=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -path|-p)
+ prepend_path=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ export PATH="$prepend_path:$PATH"
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.prependpath]$prepend_path"
+ fi
+}
+
+function Write-PipelineSetResult {
+ local result=''
+ local message=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -result|-r)
+ result=$2
+ shift
+ ;;
+ -message|-m)
+ message=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.complete result=$result;]$message"
+ fi
+}
diff --git a/src/arcade/eng/common/retain-build.ps1 b/src/arcade/eng/common/retain-build.ps1
new file mode 100644
index 000000000000..e7ba975adeb6
--- /dev/null
+++ b/src/arcade/eng/common/retain-build.ps1
@@ -0,0 +1,45 @@
+
+Param(
+[Parameter(Mandatory=$true)][int] $buildId,
+[Parameter(Mandatory=$true)][string] $azdoOrgUri,
+[Parameter(Mandatory=$true)][string] $azdoProject,
+[Parameter(Mandatory=$true)][string] $token
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+function Get-AzDOHeaders(
+ [string] $token)
+{
+ $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${token}"))
+ $headers = @{"Authorization"="Basic $base64AuthInfo"}
+ return $headers
+}
+
+function Update-BuildRetention(
+ [string] $azdoOrgUri,
+ [string] $azdoProject,
+ [int] $buildId,
+ [string] $token)
+{
+ $headers = Get-AzDOHeaders -token $token
+ $requestBody = "{
+ `"keepForever`": `"true`"
+ }"
+
+ $requestUri = "${azdoOrgUri}/${azdoProject}/_apis/build/builds/${buildId}?api-version=6.0"
+ write-Host "Attempting to retain build using the following URI: ${requestUri} ..."
+
+ try {
+ Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json"
+ Write-Host "Updated retention settings for build ${buildId}."
+ }
+ catch {
+ Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription"
+ exit 1
+ }
+}
+
+Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token
+exit 0
diff --git a/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1
new file mode 100644
index 000000000000..27f5a4115fc7
--- /dev/null
+++ b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1
@@ -0,0 +1,130 @@
+Param(
+ [string] $GuardianCliLocation,
+ [string] $WorkingDirectory,
+ [string] $TargetDirectory,
+ [string] $GdnFolder,
+ # The list of Guardian tools to configure. For each object in the array:
+ # - If the item is a [hashtable], it must contain these entries:
+ # - Name = The tool name as Guardian knows it.
+ # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
+ # among all tool entries with the same Name.
+ # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
+ # - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
+ [object[]] $ToolsList,
+ [string] $GuardianLoggerLevel='Standard',
+ # Optional: Additional params to add to any tool using CredScan.
+ [string[]] $CrScanAdditionalRunConfigParams,
+ # Optional: Additional params to add to any tool using PoliCheck.
+ [string[]] $PoliCheckAdditionalRunConfigParams,
+ # Optional: Additional params to add to any tool using CodeQL/Semmle.
+ [string[]] $CodeQLAdditionalRunConfigParams,
+ # Optional: Additional params to add to any tool using Binskim.
+ [string[]] $BinskimAdditionalRunConfigParams
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$global:LASTEXITCODE = 0
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ # Normalize tools list: all in [hashtable] form with defined values for each key.
+ $ToolsList = $ToolsList |
+ ForEach-Object {
+ if ($_ -is [string]) {
+ $_ = @{ Name = $_ }
+ }
+
+ if (-not ($_['Scenario'])) { $_.Scenario = "" }
+ if (-not ($_['Args'])) { $_.Args = @() }
+ $_
+ }
+
+ Write-Host "List of tools to configure:"
+ $ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
+
+ # We store config files in the r directory of .gdn
+ $gdnConfigPath = Join-Path $GdnFolder 'r'
+ $ValidPath = Test-Path $GuardianCliLocation
+
+ if ($ValidPath -eq $False)
+ {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+ ExitWithExitCode 1
+ }
+
+ foreach ($tool in $ToolsList) {
+ # Put together the name and scenario to make a unique key.
+ $toolConfigName = $tool.Name
+ if ($tool.Scenario) {
+ $toolConfigName += "_" + $tool.Scenario
+ }
+
+ Write-Host "=== Configuring $toolConfigName..."
+
+ $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
+
+ # For some tools, add default and automatic args.
+ switch -Exact ($tool.Name) {
+ 'credscan' {
+ if ($targetDirectory) {
+ $tool.Args += "`"TargetDirectory < $TargetDirectory`""
+ }
+ $tool.Args += "`"OutputType < pre`""
+ $tool.Args += $CrScanAdditionalRunConfigParams
+ }
+ 'policheck' {
+ if ($targetDirectory) {
+ $tool.Args += "`"Target < $TargetDirectory`""
+ }
+ $tool.Args += $PoliCheckAdditionalRunConfigParams
+ }
+ {$_ -in 'semmle', 'codeql'} {
+ if ($targetDirectory) {
+ $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
+ }
+ $tool.Args += $CodeQLAdditionalRunConfigParams
+ }
+ 'binskim' {
+ if ($targetDirectory) {
+ # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924.
+ # We are excluding all `_.pdb` files from the scan.
+ $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`""
+ }
+ $tool.Args += $BinskimAdditionalRunConfigParams
+ }
+ }
+
+ # Create variable pointing to the args array directly so we can use splat syntax later.
+ $toolArgs = $tool.Args
+
+ # Configure the tool. If args array is provided or the current tool has some default arguments
+ # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
+ # one per parameter. Doc page for "guardian configure":
+ # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
+ Exec-BlockVerbosely {
+ & $GuardianCliLocation configure `
+ --working-directory $WorkingDirectory `
+ --tool $tool.Name `
+ --output-path $gdnConfigFile `
+ --logger-level $GuardianLoggerLevel `
+ --noninteractive `
+ --force `
+ $(if ($toolArgs) { "--args" }) @toolArgs
+ Exit-IfNZEC "Sdl"
+ }
+
+ Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1
new file mode 100644
index 000000000000..68da4fbf2571
--- /dev/null
+++ b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1
@@ -0,0 +1,63 @@
+# This script looks for each archive file in a directory and extracts it into the target directory.
+# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
+# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
+param(
+ # Full path to directory where archives are stored.
+ [Parameter(Mandatory=$true)][string] $InputPath,
+ # Full path to directory to extract archives into. May be the same as $InputPath.
+ [Parameter(Mandatory=$true)][string] $ExtractPath
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ Measure-Command {
+ $jobs = @()
+
+ # Find archive files for non-Windows and Windows builds.
+ $archiveFiles = @(
+ Get-ChildItem (Join-Path $InputPath "*.tar.gz")
+ Get-ChildItem (Join-Path $InputPath "*.zip")
+ )
+
+ foreach ($targzFile in $archiveFiles) {
+ $jobs += Start-Job -ScriptBlock {
+ $file = $using:targzFile
+ $fileName = [System.IO.Path]::GetFileName($file)
+ $extractDir = Join-Path $using:ExtractPath "$fileName.extracted"
+
+ New-Item $extractDir -ItemType Directory -Force | Out-Null
+
+ Write-Host "Extracting '$file' to '$extractDir'..."
+
+ # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
+ # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
+ # error. Save output so it can be stored in the exception string along with context.
+ $output = tar -xf $file -C $extractDir 2>&1
+ # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
+ # don't have access to the outer scope.
+ if ($LASTEXITCODE -ne 0) {
+ throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
+ }
+
+ Write-Host "Extracted to $extractDir"
+ }
+ }
+
+ Receive-Job $jobs -Wait
+ }
+}
+catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1
new file mode 100644
index 000000000000..f031ed5b25e9
--- /dev/null
+++ b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1
@@ -0,0 +1,82 @@
+param(
+ [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored
+ [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+
+function ExtractArtifacts {
+ if (!(Test-Path $InputPath)) {
+ Write-Host "Input Path does not exist: $InputPath"
+ ExitWithExitCode 0
+ }
+ $Jobs = @()
+ Get-ChildItem "$InputPath\*.nupkg" |
+ ForEach-Object {
+ $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName
+ }
+
+ foreach ($Job in $Jobs) {
+ Wait-Job -Id $Job.Id | Receive-Job
+ }
+}
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ $ExtractPackage = {
+ param(
+ [string] $PackagePath # Full path to a NuGet package
+ )
+
+ if (!(Test-Path $PackagePath)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
+ ExitWithExitCode 1
+ }
+
+ $RelevantExtensions = @('.dll', '.exe', '.pdb')
+ Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
+
+ $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+ $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
+
+ Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+ [System.IO.Directory]::CreateDirectory($ExtractPath);
+
+ try {
+ $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
+
+ $zip.Entries |
+ Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
+ ForEach-Object {
+ $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName)
+ [System.IO.Directory]::CreateDirectory($TargetPath);
+
+ $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName
+ [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile)
+ }
+ }
+ catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+ }
+ finally {
+ $zip.Dispose()
+ }
+ }
+ Measure-Command { ExtractArtifacts }
+}
+catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/sdl/packages.config b/src/arcade/eng/common/sdl/packages.config
similarity index 51%
rename from eng/common/sdl/packages.config
rename to src/arcade/eng/common/sdl/packages.config
index 4585cfd6bba1..e5f543ea68c2 100644
--- a/eng/common/sdl/packages.config
+++ b/src/arcade/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-
+
diff --git a/src/arcade/eng/common/sdl/run-sdl.ps1 b/src/arcade/eng/common/sdl/run-sdl.ps1
new file mode 100644
index 000000000000..2eac8c78f103
--- /dev/null
+++ b/src/arcade/eng/common/sdl/run-sdl.ps1
@@ -0,0 +1,49 @@
+Param(
+ [string] $GuardianCliLocation,
+ [string] $WorkingDirectory,
+ [string] $GdnFolder,
+ [string] $UpdateBaseline,
+ [string] $GuardianLoggerLevel='Standard'
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$global:LASTEXITCODE = 0
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ # We store config files in the r directory of .gdn
+ $gdnConfigPath = Join-Path $GdnFolder 'r'
+ $ValidPath = Test-Path $GuardianCliLocation
+
+ if ($ValidPath -eq $False)
+ {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+ ExitWithExitCode 1
+ }
+
+ $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig'
+ Write-Host "Discovered Guardian config files:"
+ $gdnConfigFiles | Out-String | Write-Host
+
+ Exec-BlockVerbosely {
+ & $GuardianCliLocation run `
+ --working-directory $WorkingDirectory `
+ --baseline mainbaseline `
+ --update-baseline $UpdateBaseline `
+ --logger-level $GuardianLoggerLevel `
+ --config @gdnConfigFiles
+ Exit-IfNZEC "Sdl"
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/src/arcade/eng/common/templates-official/variables/pool-providers.yml b/src/arcade/eng/common/templates-official/variables/pool-providers.yml
new file mode 100644
index 000000000000..1f308b24efc4
--- /dev/null
+++ b/src/arcade/eng/common/templates-official/variables/pool-providers.yml
@@ -0,0 +1,45 @@
+# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool,
+# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches.
+
+# Motivation:
+# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS
+# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing
+# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS.
+# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services
+# team needs to move resources around and create new and potentially differently-named pools. Using this template
+# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming.
+
+# How to use:
+# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do).
+# If we find alternate naming conventions in broad usage it can be added to the condition below.
+#
+# First, import the template in an arcade-ified repo to pick up the variables, e.g.:
+#
+# variables:
+# - template: /eng/common/templates-official/variables/pool-providers.yml
+#
+# ... then anywhere specifying the pool provider use the runtime variables,
+# $(DncEngInternalBuildPool)
+#
+# pool:
+# name: $(DncEngInternalBuildPool)
+# image: 1es-windows-2022
+
+variables:
+ # Coalesce the target and source branches so we know when a PR targets a release branch
+ # If these variables are somehow missing, fall back to main (tends to have more capacity)
+
+ # Any new -Svc alternative pools should have variables added here to allow for splitting work
+
+ - name: DncEngInternalBuildPool
+ value: $[
+ replace(
+ replace(
+ eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'),
+ True,
+ 'NetCore1ESPool-Svc-Internal'
+ ),
+ False,
+ 'NetCore1ESPool-Internal'
+ )
+ ]
\ No newline at end of file
diff --git a/src/arcade/eng/common/templates-official/variables/sdl-variables.yml b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml
new file mode 100644
index 000000000000..dbdd66d4a4b3
--- /dev/null
+++ b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml
@@ -0,0 +1,7 @@
+variables:
+# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+# sync with the packages.config file.
+- name: DefaultGuardianVersion
+ value: 0.109.0
+- name: GuardianPackagesConfigFile
+ value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
\ No newline at end of file