Skip to content

Commit 9a9537e

Browse files
authored
[build] Update main with latest changes for internal builds (#17255)
* [build] Update main with latest changes for internal builds * This is only for net8 branch
1 parent 2fb0727 commit 9a9537e

File tree

13 files changed

+327
-115
lines changed

13 files changed

+327
-115
lines changed

eng/automation/SignVerifyIgnore.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
**\*.xml,ignore unsigned .xml
2+
**\cab*.cab.cab,ignore unsigned .cab

eng/common/SetupNugetSources.ps1

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -146,22 +146,22 @@ $userName = "dn-bot"
146146
# Insert credential nodes for Maestro's private feeds
147147
InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
148148

149+
# 3.1 uses a different feed url format so it's handled differently here
149150
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
150151
if ($dotnet31Source -ne $null) {
151152
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
152153
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
153154
}
154155

155-
$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']")
156-
if ($dotnet5Source -ne $null) {
157-
AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
158-
AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
159-
}
156+
$dotnetVersions = @('5','6','7','8')
160157

161-
$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']")
162-
if ($dotnet6Source -ne $null) {
163-
AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
164-
AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
158+
foreach ($dotnetVersion in $dotnetVersions) {
159+
$feedPrefix = "dotnet" + $dotnetVersion;
160+
$dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
161+
if ($dotnetSource -ne $null) {
162+
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
163+
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
164+
}
165165
}
166166

167-
$doc.Save($filename)
167+
$doc.Save($filename)

eng/common/SetupNugetSources.sh

Lines changed: 25 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -105,53 +105,33 @@ if [ "$?" == "0" ]; then
105105
PackageSources+=('dotnet3.1-internal-transport')
106106
fi
107107

108-
# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present
109-
grep -i "<add key=\"dotnet5\"" $ConfigFile
110-
if [ "$?" == "0" ]; then
111-
grep -i "<add key=\"dotnet5-internal\"" $ConfigFile
112-
if [ "$?" != "0" ]; then
113-
echo "Adding dotnet5-internal to the packageSources."
114-
PackageSourcesNodeFooter="</packageSources>"
115-
PackageSourceTemplate="${TB}<add key=\"dotnet5-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2\" />"
116-
117-
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
118-
fi
119-
PackageSources+=('dotnet5-internal')
120-
121-
grep -i "<add key=\"dotnet5-internal-transport\">" $ConfigFile
122-
if [ "$?" != "0" ]; then
123-
echo "Adding dotnet5-internal-transport to the packageSources."
124-
PackageSourcesNodeFooter="</packageSources>"
125-
PackageSourceTemplate="${TB}<add key=\"dotnet5-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2\" />"
126-
127-
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
128-
fi
129-
PackageSources+=('dotnet5-internal-transport')
130-
fi
131-
132-
# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
133-
grep -i "<add key=\"dotnet6\"" $ConfigFile
134-
if [ "$?" == "0" ]; then
135-
grep -i "<add key=\"dotnet6-internal\"" $ConfigFile
136-
if [ "$?" != "0" ]; then
137-
echo "Adding dotnet6-internal to the packageSources."
138-
PackageSourcesNodeFooter="</packageSources>"
139-
PackageSourceTemplate="${TB}<add key=\"dotnet6-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2\" />"
140-
141-
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
142-
fi
143-
PackageSources+=('dotnet6-internal')
108+
DotNetVersions=('5' '6' '7' '8')
109+
110+
for DotNetVersion in ${DotNetVersions[@]} ; do
111+
FeedPrefix="dotnet${DotNetVersion}";
112+
grep -i "<add key=\"$FeedPrefix\"" $ConfigFile
113+
if [ "$?" == "0" ]; then
114+
grep -i "<add key=\"$FeedPrefix-internal\"" $ConfigFile
115+
if [ "$?" != "0" ]; then
116+
echo "Adding $FeedPrefix-internal to the packageSources."
117+
PackageSourcesNodeFooter="</packageSources>"
118+
PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/v2\" />"
119+
120+
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
121+
fi
122+
PackageSources+=("$FeedPrefix-internal")
144123

145-
grep -i "<add key=\"dotnet6-internal-transport\">" $ConfigFile
146-
if [ "$?" != "0" ]; then
147-
echo "Adding dotnet6-internal-transport to the packageSources."
148-
PackageSourcesNodeFooter="</packageSources>"
149-
PackageSourceTemplate="${TB}<add key=\"dotnet6-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2\" />"
124+
grep -i "<add key=\"$FeedPrefix-internal-transport\">" $ConfigFile
125+
if [ "$?" != "0" ]; then
126+
echo "Adding $FeedPrefix-internal-transport to the packageSources."
127+
PackageSourcesNodeFooter="</packageSources>"
128+
PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/v2\" />"
150129

151-
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
130+
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
131+
fi
132+
PackageSources+=("$FeedPrefix-internal-transport")
152133
fi
153-
PackageSources+=('dotnet6-internal-transport')
154-
fi
134+
done
155135

156136
# I want things split line by line
157137
PrevIFS=$IFS
@@ -188,4 +168,4 @@ if [ "$?" == "0" ]; then
188168
echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
189169
fi
190170
done
191-
fi
171+
fi

eng/common/tools.ps1

Lines changed: 58 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,25 @@ function InstallDotNet([string] $dotnetRoot,
287287
[string] $runtimeSourceFeedKey = '',
288288
[switch] $noPath) {
289289

290+
$dotnetVersionLabel = "'sdk v$version'"
291+
292+
if ($runtime -ne '' -and $runtime -ne 'sdk') {
293+
$runtimePath = $dotnetRoot
294+
$runtimePath = $runtimePath + "\shared"
295+
if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" }
296+
if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" }
297+
if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" }
298+
$runtimePath = $runtimePath + "\" + $version
299+
300+
$dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'"
301+
302+
if (Test-Path $runtimePath) {
303+
Write-Host " Runtime toolset '$runtime/$architecture v$version' already installed."
304+
$installSuccess = $true
305+
Exit
306+
}
307+
}
308+
290309
$installScript = GetDotNetInstallScript $dotnetRoot
291310
$installParameters = @{
292311
Version = $version
@@ -323,18 +342,18 @@ function InstallDotNet([string] $dotnetRoot,
323342
} else {
324343
$location = "public location";
325344
}
326-
Write-Host "Attempting to install dotnet from $location."
345+
Write-Host " Attempting to install $dotnetVersionLabel from $location."
327346
try {
328347
& $installScript @variation
329348
$installSuccess = $true
330349
break
331350
}
332351
catch {
333-
Write-Host "Failed to install dotnet from $location."
352+
Write-Host " Failed to install $dotnetVersionLabel from $location."
334353
}
335354
}
336355
if (-not $installSuccess) {
337-
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations."
356+
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations."
338357
ExitWithExitCode 1
339358
}
340359
}
@@ -360,15 +379,22 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
360379
}
361380

362381
# Minimum VS version to require.
363-
$vsMinVersionReqdStr = '16.8'
382+
$vsMinVersionReqdStr = '17.6'
364383
$vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
365384

366385
# If the version of msbuild is going to be xcopied,
367386
# use this version. Version matches a package here:
368-
# https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview
369-
$defaultXCopyMSBuildVersion = '16.10.0-preview2'
387+
# https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.6.0-2
388+
$defaultXCopyMSBuildVersion = '17.6.0-2'
370389

371-
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
390+
if (!$vsRequirements) {
391+
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
392+
$vsRequirements = $GlobalJson.tools.vs
393+
}
394+
else {
395+
$vsRequirements = New-Object PSObject -Property @{ version = $vsMinVersionReqdStr }
396+
}
397+
}
372398
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
373399
$vsMinVersion = [Version]::new($vsMinVersionStr)
374400

@@ -392,7 +418,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
392418
# Locate Visual Studio installation or download x-copy msbuild.
393419
$vsInfo = LocateVisualStudio $vsRequirements
394420
if ($vsInfo -ne $null) {
395-
$vsInstallDir = $vsInfo.installationPath
421+
# Ensure vsInstallDir has a trailing slash
422+
$vsInstallDir = Join-Path $vsInfo.installationPath "\"
396423
$vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
397424

398425
InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
@@ -406,6 +433,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
406433
if($vsMinVersion -lt $vsMinVersionReqd){
407434
Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
408435
$xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
436+
$vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
409437
}
410438
else{
411439
# If the VS version IS compatible, look for an xcopy msbuild package
@@ -573,7 +601,7 @@ function InitializeBuildTool() {
573601
ExitWithExitCode 1
574602
}
575603
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
576-
$buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
604+
$buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net8.0' }
577605
} elseif ($msbuildEngine -eq "vs") {
578606
try {
579607
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -635,18 +663,26 @@ function InitializeNativeTools() {
635663
InstallDirectory = "$ToolsDir"
636664
}
637665
}
666+
if ($env:NativeToolsOnMachine) {
667+
Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..."
668+
$nativeArgs += @{ PathPromotion = $true }
669+
}
638670
& "$PSScriptRoot/init-tools-native.ps1" @nativeArgs
639671
}
640672
}
641673

674+
function Read-ArcadeSdkVersion() {
675+
return $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
676+
}
677+
642678
function InitializeToolset() {
643679
if (Test-Path variable:global:_ToolsetBuildProj) {
644680
return $global:_ToolsetBuildProj
645681
}
646682

647683
$nugetCache = GetNuGetPackageCachePath
648684

649-
$toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
685+
$toolsetVersion = Read-ArcadeSdkVersion
650686
$toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
651687

652688
if (Test-Path $toolsetLocationFile) {
@@ -731,6 +767,8 @@ function MSBuild() {
731767
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
732768
(Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')),
733769
(Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
770+
(Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
771+
(Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
734772
)
735773
$selectedPath = $null
736774
foreach ($path in $possiblePaths) {
@@ -803,7 +841,8 @@ function MSBuild-Core() {
803841
Write-Host "See log: $buildLog" -ForegroundColor DarkGray
804842
}
805843

806-
if ($ci) {
844+
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
845+
if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) {
807846
Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
808847
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
809848
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
@@ -898,11 +937,13 @@ if (!$disableConfigureToolsetImport) {
898937
function Enable-Nuget-EnhancedRetry() {
899938
if ($ci) {
900939
Write-Host "Setting NUGET enhanced retry environment variables"
901-
$env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true'
902-
$env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6
903-
$env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
904-
Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true'
905-
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6'
906-
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
940+
$env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true'
941+
$env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6
942+
$env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
943+
$env:NUGET_RETRY_HTTP_429 = 'true'
944+
Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true'
945+
Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6'
946+
Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
947+
Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true'
907948
}
908949
}

0 commit comments

Comments
 (0)