Update to .NET 9

Directory.Build.props

@@ -3,17 +3,18 @@
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 
   <PropertyGroup>
-    <DefaultNetCoreTargetFramework>net8.0</DefaultNetCoreTargetFramework>
+    <DefaultNetCoreTargetFramework>net9.0</DefaultNetCoreTargetFramework>
     <LangVersion>latest</LangVersion>
-    <NoWarn>$(NoWarn);CS1591</NoWarn>
+    <NoWarn>$(NoWarn);CA1515;CS1591</NoWarn>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <DebugSymbols>true</DebugSymbols>
     <DebugType>portable</DebugType>
+    <SuppressSymbolPackageFormatValidation>true</SuppressSymbolPackageFormatValidation>
     <SymbolPackageFormat>snupkg</SymbolPackageFormat>
     <DefineConstants Condition=" '$(Configuration)' == 'Debug' ">$(DefineConstants);JETBRAINS_ANNOTATIONS</DefineConstants>
     <Nullable>enable</Nullable>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
-    <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
+    <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -64,6 +65,8 @@
 
   <PropertyGroup>
     <EnablePackageValidation>$(IsPackable)</EnablePackageValidation>
+    <!-- TODO Re-enable when the 9.0.0 packages are published to NuGet.org -->
+    <EnablePackageValidation>false</EnablePackageValidation>
   </PropertyGroup>
 
   <PropertyGroup>

Directory.Packages.props

@@ -1,14 +1,16 @@
 <Project>
 
   <ItemGroup>
-    <PackageVersion Include="AngleSharp" Version="1.0.6" />
+    <PackageVersion Include="AngleSharp" Version="1.1.2" />
     <PackageVersion Include="JetBrains.Annotations" Version="2024.2.0" />
     <PackageVersion Include="JustEat.HttpClientInterception" Version="4.3.0" />
     <PackageVersion Include="MartinCostello.Logging.XUnit" Version="0.4.0" />
-    <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.8" />
-    <PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="8.0.8" />
-    <PackageVersion Include="Microsoft.IdentityModel.Protocols" Version="7.0.3" />
+    <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="9.0.0-preview.7.24406.2" />
+    <PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="9.0.0-preview.7.24406.2" />
+    <PackageVersion Include="Microsoft.IdentityModel.Protocols" Version="8.0.0" />
     <PackageVersion Include="Shouldly" Version="4.2.1" />
+    <PackageVersion Include="System.Net.Http" Version="4.3.4" />
+    <PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />
   </ItemGroup>
 
 </Project>

eng/Version.Details.xml

@@ -5,15 +5,15 @@
   </ProductDependencies>
 
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21519.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24453.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>85f3aa16d8797b5020f1fda11df1a958feb5f8df</Sha>
+      <Sha>dd332f2d4e21daa8b79f84251ab156af9a0b11b2</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
 
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21519.3">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.24453.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>85f3aa16d8797b5020f1fda11df1a958feb5f8df</Sha>
+      <Sha>dd332f2d4e21daa8b79f84251ab156af9a0b11b2</Sha>
     </Dependency>
   </ToolsetDependencies>
 

eng/Versions.props

@@ -1,13 +1,13 @@
 <Project>
 
   <PropertyGroup>
-    <MajorVersion>8</MajorVersion>
+    <MajorVersion>9</MajorVersion>
     <MinorVersion>0</MinorVersion>
-    <PatchVersion>1</PatchVersion>
+    <PatchVersion>0</PatchVersion>
     <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>
-    <PackageValidationBaselineVersion>8.0.0</PackageValidationBaselineVersion>
+    <PackageValidationBaselineVersion Condition=" '$(EnablePackageValidation)' == 'true' AND '$(PackageValidationBaselineVersion)' == '' ">9.0.0</PackageValidationBaselineVersion>
     <PreReleaseVersionLabel>preview</PreReleaseVersionLabel>
-    <PreReleaseVersionIteration></PreReleaseVersionIteration>
+    <PreReleaseVersionIteration>7</PreReleaseVersionIteration>
     <PreReleaseBrandingLabel>Preview $(PreReleaseVersionIteration)</PreReleaseBrandingLabel>
     <StabilizePackageVersion Condition="'$(StabilizePackageVersion)' == ''">false</StabilizePackageVersion>
     <DotNetFinalVersionKind Condition="'$(StabilizePackageVersion)' == 'true'">release</DotNetFinalVersionKind>

eng/common/SetupNugetSources.ps1

@@ -1,17 +1,10 @@
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
+# See example call for this script below.
 #
 #  - task: PowerShell@2
 #    displayName: Setup Private Feeds Credentials
@@ -21,11 +14,18 @@
 #      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
 #    env:
 #      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed.
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 [CmdletBinding()]
 param (
     [Parameter(Mandatory = $true)][string]$ConfigFile,
-    [Parameter(Mandatory = $true)][string]$Password
+    $Password
 )
 
 $ErrorActionPreference = "Stop"
@@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0
 . $PSScriptRoot\tools.ps1
 
 # Add source entry to PackageSources
-function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
     $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
 
     if ($packageSource -eq $null)
@@ -48,12 +48,17 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern
     else {
         Write-Host "Package source $SourceName already present."
     }
-    
-    AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
+
+    AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
 }
 
 # Add a credential node for the specified source
-function AddCredential($creds, $source, $username, $password) {
+function AddCredential($creds, $source, $username, $pwd) {
+    # If no cred supplied, don't do anything.
+    if (!$pwd) {
+        return;
+    }
+
     # Looks for credential configuration for the given SourceName. Create it if none is found.
     $sourceElement = $creds.SelectSingleNode($Source)
     if ($sourceElement -eq $null)
@@ -82,17 +87,18 @@ function AddCredential($creds, $source, $username, $password) {
         $passwordElement.SetAttribute("key", "ClearTextPassword")
         $sourceElement.AppendChild($passwordElement) | Out-Null
     }
-    $passwordElement.SetAttribute("value", $Password)
+
+    $passwordElement.SetAttribute("value", $pwd)
 }
 
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
     $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
 
     Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
 
     ForEach ($PackageSource in $maestroPrivateSources) {
         Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
-        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
+        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
     }
 }
 
@@ -110,11 +116,6 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) {
   ExitWithExitCode 1
 }
 
-if (!$Password) {
-    Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
-    ExitWithExitCode 1
-}
-
 # Load NuGet.config
 $doc = New-Object System.Xml.XmlDocument
 $filename = (Get-Item $ConfigFile).FullName
@@ -127,11 +128,14 @@ if ($sources -eq $null) {
     $doc.DocumentElement.AppendChild($sources) | Out-Null
 }
 
-# Looks for a <PackageSourceCredentials> node. Create it if none is found.
-$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
-if ($creds -eq $null) {
-    $creds = $doc.CreateElement("packageSourceCredentials")
-    $doc.DocumentElement.AppendChild($creds) | Out-Null
+$creds = $null
+if ($Password) {
+    # Looks for a <PackageSourceCredentials> node. Create it if none is found.
+    $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+    if ($creds -eq $null) {
+        $creds = $doc.CreateElement("packageSourceCredentials")
+        $doc.DocumentElement.AppendChild($creds) | Out-Null
+    }
 }
 
 # Check for disabledPackageSources; we'll enable any darc-int ones we find there
@@ -144,23 +148,23 @@ if ($disabledSources -ne $null) {
 $userName = "dn-bot"
 
 # Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
 
 # 3.1 uses a different feed url format so it's handled differently here
 $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
 if ($dotnet31Source -ne $null) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
 }
 
-$dotnetVersions = @('5','6','7','8')
+$dotnetVersions = @('5','6','7','8','9')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;
     $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
     if ($dotnetSource -ne $null) {
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
     }
 }
 

eng/common/SetupNugetSources.sh

@@ -1,28 +1,27 @@
 #!/usr/bin/env bash
 
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
+# 
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
-#
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
+# See example call for this script below.
 #
 #  - task: Bash@3
-#    displayName: Setup Private Feeds Credentials
+#    displayName: Setup Internal Feeds
 #    inputs:
 #      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-#      arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+#      arguments: $(Build.SourcesDirectory)/NuGet.config
 #    condition: ne(variables['Agent.OS'], 'Windows_NT')
-#    env:
-#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#  - task: NuGetAuthenticate@1
+#
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed.
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 ConfigFile=$1
 CredToken=$2
@@ -48,11 +47,6 @@ if [ ! -f "$ConfigFile" ]; then
     ExitWithExitCode 1
 fi
 
-if [ -z "$CredToken" ]; then
-    Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
-    ExitWithExitCode 1
-fi
-
 if [[ `uname -s` == "Darwin" ]]; then
     NL=$'\\\n'
     TB=''
@@ -105,7 +99,7 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet3.1-internal-transport')
 fi
 
-DotNetVersions=('5' '6' '7' '8')
+DotNetVersions=('5' '6' '7' '8' '9')
 
 for DotNetVersion in ${DotNetVersions[@]} ; do
     FeedPrefix="dotnet${DotNetVersion}";
@@ -140,18 +134,20 @@ PackageSources+="$IFS"
 PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
 IFS=$PrevIFS
 
-for FeedName in ${PackageSources[@]} ; do
-    # Check if there is no existing credential for this FeedName
-    grep -i "<$FeedName>" $ConfigFile 
-    if [ "$?" != "0" ]; then
-        echo "Adding credentials for $FeedName."
+if [ "$CredToken" ]; then
+    for FeedName in ${PackageSources[@]} ; do
+        # Check if there is no existing credential for this FeedName
+        grep -i "<$FeedName>" $ConfigFile 
+        if [ "$?" != "0" ]; then
+            echo "Adding credentials for $FeedName."
 
-        PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
-        NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
+            PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
+            NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
 
-        sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
-    fi
-done
+            sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
+        fi
+    done
+fi
 
 # Re-enable any entries in disabledPackageSources where the feed name contains darc-int
 grep -i "<disabledPackageSources>" $ConfigFile

eng/common/build.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" %*"
+exit /b %ErrorLevel%

eng/common/build.ps1

@@ -19,6 +19,7 @@ Param(
   [switch] $pack,
   [switch] $publish,
   [switch] $clean,
+  [switch][Alias('pb')]$productBuild,
   [switch][Alias('bl')]$binaryLog,
   [switch][Alias('nobl')]$excludeCIBinarylog,
   [switch] $ci,
@@ -58,6 +59,7 @@ function Print-Usage() {
   Write-Host "  -sign                   Sign build outputs"
   Write-Host "  -publish                Publish artifacts (e.g. symbols)"
   Write-Host "  -clean                  Clean the solution"
+  Write-Host "  -productBuild           Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)"
   Write-Host ""
 
   Write-Host "Advanced settings:"
@@ -120,6 +122,7 @@ function Build {
     /p:Deploy=$deploy `
     /p:Test=$test `
     /p:Pack=$pack `
+    /p:DotNetBuildRepo=$productBuild `
     /p:IntegrationTest=$integrationTest `
     /p:PerformanceTest=$performanceTest `
     /p:Sign=$sign `