[release/dev17.8] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -98,9 +98,9 @@
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
       <Sha>3a25a7f1cc446b60678ed25c9d829420d6321eba</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24570.5">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.25515.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>3c7e11bf80279cde53a6251c4d0fa10e613fc739</Sha>
+      <Sha>6544413e02741855b701468aa8afc6cf8ca62c72</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.23475.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">

eng/common/SetupNugetSources.ps1

@@ -17,8 +17,8 @@
 #    displayName: Setup Private Feeds Credentials
 #    condition: eq(variables['Agent.OS'], 'Windows_NT')
 #    inputs:
-#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
-#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+#      filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
+#      arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token
 #    env:
 #      Token: $(dn-bot-dnceng-artifact-feeds-rw)
 

eng/common/SetupNugetSources.sh

@@ -18,8 +18,8 @@
 #  - task: Bash@3
 #    displayName: Setup Private Feeds Credentials
 #    inputs:
-#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-#      arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+#      filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh
+#      arguments: $(System.DefaultWorkingDirectory)/NuGet.config $Token
 #    condition: ne(variables['Agent.OS'], 'Windows_NT')
 #    env:
 #      Token: $(dn-bot-dnceng-artifact-feeds-rw)

eng/common/cross/toolchain.cmake

@@ -40,7 +40,7 @@ if(TARGET_ARCH_NAME STREQUAL "arm")
     set(TOOLCHAIN "arm-linux-gnueabihf")
   endif()
   if(TIZEN)
-    set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0")
+    set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "arm64")
   set(CMAKE_SYSTEM_PROCESSOR aarch64)
@@ -49,7 +49,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
   elseif(LINUX)
     set(TOOLCHAIN "aarch64-linux-gnu")
     if(TIZEN)
-      set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
+      set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu")
     endif()
   elseif(FREEBSD)
     set(triple "aarch64-unknown-freebsd12")
@@ -58,7 +58,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "armel")
   set(CMAKE_SYSTEM_PROCESSOR armv7l)
   set(TOOLCHAIN "arm-linux-gnueabi")
   if(TIZEN)
-    set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
+    set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "armv6")
   set(CMAKE_SYSTEM_PROCESSOR armv6l)
@@ -95,7 +95,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64")
   elseif(LINUX)
     set(TOOLCHAIN "x86_64-linux-gnu")
     if(TIZEN)
-      set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0")
+      set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu")
     endif()
   elseif(FREEBSD)
     set(triple "x86_64-unknown-freebsd12")
@@ -112,7 +112,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
     set(TOOLCHAIN "i686-linux-gnu")
   endif()
   if(TIZEN)
-    set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0")
+    set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu")
   endif()
 else()
   message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!")
@@ -124,26 +124,25 @@ endif()
 
 # Specify include paths
 if(TIZEN)
-  if(TARGET_ARCH_NAME STREQUAL "arm")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "armel")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "arm64")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "x86")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "x64")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu)
+  function(find_toolchain_dir prefix)
+    # Dynamically find the version subdirectory
+    file(GLOB DIRECTORIES "${prefix}/*")
+    list(GET DIRECTORIES 0 FIRST_MATCH)
+    get_filename_component(TOOLCHAIN_VERSION ${FIRST_MATCH} NAME)
+
+    set(TIZEN_TOOLCHAIN_PATH "${prefix}/${TOOLCHAIN_VERSION}" PARENT_SCOPE)
+  endfunction()
+
+  if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
+    find_toolchain_dir("${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+  else()
+    find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
   endif()
+
+  message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}")
+
+  include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++)
+  include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN})
 endif()
 
 if(ANDROID)
@@ -265,33 +264,35 @@ endif()
 
 if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
   if(TIZEN)
-    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
-    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
   endif()
 elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
   if(TIZEN)
-    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64")
-    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
 
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
-    add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-Wl,--rpath-link=${TIZEN_TOOLCHAIN_PATH}")
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_toolchain_linker_flag("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_toolchain_linker_flag("--target=${TOOLCHAIN}")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}")
   endif()
   add_toolchain_linker_flag(-m32)
   if(TIZEN)
-    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
-    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
   endif()
 elseif(ILLUMOS)
   add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64")
@@ -328,6 +329,8 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
   if(TARGET_ARCH_NAME STREQUAL "armel")
     add_compile_options(-mfloat-abi=softfp)
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_compile_options("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_compile_options(--target=${TOOLCHAIN})

eng/common/generate-sbom-prep.ps1

@@ -4,18 +4,26 @@ Param(
 
 . $PSScriptRoot\pipeline-logging-functions.ps1
 
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM"
+$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_'
+$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName
+
+Write-Host "Artifact name before : $ArtifactName"
+Write-Host "Artifact name after : $SafeArtifactName"
+
 Write-Host "Creating dir $ManifestDirPath"
+
 # create directory for sbom manifest to be placed
-if (!(Test-Path -path $ManifestDirPath))
+if (!(Test-Path -path $SbomGenerationDir))
 {
-  New-Item -ItemType Directory -path $ManifestDirPath
-  Write-Host "Successfully created directory $ManifestDirPath"
+  New-Item -ItemType Directory -path $SbomGenerationDir
+  Write-Host "Successfully created directory $SbomGenerationDir"
 }
 else{
   Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
 }
 
 Write-Host "Updating artifact name"
-$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
-Write-Host "Artifact name $artifact_name"
-Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName"

eng/common/generate-sbom-prep.sh

@@ -14,19 +14,24 @@ done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 . $scriptroot/pipeline-logging-functions.sh
 
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
+
 manifest_dir=$1
 
-if [ ! -d "$manifest_dir" ] ; then
-  mkdir -p "$manifest_dir"
-  echo "Sbom directory created." $manifest_dir
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+sbom_generation_dir="$manifest_dir/$safe_artifact_name"
+
+if [ ! -d "$sbom_generation_dir" ] ; then
+  mkdir -p "$sbom_generation_dir"
+  echo "Sbom directory created." $sbom_generation_dir
 else
   Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
 fi
 
-artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
 echo "Artifact name before : "$artifact_name
-# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
-safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
 echo "Artifact name after : "$safe_artifact_name
 export ARTIFACT_NAME=$safe_artifact_name
 echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"

eng/common/post-build/nuget-validation.ps1

@@ -2,20 +2,13 @@
 # tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
 
 param(
-  [Parameter(Mandatory=$true)][string] $PackagesPath,           # Path to where the packages to be validated are
-  [Parameter(Mandatory=$true)][string] $ToolDestinationPath     # Where the validation tool should be downloaded to
+  [Parameter(Mandatory=$true)][string] $PackagesPath # Path to where the packages to be validated are
 )
 
 try {
   . $PSScriptRoot\post-build-utils.ps1
 
-  $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
-
-  New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
-
-  Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 
-
-  & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+  & $PSScriptRoot\nuget-verification.ps1 ${PackagesPath}\*.nupkg
 } 
 catch {
   Write-Host $_.ScriptStackTrace

eng/common/post-build/nuget-verification.ps1

@@ -0,0 +1,121 @@
+<#
+.SYNOPSIS
+    Verifies that Microsoft NuGet packages have proper metadata.
+.DESCRIPTION
+    Downloads a verification tool and runs metadata validation on the provided NuGet packages. This script writes an
+    error if any of the provided packages fail validation. All arguments provided to this PowerShell script that do not
+    match PowerShell parameters are passed on to the verification tool downloaded during the execution of this script.
+.PARAMETER NuGetExePath
+    The path to the nuget.exe binary to use. If not provided, nuget.exe will be downloaded into the -DownloadPath
+    directory.
+.PARAMETER PackageSource
+    The package source to use to download the verification tool. If not provided, nuget.org will be used.
+.PARAMETER DownloadPath
+    The directory path to download the verification tool and nuget.exe to. If not provided,
+    %TEMP%\NuGet.VerifyNuGetPackage will be used.
+.PARAMETER args
+    Arguments that will be passed to the verification tool.
+.EXAMPLE
+    PS> .\verify.ps1 *.nupkg
+    Verifies the metadata of all .nupkg files in the currect working directory.
+.EXAMPLE
+    PS> .\verify.ps1 --help
+    Displays the help text of the downloaded verifiction tool.
+.LINK
+    https://github.com/NuGet/NuGetGallery/blob/master/src/VerifyMicrosoftPackage/README.md
+#>
+
+# This script was copied from https://github.com/NuGet/NuGetGallery/blob/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1
+
+[CmdletBinding(PositionalBinding = $false)]
+param(
+   [string]$NuGetExePath,
+   [string]$PackageSource = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json",
+   [string]$DownloadPath,
+   [Parameter(ValueFromRemainingArguments = $true)]
+   [string[]]$args
+)
+
+# The URL to download nuget.exe.
+$nugetExeUrl = "https://dist.nuget.org/win-x86-commandline/v4.9.4/nuget.exe"
+
+# The package ID of the verification tool.
+$packageId = "NuGet.VerifyMicrosoftPackage"
+
+# The location that nuget.exe and the verification tool will be downloaded to.
+if (!$DownloadPath) {
+    $DownloadPath = (Join-Path $env:TEMP "NuGet.VerifyMicrosoftPackage")
+}
+
+$fence = New-Object -TypeName string -ArgumentList '=', 80
+
+# Create the download directory, if it doesn't already exist.
+if (!(Test-Path $DownloadPath)) {
+    New-Item -ItemType Directory $DownloadPath | Out-Null
+}
+Write-Host "Using download path: $DownloadPath"
+
+if ($NuGetExePath) {
+    $nuget = $NuGetExePath
+} else {
+    $downloadedNuGetExe = Join-Path $DownloadPath "nuget.exe"
+
+    # Download nuget.exe, if it doesn't already exist.
+    if (!(Test-Path $downloadedNuGetExe)) {
+        Write-Host "Downloading nuget.exe from $nugetExeUrl..."
+        $ProgressPreference = 'SilentlyContinue'
+        try {
+            Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe
+            $ProgressPreference = 'Continue'
+        } catch {
+            $ProgressPreference = 'Continue'
+            Write-Error $_
+            Write-Error "nuget.exe failed to download."
+            exit
+        }
+    }
+
+    $nuget = $downloadedNuGetExe
+}
+
+Write-Host "Using nuget.exe path: $nuget"
+Write-Host " "
+
+# Download the latest version of the verification tool.
+Write-Host "Downloading the latest version of $packageId from $packageSource..."
+Write-Host $fence
+& $nuget install $packageId `
+    -Prerelease `
+    -OutputDirectory $DownloadPath `
+    -Source $PackageSource
+Write-Host $fence
+Write-Host " "
+
+if ($LASTEXITCODE -ne 0) {
+    Write-Error "nuget.exe failed to fetch the verify tool."
+    exit
+}
+
+# Find the most recently downloaded tool
+Write-Host "Finding the most recently downloaded verification tool."
+$verifyProbePath = Join-Path $DownloadPath "$packageId.*"
+$verifyPath = Get-ChildItem -Path $verifyProbePath -Directory `
+    | Sort-Object -Property LastWriteTime -Descending `
+    | Select-Object -First 1
+$verify = Join-Path $verifyPath "tools\NuGet.VerifyMicrosoftPackage.exe"
+Write-Host "Using verification tool: $verify"
+Write-Host " "
+
+# Execute the verification tool.
+Write-Host "Executing the verify tool..."
+Write-Host $fence
+& $verify $args
+Write-Host $fence
+Write-Host " "
+
+# Respond to the exit code.
+if ($LASTEXITCODE -ne 0) {
+    Write-Error "The verify tool found some problems."
+} else {
+    Write-Output "The verify tool succeeded."
+}

eng/common/sdk-task.ps1

@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true