diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 4f1c93ddb9c5..79bd984b5df8 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dotnet.darc": { - "version": "1.1.0-beta.25209.3", + "version": "1.1.0-beta.25211.1", "commands": [ "darc" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 167ff1b49447..f07a47ed5da7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,5 +1,6 @@ + https://github.com/dotnet/templating @@ -682,13 +683,13 @@ https://github.com/dotnet/runtime 7618121e0f64098e2b81ed7c8cdf7024a1b87795 - + https://github.com/dotnet/arcade-services - 98b747c01a9f431a5676a6ecf6ab2cf272ce69ee + 8c5b42c0fb17f3bcb3b275c24fc99cea305fbef5 - + https://github.com/dotnet/arcade-services - 98b747c01a9f431a5676a6ecf6ab2cf272ce69ee + 8c5b42c0fb17f3bcb3b275c24fc99cea305fbef5 https://github.com/dotnet/scenario-tests diff --git a/eng/Versions.props b/eng/Versions.props index 99601a6d08c8..7df28cca6c58 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -84,7 +84,7 @@ - 1.1.0-beta.25209.3 + 1.1.0-beta.25211.1 diff --git a/eng/pipelines/templates/steps/vmr-pull-updates.yml b/eng/pipelines/templates/steps/vmr-pull-updates.yml index 3a59083f087f..323744f3bbd5 100644 --- a/eng/pipelines/templates/steps/vmr-pull-updates.yml +++ b/eng/pipelines/templates/steps/vmr-pull-updates.yml @@ -43,16 +43,12 @@ steps: workingDirectory: ${{ parameters.vmrPath }} - script: | - ./eng/vmr-sync.sh \ - --vmr ${{ parameters.vmrPath }} \ - --tmp $(Agent.TempDirectory) \ - --azdev-pat '$(dn-bot-all-orgs-code-r)' \ - --branch ${{ parameters.vmrBranch }} \ - --repository "sdk:${{ parameters.targetRef }}" \ - --recursive \ - --remote "sdk:$(Agent.BuildDirectory)/sdk" \ - --tpn-template $(Agent.BuildDirectory)/sdk/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt \ - --ci \ + ./eng/vmr-sync.sh \ + --vmr ${{ parameters.vmrPath }} \ + --tmp $(Agent.TempDirectory) \ + --azdev-pat '$(dn-bot-all-orgs-code-r)' \ + --branch ${{ parameters.vmrBranch }} \ + --ci \ --debug if [ "$?" -ne 0 ]; then @@ -70,14 +66,12 @@ steps: condition: eq(variables['Agent.OS'], 'Windows_NT') - powershell: | + cp .\src\VirtualMonoRepo\source-mappings.json '${{ parameters.vmrPath }}' ./eng/vmr-sync.ps1 ` -vmr ${{ parameters.vmrPath }} ` -tmp $(Agent.TempDirectory) ` -azdevPat '$(dn-bot-all-orgs-code-r)' ` -branch ${{ parameters.vmrBranch }} ` - -repository "sdk:${{ parameters.targetRef }}" ` - -recursive ` - -tpnTemplate $(Agent.BuildDirectory)/sdk/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt ` -ci ` -debugOutput diff --git a/eng/vmr-sync.ps1 b/eng/vmr-sync.ps1 index a0e24f80f7ef..0a165c49456a 100644 --- a/eng/vmr-sync.ps1 +++ b/eng/vmr-sync.ps1 @@ -1,57 +1,26 @@ <# .SYNOPSIS -This script is used for synchronizing the dotnet/dotnet VMR locally. This means pulling new -code from various repositories into the 'dotnet/dotnet' repository. +This script is used for synchronizing the current repository into a local VMR. +It pulls the current repository's code into the specified VMR directory for local testing or +Source-Build validation. .DESCRIPTION -The script is used during CI to ingest new code based on dotnet/sdk but it can also help -for reproducing potential failures during sdk's PRs, namely to fix the Source-Build. -Another usecase is to try manually synchronizing a given commit of some repo into the VMR and -trying to Source-Build the VMR. This can help when fixing the Source-Build but using a commit -from a not-yet merged branch (or fork) to test the fix will help. - -The tooling that synchronizes the VMR will need to clone the various repositories into a temporary -folder. These clones can be re-used in future synchronizations so it is advised you dedicate a -folder to this to speed up your re-runs. +The tooling used for synchronization will clone the VMR repository into a temporary folder if +it does not already exist. These clones can be reused in future synchronizations, so it is +recommended to dedicate a folder for this to speed up re-runs. .EXAMPLE - Synchronize current sdk and all dependencies into a local VMR: + Synchronize current repository into a local VMR: ./vmr-sync.ps1 -vmrDir "$HOME/repos/dotnet" -tmpDir "$HOME/repos/tmp" - Synchronize the VMR to a specific commit of dotnet/runtime using custom fork: - ./vmr-sync.ps1 ` - -repository runtime:e7e71da303af8dc97df99b098f21f526398c3943 ` - -remote runtime:https://github.com/yourfork/runtime ` - -tmpDir "$HOME/repos/tmp" - .PARAMETER tmpDir Required. Path to the temporary folder where repositories will be cloned .PARAMETER vmrBranch Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch -.PARAMETER recursive -Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml) -This is used when performing the full synchronization during sdk's CI and the final VMR sync. -Defaults to false unless no repository is supplied in which case a recursive sync of sdk is performed. - -.PARAMETER remote -Optional. Additional remote to use during the synchronization -This can be used to synchronize to a commit from a fork of the repository -Example: 'runtime:https://github.com/yourfork/runtime' - -.PARAMETER repository -Optional. Repository + git ref separated by colon to synchronize to. -This can be a specific commit, branch, tag. -If not supplied, the revision of the parent sdk repository of this script will be used (recursively). -Example: 'runtime:my-branch-name' - -.PARAMETER tpnTemplate -Optional. Template for the header of VMRs THIRD-PARTY-NOTICES file. -Defaults to src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt - .PARAMETER azdevPat Optional. Azure DevOps PAT to use for cloning private repositories. @@ -68,10 +37,7 @@ param ( [Parameter(Mandatory=$true, HelpMessage="Path to the temporary folder where repositories will be cloned")] [string][Alias('t', 'tmp')]$tmpDir, [string][Alias('b', 'branch')]$vmrBranch, - [switch]$recursive, [string]$remote, - [string][Alias('r')]$repository, - [string]$tpnTemplate = "src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt", [string]$azdevPat, [string][Alias('v', 'vmr')]$vmrDir, [switch]$ci, @@ -88,23 +54,13 @@ function Highlight { Write-Host "> $($args[0])" -ForegroundColor 'Cyan' } -$sdkDir = (Split-Path -Parent $scriptRoot) - -# If sdk is a repo, we're in an sdk and not in the dotnet/dotnet repo -if (Test-Path -Path "$sdkDir/.git" -PathType Container) { - $additionalRemotes = "sdk:$sdkDir" -} - -if ($remote) { - $additionalRemotes = "$additionalRemotes $remote" -} - $verbosity = 'verbose' if ($debugOutput) { $verbosity = 'debug' } # Validation +$sdkDir = (Split-Path -Parent $scriptRoot) if (-not (Test-Path -Path $sdkDir -PathType Container)) { Fail "Directory '$sdkDir' does not exist. Please specify the path to the dotnet/sdk repo" exit 1 @@ -115,19 +71,8 @@ if (-not $tmpDir) { exit 1 } -if (-not (Test-Path -Path $tpnTemplate -PathType Leaf)) { - Fail "File '$tpnTemplate' does not exist. Please specify a valid path to the THIRD-PARTY-NOTICES template" - exit 1 -} - # Sanitize the input -# Default when no repository is provided -if (-not $repository) { - $repository = "sdk:$(git -C $sdkDir rev-parse HEAD)" - $recursive = $true -} - if (-not $vmrDir) { $vmrDir = Join-Path $tmpDir 'dotnet' } @@ -169,32 +114,20 @@ $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" & "$dotnet" tool restore -Highlight "Starting the synchronization of '$repository'.." +Highlight "Starting the synchronization of VMR.." # Synchronize the VMR $darcArgs = ( - "darc", "vmr", "update", - "--vmr", $vmrDir, + "darc", "vmr", "forwardflow", + $vmrDir, "--tmp", $tmpDir, - "--$verbosity", - "--tpn-template", $tpnTemplate, - "--discard-patches", - "--generate-credscansuppressions", - $repository + "--$verbosity" ) -if ($recursive) { - $darcArgs += ("--recursive") -} - if ($ci) { $darcArgs += ("--ci") } -if ($additionalRemotes) { - $darcArgs += ("--additional-remotes", $additionalRemotes) -} - if ($azdevPat) { $darcArgs += ("--azdev-pat", $azdevPat) } @@ -205,7 +138,7 @@ if ($LASTEXITCODE -eq 0) { Highlight "Synchronization succeeded" } else { - Fail "Synchronization of dotnet/dotnet to '$repository' failed!" + Fail "Synchronization of repo to VMR failed!" Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." diff --git a/eng/vmr-sync.sh b/eng/vmr-sync.sh index 425bbe45995a..43fe08b63cf1 100755 --- a/eng/vmr-sync.sh +++ b/eng/vmr-sync.sh @@ -1,27 +1,16 @@ #!/bin/bash -### This script is used for synchronizing the dotnet/dotnet VMR locally. This means pulling new -### code from various repositories into the 'dotnet/dotnet' repository. +### This script is used for synchronizing the current repository into a local VMR. +### It pulls the current repository's code into the specified VMR directory for local testing or +### Source-Build validation. ### -### The script is used during CI to ingest new code based on dotnet/sdk but it can also help -### for reproducing potential failures during sdk's PRs, namely to fix the Source-Build. -### Another usecase is to try manually synchronizing a given commit of some repo into the VMR and -### trying to Source-Build the VMR. This can help when fixing the Source-Build but using a commit -### from a not-yet merged branch (or fork) to test the fix will help. -### -### The tooling that synchronizes the VMR will need to clone the various repositories into a temporary -### folder. These clones can be re-used in future synchronizations so it is advised you dedicate a -### folder to this to speed up your re-runs. +### The tooling used for synchronization will clone the VMR repository into a temporary folder if +### it does not already exist. These clones can be reused in future synchronizations, so it is +### recommended to dedicate a folder for this to speed up re-runs. ### ### USAGE: -### Synchronize current sdk and all dependencies into a local VMR: -### ./vmr-sync.sh --vmr "$HOME/repos/dotnet" --tmp "$HOME/repos/tmp" -### -### Synchronize the VMR to a specific commit of dotnet/runtime using custom fork: -### ./vmr-sync.sh \ -### --repository runtime:e7e71da303af8dc97df99b098f21f526398c3943 \ -### --remote runtime:https://github.com/yourfork/runtime \ -### --tmp "$HOME/repos/tmp" +### Synchronize current repository into a local VMR: +### ./vmr-sync.sh --tmp "$HOME/repos/tmp" "$HOME/repos/dotnet" ### ### Options: ### -t, --tmp, --tmp-dir PATH @@ -33,26 +22,11 @@ ### --debug ### Optional. Turns on the most verbose logging for the VMR tooling ### -### --recursive -### Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml) -### This is used when performing the full synchronization during sdk's CI and the final VMR sync. -### Defaults to false unless no repository is supplied in which case a recursive sync of sdk is performed. -### ### --remote name:URI ### Optional. Additional remote to use during the synchronization ### This can be used to synchronize to a commit from a fork of the repository ### Example: 'runtime:https://github.com/yourfork/runtime' ### -### -r, --repository name:GIT_REF -### Optional. Repository + git ref separated by colon to synchronize to. -### This can be a specific commit, branch, tag. -### If not supplied, the revision of the parent sdk repository of this script will be used (recursively). -### Example: 'runtime:my-branch-name' -### -### --tpn-template -### Optional. Template for the header of VMRs THIRD-PARTY-NOTICES file. -### Defaults to src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt -### ### --azdev-pat ### Optional. Azure DevOps PAT to use for cloning private repositories. ### @@ -90,25 +64,16 @@ function highlight () { } # realpath is not available in macOS 12, try horrible-but-portable workaround -sdk_dir=$(cd "$scriptroot/../"; pwd -P) +repo_dir=$(cd "$scriptroot/../"; pwd -P) tmp_dir='' vmr_dir='' vmr_branch='' -repository='' additional_remotes='' -recursive=false verbosity=verbose -tpn_template="$sdk_dir/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt" -enable_build_lookup='' azdev_pat='' ci=false -# If sdk is a repo, we're in an sdk and not in the dotnet/dotnet repo -if [[ -d "$sdk_dir/.git" ]]; then - additional_remotes="sdk:$sdk_dir" -fi - while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in @@ -124,24 +89,10 @@ while [[ $# -gt 0 ]]; do vmr_branch=$2 shift ;; - -r|--repository) - repository=$2 - shift - ;; - --recursive) - recursive=true - ;; --remote) additional_remotes="$additional_remotes $2" shift ;; - --tpn-template) - tpn_template=$2 - shift - ;; - --enable-build-lookup) - enable_build_lookup="--enable-build-lookup" - ;; --azdev-pat) azdev_pat=$2 shift @@ -168,8 +119,8 @@ done # Validation -if [[ ! -d "$sdk_dir" ]]; then - fail "Directory '$sdk_dir' does not exist. Please specify the path to the dotnet/sdk repo" +if [[ ! -d "$repo_dir" ]]; then + fail "Directory '$repo_dir' does not exist. Please specify the path to the dotnet/sdk repo" exit 1 fi @@ -178,19 +129,8 @@ if [[ -z "$tmp_dir" ]]; then exit 1 fi -if [[ ! -f "$tpn_template" ]]; then - fail "File '$tpn_template' does not exist. Please specify a valid path to the THIRD-PARTY-NOTICES template" - exit 1 -fi - # Sanitize the input -# Default when no repository is provided -if [[ -z "$repository" ]]; then - repository="sdk:$(git -C "$sdk_dir" rev-parse HEAD)" - recursive=true -fi - if [[ -z "$vmr_dir" ]]; then vmr_dir="$tmp_dir/dotnet" fi @@ -236,14 +176,9 @@ dotnetDir=$( cd $scriptroot/../.dotnet/; pwd -P ) dotnet=$dotnetDir/dotnet "$dotnet" tool restore -highlight "Starting the synchronization of '$repository'.." +highlight "Starting the synchronization of VMR.." set +e -recursive_arg='' -if [[ "$recursive" == "true" ]]; then - recursive_arg="--recursive" -fi - if [[ -n "$additional_remotes" ]]; then additional_remotes="--additional-remotes $additional_remotes" fi @@ -259,24 +194,17 @@ fi # Synchronize the VMR -"$dotnet" darc vmr update \ - --vmr "$vmr_dir" \ +"$dotnet" darc vmr forwardflow "$vmr_dir" \ --tmp "$tmp_dir" \ $azdev_pat \ --$verbosity \ - $recursive_arg \ $ci_arg \ - $additional_remotes \ - --tpn-template "$tpn_template" \ - --discard-patches \ - --generate-credscansuppressions \ - $enable_build_lookup \ - "$repository" + $additional_remotes if [[ $? == 0 ]]; then highlight "Synchronization succeeded" else - fail "Synchronization of dotnet/dotnet to '$repository' failed!" + fail "Synchronization of repo to VMR failed!" fail "'$vmr_dir' is left in its last state (re-run of this script will reset it)." fail "Please inspect the logs which contain path to the failing patch file (use --debug to get all the details)." fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." diff --git a/src/VirtualMonoRepo/source-mappings.json b/src/VirtualMonoRepo/source-mappings.json index 3c486bcab22d..415ced5c52ea 100644 --- a/src/VirtualMonoRepo/source-mappings.json +++ b/src/VirtualMonoRepo/source-mappings.json @@ -10,9 +10,9 @@ { // Location within the VMR where the source-build patches are stored // These patches are applied on top of the code synchronized into the VMR - "patchesPath": "src/sdk/src/SourceBuild/patches", + "patchesPath": "src/VirtualMonoRepo/Tasks", - "sourceMappingsPath": "src/sdk/src/VirtualMonoRepo/source-mappings.json", + "sourceMappingsPath": "source-mappings.json", "thirdPartyNoticesTemplatePath": "src/sdk/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt", @@ -33,6 +33,10 @@ } ], + // Some files are copied outside of the src/ directory into other locations + // When files in the source paths are changed, they are automatically synchronized too + "additionalMappings": [], + // These defaults are added to all mappings unless `ignoreDefaults: true` is specified // When no "include" filter is specified, "**/*" is used // The default filters do not apply to submodules