Fix issues with internal staging pipeline

eng/update-dependencies/BaseUrlUpdater.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Data;
 using System.Diagnostics;
 using System.Linq;
 using Microsoft.DotNet.VersionTools.Dependencies;
@@ -25,28 +26,41 @@ internal class BaseUrlUpdater : FileRegexUpdater
     /// If the base URL variable cannot be found in the manifest, the updater
     /// won't do anything.
     /// </summary>
-    public static IDependencyUpdater Create(ManifestVariables manifestVariables, SpecificCommandOptions options)
+    public static IEnumerable<IDependencyUpdater> CreateUpdaters(ManifestVariables manifestVariables, SpecificCommandOptions options)
     {
         if (manifestVariables is null)
         {
             Trace.TraceWarning("BaseUrlUpdater: manifest variables missing - skipping base URL update.");
-            return new EmptyDependencyUpdater();
+            return [];
         }
 
         var upstreamBranch = manifestVariables.GetValue("branch");
-        string baseUrlVarName = ManifestHelper.GetBaseUrlVariableName(
+        var baseUrlVarNames = ManifestHelper.GetBaseUrlVariableNames(
             dockerfileVersion: options.DockerfileVersion,
             branch: upstreamBranch,
             versionSourceName: options.VersionSourceName,
             sdkOnlyRelease: options.IsSdkOnly);
 
-        if (!manifestVariables.HasValue(baseUrlVarName))
+        IEnumerable<IDependencyUpdater> updaters = baseUrlVarNames
+            .SelectMany(variable => CreateUpdater(variable, manifestVariables, options));
+
+        return updaters;
+    }
+
+    private static IEnumerable<IDependencyUpdater> CreateUpdater(
+        string baseUrlVarName,
+        ManifestVariables manifestVariables,
+        SpecificCommandOptions options)
+    {
+        var variableHasValue = manifestVariables.HasValue(baseUrlVarName);
+
+        if (!variableHasValue)
         {
             Trace.TraceWarning($"BaseUrlUpdater: variable '{baseUrlVarName}' not found - skipping base URL update.");
-            return new EmptyDependencyUpdater();
+            return [];
         }
 
-        return new BaseUrlUpdater(options, manifestVariables, baseUrlVarName);
+        return [new BaseUrlUpdater(options, manifestVariables, baseUrlVarName)];
     }
 
     private BaseUrlUpdater(

eng/update-dependencies/DockerfileShaUpdater.cs

@@ -137,7 +137,7 @@ public static IEnumerable<IDependencyUpdater> CreateUpdaters(
         {
             usedBuildInfos = [dependencyBuildInfos.First(info => info.SimpleName == _productName)];
 
-            string baseUrl = ManifestHelper.GetBaseUrl(_manifestVariables.Variables, _options);
+            string baseUrl = ManifestHelper.GetBaseUrls(_manifestVariables.Variables, _options).First();
             // Remove Aspire Dashboard case once https://github.com/dotnet/aspire/issues/2035 is fixed.
             string archiveExt = _os.Contains("win") || _productName.Contains("aspire-dashboard") ? "zip" : "tar.gz";
             string versionDir = _buildVersion ?? "";
@@ -275,8 +275,12 @@ private static string GetArch(string[] variableParts)
             // corresponding build in the daily build location, for example, will not be signed due. So when we're targeting
             // the daily build location, we wouldn't use the release checksums file and instead use the other means of
             // retrieving the checksums.
-            string baseUrl = ManifestHelper.GetBaseUrl(_manifestVariables.Variables, _options);
-            if (baseUrl != ReleaseDotnetBaseCdnUrl)
+            string? baseUrl = ManifestHelper
+                .GetBaseUrls(_manifestVariables.Variables, _options)
+                .Where(url => url == ReleaseDotnetBaseCdnUrl)
+                .FirstOrDefault();
+
+            if (string.IsNullOrWhiteSpace(baseUrl))
             {
                 return null;
             }

eng/update-dependencies/FromStagingPipelineCommand.cs

@@ -3,6 +3,7 @@
 
 using Dotnet.Docker.Git;
 using Dotnet.Docker.Sync;
+using Microsoft.DotNet.Docker.Shared;
 using Microsoft.Extensions.Logging;
 
 namespace Dotnet.Docker;
@@ -66,12 +67,13 @@
             options.StagingPipelineRunId);
 
         string dotnetProductVersion = VersionHelper.ResolveProductVersion(releaseConfig.RuntimeBuild);
-        string dockerfileVersion = VersionHelper.ResolveMajorMinorVersion(releaseConfig.RuntimeBuild).ToString();
+        DotNetVersion dotNetVersion = DotNetVersion.Parse(releaseConfig.RuntimeBuild);
+        string majorMinorVersionString = dotNetVersion.ToString(2);
 
-        // Record pipeline run ID for this dockerfileVersion, for later use by sync-internal-release command
+        // Record pipeline run ID for this internal version, for later use by sync-internal-release command
         _internalVersionsService.RecordInternalStagingBuild(
             repoRoot: gitRepoContext.LocalRepoPath,
-            dockerfileVersion: dockerfileVersion,
+            dotNetVersion: dotNetVersion,
             stagingPipelineRunId: options.StagingPipelineRunId);
 
         var productVersions = (options.Internal, releaseConfig.SdkOnly) switch
@@ -130,7 +132,7 @@
         var updateDependenciesOptions = new SpecificCommandOptions()
         {
             RepoRoot = gitRepoContext.LocalRepoPath,
-            DockerfileVersion = dockerfileVersion.ToString(),
+            DockerfileVersion = majorMinorVersionString,
             ProductVersions = productVersions,
             InternalBaseUrl = internalBaseUrl,
         };
@@ -144,10 +146,10 @@
             return exitCode;
         }
 
-        var commitMessage = $"Update .NET {dockerfileVersion} to {productVersions["sdk"]} SDK / {productVersions["runtime"]} Runtime";
+        var commitMessage = $"Update .NET {majorMinorVersionString} to {productVersions["sdk"]} SDK / {productVersions["runtime"]} Runtime";
         var prTitle = $"[{options.TargetBranch}] {commitMessage}";
         var prBody = $"""
-            This pull request updates .NET {dockerfileVersion} to the following versions:
+            This pull request updates .NET {majorMinorVersionString} to the following versions:
 
             - SDK: {productVersions["sdk"]}
             - Runtime: {productVersions["runtime"]}
@@ -241,7 +243,7 @@
             {
                 logger.LogInformation("No git operations will be performed in {Mode} mode.", options.Mode);
                 localRepoPath = options.RepoRoot;
                 createPullRequest = async (commitMessage, prTitle, prBody) =>
                 {
                     logger.LogInformation("Skipping commit and pull request creation in {Mode} mode.", options.Mode);
                 };

eng/update-dependencies/ManifestHelper.cs

@@ -15,26 +15,30 @@ public static partial class ManifestHelper
     private const string VariablePattern = $"\\$\\((?<{VariableGroupName}>[\\w:\\-.|]+)\\)";
 
     /// <summary>
-    /// Gets the base URL based on the configured context.
+    /// Gets the base URLs based on the configured context.
     /// </summary>
     /// <param name="manifestVariables">JSON object of the variables from the manifest.</param>
     /// <param name="options">Configured options from the app.</param>
-    public static string GetBaseUrl(JObject manifestVariables, SpecificCommandOptions options)
+    public static IEnumerable<string> GetBaseUrls(JObject manifestVariables, SpecificCommandOptions options)
     {
         // The upstream branch represents which GitHub branch the current
         // branch branched off of. This is either "nightly" or "main".
         var upstreamBranch = ResolveVariableValue("branch", manifestVariables);
 
-        var baseUrlVariableName = GetBaseUrlVariableName(
+        var baseUrlVariableNames = GetBaseUrlVariableNames(
             dockerfileVersion: options.DockerfileVersion,
             branch: upstreamBranch,
             versionSourceName: options.VersionSourceName);
 
-        return ResolveVariableValue(baseUrlVariableName, manifestVariables);
+        var baseUrlValues = baseUrlVariableNames
+            .Select(variable => ResolveVariableValue(variable, manifestVariables));
+
+        return baseUrlValues;
     }
 
     /// <summary>
-    /// Constructs the name of the product version base URL variable.
+    /// Constructs the base URL variables for the given dockerfile, branch,
+    /// and product combination.
     /// </summary>
     /// <param name="dockerfileVersion">
     /// Dockerfile version. This should be a major.minor version e.g. "8.0",
@@ -43,12 +47,11 @@ public static string GetBaseUrl(JObject manifestVariables, SpecificCommandOption
     /// <param name="branch">
     /// Name of the branch. This is typically "main" or "nightly".
     /// </param>
-    public static string GetBaseUrlVariableName(
+    public static IEnumerable<string> GetBaseUrlVariableNames(
         string dockerfileVersion,
         string branch,
         string versionSourceName = "",
-        bool sdkOnlyRelease = false
-    )
+        bool sdkOnlyRelease = false)
     {
         string product;
         if (sdkOnlyRelease)
@@ -65,7 +68,10 @@ string v when v.Contains("aspire-dashboard") => "aspire-dashboard",
             };
         }
 
-        return $"{product}|{dockerfileVersion}|base-url|{branch}";
+        return [
+            $"{product}|{dockerfileVersion}|base-url|{branch}",
+            $"{product}|{dockerfileVersion}|base-url|checksums|{branch}",
+        ];
     }
 
     /// <summary>

eng/update-dependencies/SpecificCommand.cs

@@ -423,7 +423,7 @@ private static IEnumerable<IDependencyUpdater> GetProductUpdaters(ManifestVariab
             List<IDependencyUpdater> updaters =
             [
                 new NuGetConfigUpdater(manifestVariables, Options),
-                BaseUrlUpdater.Create(manifestVariables, Options)
+                ..BaseUrlUpdater.CreateUpdaters(manifestVariables, Options)
             ];
 
             foreach (string productName in Options.ProductVersions.Keys)

eng/update-dependencies/Sync/DotNetVersionExtensions.cs

@@ -0,0 +1,20 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.DotNet.Docker.Shared;
+using NuGet.Versioning;
+
+namespace Dotnet.Docker.Sync;
+
+/// <summary>
+/// <see cref="DotNetVersion"/> Extensions that are only used for release branch synchronization.
+/// </summary>
+internal static class DotNetVersionExtensions
+{
+    /// <summary>
+    /// Converts a <see cref="DotNetVersion"/> to a new <see cref="DotNetVersion"/>
+    /// with only the major and minor version parts.
+    /// </summary>
+    public static DotNetVersion ToMajorMinorVersion(this DotNetVersion version) =>
+        new DotNetVersion(new SemanticVersion(version.Major, version.Minor, 0));
+}

eng/update-dependencies/Sync/IInternalVersionsService.cs

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using Microsoft.DotNet.Docker.Shared;
+
 namespace Dotnet.Docker.Sync;
 
 /// <summary>
@@ -18,13 +20,13 @@ internal interface IInternalVersionsService
     /// Records a staging pipeline run ID in the repo.
     /// </summary>
     /// <remarks>
-    /// This will only store one staging pipeline run ID per dockerfileVersion.
+    /// This will only store one staging pipeline run ID per <paramref name="dotNetVersion"/>.
     /// If a version already exists for the same dockerfileVersion, it will be
     /// overwritten.
     /// </remarks>
-    /// <param name="dockerfileVersion">major-minor version</param>
+    /// <param name="dotNetVersion">.NET build or product version</param>
     /// <param name="stagingPipelineRunId">the build ID of the staging pipeline run</param>
-    void RecordInternalStagingBuild(string repoRoot, string dockerfileVersion, int stagingPipelineRunId);
+    void RecordInternalStagingBuild(string repoRoot, DotNetVersion dotNetVersion, int stagingPipelineRunId);
 
     /// <summary>
     /// Gets any previously recorded internal staging builds in the repo.

eng/update-dependencies/Sync/InternalStagingBuilds.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Collections.Immutable;
+using Microsoft.DotNet.Docker.Shared;
 
 namespace Dotnet.Docker.Sync;
 
@@ -12,7 +13,7 @@ namespace Dotnet.Docker.Sync;
 /// <param name="Versions">
 /// Mapping of Major.Minor .NET version to staging pipeline run ID.
 /// </param>
-internal sealed record InternalStagingBuilds(ImmutableDictionary<string, int> Versions)
+internal sealed record InternalStagingBuilds(ImmutableDictionary<DotNetVersion, int> Versions)
 {
     /// <summary>
     /// Parses <see cref=" InternalStagingBuilds"/> from lines of text.
@@ -25,7 +26,11 @@ public static InternalStagingBuilds Parse(IEnumerable<string> lines)
         var versions = lines
             .Select(line => line.Split('=', 2))
             .Where(parts => parts.Length == 2)
-            .ToImmutableDictionary(parts => parts[0], parts => int.Parse(parts[1]));
+            .ToImmutableDictionary(
+                // Reduce the version to major.minor only.
+                // If we don't, we could end up with multiple entries for the same version.
+                parts => DotNetVersion.Parse(parts[0]).ToMajorMinorVersion(),
+                parts => int.Parse(parts[1]));
 
         return new InternalStagingBuilds(versions);
     }
@@ -34,9 +39,14 @@ public static InternalStagingBuilds Parse(IEnumerable<string> lines)
     /// Returns a new <see cref="InternalStagingBuilds"/> with the specified
     /// version added.
     /// </summary>
-    public InternalStagingBuilds Add(string dockerfileVersion, int stagingPipelineRunId) =>
-        this with { Versions = Versions.SetItem(dockerfileVersion, stagingPipelineRunId) };
+    public InternalStagingBuilds Add(DotNetVersion dotNetVersion, int stagingPipelineRunId) =>
+        this with { Versions = Versions.SetItem(dotNetVersion.ToMajorMinorVersion(), stagingPipelineRunId) };
 
+    // Internal versions file should have one line per dockerfileVersion, and
+    // each line should be formatted as: <dockerfileVersion>=<stagingPipelineRunId>
     public override string ToString() =>
-        string.Join(Environment.NewLine, Versions.Select(kv => $"{kv.Key}={kv.Value}"));
+        string.Join(Environment.NewLine,
+            Versions
+                .OrderBy(kv => kv.Key)
+                .Select(kv => $"{kv.Key.ToString(2)}={kv.Value}"));
 }

eng/update-dependencies/Sync/InternalVersionsService.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Collections.Immutable;
+using Microsoft.DotNet.Docker.Shared;
 
 namespace Dotnet.Docker.Sync;
 
@@ -21,12 +22,12 @@ public InternalStagingBuilds GetInternalStagingBuilds(string repoRoot)
         }
         catch (FileNotFoundException)
         {
-            return new InternalStagingBuilds(ImmutableDictionary<string, int>.Empty);
+            return new InternalStagingBuilds(ImmutableDictionary<DotNetVersion, int>.Empty);
         }
     }
 
     /// <inheritdoc/>
-    public void RecordInternalStagingBuild(string repoRoot, string dockerfileVersion, int stagingPipelineRunId)
+    public void RecordInternalStagingBuild(string repoRoot, DotNetVersion dotNetVersion, int stagingPipelineRunId)
     {
         // Internal versions file should have one line per dockerfileVersion
         // Each line should be formatted as: <dockerfileVersion>=<stagingPipelineRunId>
@@ -38,10 +39,7 @@ public void RecordInternalStagingBuild(string repoRoot, string dockerfileVersion
         // 2) lots of regex JSON manipulation which is error-prone and harder to maintain
         //
         // So for now, the separate file and format is a compromise.
-
-        // Internal versions file should have one line per dockerfileVersion
-        // Each line should be formatted as: <dockerfileVersion>=<stagingPipelineRunId>
-        var builds = GetInternalStagingBuilds(repoRoot).Add(dockerfileVersion, stagingPipelineRunId);
+        var builds = GetInternalStagingBuilds(repoRoot).Add(dotNetVersion, stagingPipelineRunId);
         var internalVersionFile = Path.Combine(repoRoot, InternalVersionsFileName);
         File.WriteAllText(internalVersionFile, builds.ToString());
     }

tests/UpdateDependencies.Tests/InternalStagingBuildsTests.cs

@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Immutable;
+using System.Collections.Generic;
+using System.Linq;
+using Dotnet.Docker.Sync;
+using Microsoft.DotNet.Docker.Shared;
+using Shouldly;
+using Xunit;
+
+namespace UpdateDependencies.Tests;
+
+public sealed class InternalStagingBuildsTests
+{
+    [Fact]
+    public void ToStringOrdersVersionsAscending()
+    {
+        var builds = new InternalStagingBuilds(
+            ImmutableDictionary.CreateRange<DotNetVersion, int>(
+                [
+                    KeyValuePair.Create(DotNetVersion.Parse("10.0"), 1000),
+                    KeyValuePair.Create(DotNetVersion.Parse("8.0"), 800),
+                    KeyValuePair.Create(DotNetVersion.Parse("9.0"), 900)
+                ]));
+
+        var lines = builds.ToString().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
+
+        lines.ShouldBe(["8.0=800", "9.0=900", "10.0=1000"]);
+    }
+
+    [Fact]
+    public void AddReplacesEntryForSameMajorMinorVersion()
+    {
+        var builds = new InternalStagingBuilds(
+            ImmutableDictionary.CreateRange<DotNetVersion, int>(
+                [KeyValuePair.Create(DotNetVersion.Parse("8.0"), 100)]));
+
+        builds = builds.Add(DotNetVersion.Parse("8.0.101"), 200);
+        builds = builds.Add(DotNetVersion.Parse("8.0.202"), 300);
+
+        builds.Versions.Count.ShouldBe(1);
+        builds.Versions.Single().ShouldBe(KeyValuePair.Create(DotNetVersion.Parse("8.0"), 300));
+    }
+
+    [Fact]
+    public void ToStringOmitsPatchVersion()
+    {
+        var builds = new InternalStagingBuilds(ImmutableDictionary<DotNetVersion, int>.Empty);
+        builds = builds.Add(DotNetVersion.Parse("8.0.101"), 100);
+        builds = builds.Add(DotNetVersion.Parse("9.1.205-servicing.1"), 200);
+
+        foreach (var line in builds.ToString().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries))
+        {
+            var versionPart = line.Split('=')[0];
+            versionPart.Count(c => c == '.').ShouldBe(1);
+        }
+    }
+}