From 816c23a5a7f8bbc0a92c3c252fe31b59e9a7b06d Mon Sep 17 00:00:00 2001 From: dkurepa Date: Thu, 6 Mar 2025 14:17:22 +0100 Subject: [PATCH 1/9] Hmm --- .../VirtualMonoRepo/SourceMappingParser.cs | 22 +++++++++---------- .../Program.cs | 6 ++++- .../ReproToolConfiguration.cs | 3 ++- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/SourceMappingParser.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/SourceMappingParser.cs index 53721bc896..622104504d 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/SourceMappingParser.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/SourceMappingParser.cs @@ -4,10 +4,10 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.IO; using System.Linq; using System.Text.Json; using System.Threading.Tasks; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; #nullable enable @@ -16,6 +16,7 @@ namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; public interface ISourceMappingParser { Task> ParseMappings(string mappingFilePath); + IReadOnlyCollection ParseMappingsFromJson(string json); } /// @@ -26,23 +27,21 @@ public interface ISourceMappingParser public class SourceMappingParser : ISourceMappingParser { private readonly IVmrInfo _vmrInfo; + private readonly IFileSystem _fileSystem; - public SourceMappingParser(IVmrInfo vmrInfo) + public SourceMappingParser(IVmrInfo vmrInfo, IFileSystem fileSystem) { _vmrInfo = vmrInfo; + _fileSystem = fileSystem; } public async Task> ParseMappings(string mappingFilePath) { - var mappingFile = new FileInfo(mappingFilePath); - - if (!mappingFile.Exists) - { - throw new FileNotFoundException( - $"Failed to find {VmrInfo.SourceMappingsFileName} file. Please ensure this repo is a VMR.", - mappingFilePath); - } + return ParseMappingsFromJson(await _fileSystem.ReadAllTextAsync(mappingFilePath)); + } + public IReadOnlyCollection ParseMappingsFromJson(string json) + { var options = new JsonSerializerOptions { AllowTrailingCommas = true, @@ -50,8 +49,7 @@ public async Task> ParseMappings(string mappi ReadCommentHandling = JsonCommentHandling.Skip, }; - using var stream = File.Open(mappingFile.FullName, FileMode.Open); - var settings = await JsonSerializer.DeserializeAsync(stream, options) + var settings = JsonSerializer.Deserialize(json, options) ?? throw new Exception($"Failed to deserialize {VmrInfo.SourceMappingsFileName}"); _vmrInfo.PatchesPath = NormalizePath(settings.PatchesPath); diff --git a/tools/ProductConstructionService.ReproTool/Program.cs b/tools/ProductConstructionService.ReproTool/Program.cs index c5c907161f..9331969b9b 100644 --- a/tools/ProductConstructionService.ReproTool/Program.cs +++ b/tools/ProductConstructionService.ReproTool/Program.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.ReproTool; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; Parser.Default.ParseArguments(args) .WithParsed(o => @@ -19,8 +20,11 @@ var services = new ServiceCollection(); services.RegisterServices(o); + services.AddSingleton(); + + services.AddMultiVmrSupport(Path.GetTempPath()); var provider = services.BuildServiceProvider(); - ActivatorUtilities.CreateInstance(provider).ReproduceCodeFlow().GetAwaiter().GetResult(); + ActivatorUtilities.CreateInstance(provider).TestFlatFlow().GetAwaiter().GetResult(); }); diff --git a/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs b/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs index f3c467e7e6..452011a244 100644 --- a/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs +++ b/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs @@ -42,7 +42,8 @@ internal static ServiceCollection RegisterServices( MaestroProdUri)); services.AddSingleton(sp => ActivatorUtilities.CreateInstance(sp, "git")); services.AddSingleton(); - services.AddSingleton(PcsApiFactory.GetAnonymous(PcsLocalUri)); + services.AddKeyedSingleton("local", PcsApiFactory.GetAnonymous(PcsLocalUri)); + services.AddKeyedSingleton("prod", PcsApiFactory.GetAuthenticated("https://maestro.dot.net/", null, null, false)); services.AddSingleton(_ => new GitHubClient(new ProductHeaderValue("repro-tool")) { Credentials = new Credentials(options.GitHubToken) From b4e962e2a24723fe6509d8d7b7d3919b970a6650 Mon Sep 17 00:00:00 2001 From: dkurepa Date: Thu, 6 Mar 2025 14:17:45 +0100 Subject: [PATCH 2/9] Add new files.. --- .../FlatFlowTestOperation.cs | 227 ++++++++++++++++++ .../VmrDependencyResolver.cs | 155 ++++++++++++ 2 files changed, 382 insertions(+) create mode 100644 tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs create mode 100644 tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs diff --git a/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs new file mode 100644 index 0000000000..62f538ef49 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs @@ -0,0 +1,227 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.DotNet.ProductConstructionService.Client.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Octokit; +using GitHubClient = Octokit.GitHubClient; + +namespace ProductConstructionService.ReproTool; + +internal class FlatFlowTestOperation( + VmrDependencyResolver vmrDependencyResolver, + ILogger logger, + GitHubClient ghClient, + BuildAssetRegistryContext context, + DarcProcessManager darcProcessManager, + IBarApiClient prodBarClient, + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi) +{ + private const string MaestroAuthTestOrgName = "maestro-auth-test"; + private const string VmrForkRepoName = "dotnet"; + private const string VmrForkUri = $"https://github.com/{MaestroAuthTestOrgName}/{VmrForkRepoName}"; + private const string ProductRepoFormat = $"https://github.com/{MaestroAuthTestOrgName}/"; + private const long InstallationId = 289474; + private const string SourceMappingsPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceMappingsFileName}"; + private const string SourceManifestPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceManifestFileName}"; + private const string DarcPRBranchPrefix = "darc"; + + internal async Task TestFlatFlow() + { + await darcProcessManager.InitializeAsync(); + + var vmrDependencies = await vmrDependencyResolver.GetVmrDependenciesAsync( + "https://github.com/dotnet/dotnet", + "https://github.com/dotnet/sdk", + "main"); + + logger.LogInformation("Preparing VMR fork"); + // Sync the VMR fork branch + await SyncForkAsync("dotnet", "dotnet", "main"); + // Check if the user has the forked VMR in local DB + await AddRepositoryToBarIfMissingAsync(VmrForkUri); + + var vmrTestBranch = await CreateTmpBranchAsync(VmrForkRepoName, "main", true); + + var channelName = $"repro-{Guid.NewGuid()}"; + await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, true); + + foreach (var vmrDependency in vmrDependencies) + { + var productRepoForkUri = $"{ProductRepoFormat}{vmrDependency.Mapping.Name}"; + var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrDependency.Mapping.DefaultRemote, productRepoForkUri, vmrDependency.Mapping.DefaultRef, false); + + var latestBuild = await prodBarClient.GetLatestBuildAsync(vmrDependency.Mapping.DefaultRemote, vmrDependency.Channel.Channel.Id); + var localBuild = CreateBuildAsync( + productRepoForkUri, + productRepoTmpBranch.Value, + latestBuild.Commit, + []); + + await PrepareVmrForkAsync( + vmrTestBranch.Value, + vmrDependency.Mapping.DefaultRemote, productRepoForkUri, true); + + await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( + channel: channelName, + sourceRepo: productRepoForkUri, + targetRepo: VmrForkUri, + targetBranch: vmrTestBranch.Value, + sourceDirectory: null, + targetDirectory: vmrDependency.Mapping.Name, + skipCleanup: false); + + await darcProcessManager.AddBuildToChannelAsync(localBuild.Id, channelName, false); + + await TriggerSubscriptionAsync(testSubscription.Value); + } + } + + private async Task SyncForkAsync(string originOrg, string repoName, string branch) + { + logger.LogInformation("Syncing fork {fork} branch {branch} with upstream repo {upstream}", $"{MaestroAuthTestOrgName}/{repoName}", branch, $"{originOrg}/{repoName}"); + var reference = $"heads/{branch}"; + var upstream = await ghClient.Git.Reference.Get(originOrg, repoName, reference); + await ghClient.Git.Reference.Update(MaestroAuthTestOrgName, repoName, reference, new ReferenceUpdate(upstream.Object.Sha, true)); + } + + private async Task PrepareVmrForkAsync( + string branch, + string productRepoUri, + string productRepoForkUri, + bool skipCleanup) + { + // Fetch source mappings and source manifest files and replace the mapping for the repo we're testing on + logger.LogInformation("Updating source mappings and source manifest files in VMR fork to replace original product repo mapping with fork mapping"); + await UpdateRemoteVmrForkFileAsync(branch, productRepoUri, productRepoForkUri, SourceMappingsPath); + await UpdateRemoteVmrForkFileAsync(branch, productRepoUri, productRepoForkUri, SourceManifestPath); + } + + private async Task AddRepositoryToBarIfMissingAsync(string repositoryName) + { + if ((await context.Repositories.FirstOrDefaultAsync(repo => repo.RepositoryName == repositoryName)) == null) + { + logger.LogInformation("Repo {repo} missing in local BAR. Adding an entry for it", repositoryName); + context.Repositories.Add(new Maestro.Data.Models.Repository + { + RepositoryName = repositoryName, + InstallationId = InstallationId + }); + await context.SaveChangesAsync(); + } + } + + private async Task UpdateRemoteVmrForkFileAsync(string branch, string productRepoUri, string productRepoForkUri, string filePath) + { + logger.LogInformation("Updating file {file} on branch {branch} in the VMR fork", filePath, branch); + // Fetch remote file and replace the product repo URI with the repo we're testing on + var sourceMappingsFile = (await ghClient.Repository.Content.GetAllContentsByRef( + MaestroAuthTestOrgName, + VmrForkRepoName, + filePath, + branch)) + .FirstOrDefault() + ?? throw new Exception($"Failed to find file {SourceMappingsPath} in {MaestroAuthTestOrgName}" + + $"/{VmrForkRepoName} on branch {SourceMappingsPath}"); + + // Replace the product repo uri with the forked one + var updatedSourceMappings = sourceMappingsFile.Content.Replace(productRepoUri, productRepoForkUri); + UpdateFileRequest update = new( + $"Update {productRepoUri} source mapping", + updatedSourceMappings, + sourceMappingsFile.Sha, + branch); + + await ghClient.Repository.Content.UpdateFile( + MaestroAuthTestOrgName, + VmrForkRepoName, + filePath, + update); + } + + private async Task> CreateTmpBranchAsync(string repoName, string originalBranch, bool skipCleanup) + { + var newBranchName = $"repro/{Guid.NewGuid()}"; + logger.LogInformation("Creating temporary branch {branch} in {repo}", newBranchName, $"{MaestroAuthTestOrgName}/{repoName}"); + + var baseBranch = await ghClient.Git.Reference.Get(MaestroAuthTestOrgName, repoName, $"heads/{originalBranch}"); + var newBranch = new NewReference($"refs/heads/{newBranchName}", baseBranch.Object.Sha); + await ghClient.Git.Reference.Create(MaestroAuthTestOrgName, repoName, newBranch); + + return AsyncDisposableValue.Create(newBranchName, async () => + { + if (skipCleanup) + { + return; + } + + logger.LogInformation("Cleaning up temporary branch {branchName}", newBranchName); + await Task.Delay(1); + return; + }); + } + + private async Task> PrepareProductRepoForkAsync( + string productRepoUri, + string productRepoForkUri, + string productRepoBranch, + bool skipCleanup) + { + logger.LogInformation("Preparing product repo {repo} fork", productRepoUri); + (var name, var org) = GitRepoUrlParser.GetRepoNameAndOwner(productRepoUri); + // Check if the product repo fork already exists + var allRepos = await ghClient.Repository.GetAllForOrg(MaestroAuthTestOrgName); + + // If we already have a fork in maestro-auth-test, sync the branch we need with the source + if (allRepos.FirstOrDefault(repo => repo.HtmlUrl == productRepoForkUri) != null) + { + logger.LogInformation("Product repo fork {fork} already exists, syncing branch {branch} with source", productRepoForkUri, productRepoBranch); + await SyncForkAsync(org, name, productRepoBranch); + } + // If we don't, create a fork + else + { + logger.LogInformation("Forking product repo {source} to fork {fork}", productRepoUri, productRepoForkUri); + await ghClient.Repository.Forks.Create(org, name, new NewRepositoryFork { Organization = MaestroAuthTestOrgName }); + } + await AddRepositoryToBarIfMissingAsync(productRepoForkUri); + + return await CreateTmpBranchAsync(name, productRepoBranch, skipCleanup); + } + + private async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, List assets) + { + logger.LogInformation("Creating a test build"); + + Build build = await localPcsApi.Builds.CreateAsync(new BuildData( + commit: commit, + azureDevOpsAccount: "test", + azureDevOpsProject: "test", + azureDevOpsBuildNumber: $"{DateTime.UtcNow:yyyyMMdd}.{new Random().Next(1, 75)}", + azureDevOpsRepository: repositoryUrl, + azureDevOpsBranch: branch, + released: false, + stable: false) + { + GitHubRepository = repositoryUrl, + GitHubBranch = branch, + Assets = assets + }); + + return build; + } + + private async Task TriggerSubscriptionAsync(string subscriptionId) + { + logger.LogInformation("Triggering subscription {subscriptionId}", subscriptionId); + await localPcsApi.Subscriptions.TriggerSubscriptionAsync(default, Guid.Parse(subscriptionId)); + } +} diff --git a/tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs b/tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs new file mode 100644 index 0000000000..941d2909f5 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs @@ -0,0 +1,155 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.DotNet.ProductConstructionService.Client.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace ProductConstructionService.ReproTool; + +internal record VmrDependency(SourceMapping Mapping, DefaultChannel Channel); + +internal class VmrDependencyResolver +{ + private readonly IProductConstructionServiceApi _pcsClient; + private readonly IGitRepoFactory _gitRepoFactory; + private readonly ISourceMappingParser _sourceMappingParser; + private readonly ILogger _logger; + + public VmrDependencyResolver( + [FromKeyedServices("prod")] IProductConstructionServiceApi pcsClient, + IGitRepoFactory gitRepoFactory, + ISourceMappingParser sourceMappingParser, + ILogger logger) + { + _pcsClient = pcsClient; + _logger = logger; + _gitRepoFactory = gitRepoFactory; + _sourceMappingParser = sourceMappingParser; + } + + public async Task> GetVmrDependenciesAsync(string vmrUri, string rootRepoUri, string branch) + { + IGitRepo vmr = _gitRepoFactory.CreateClient(vmrUri); + var sourceMappingsJson = await vmr.GetFileContentsAsync(VmrInfo.DefaultRelativeSourceMappingsPath, vmrUri, "main"); + IReadOnlyCollection sourceMappings = _sourceMappingParser.ParseMappingsFromJson(sourceMappingsJson); + + DefaultChannel sdkChannel = (await _pcsClient.DefaultChannels.ListAsync(repository: rootRepoUri, branch: branch)) + .Single(); + + var repositories = new Queue( + [ + new VmrDependency(sourceMappings.First(m => m.Name == "sdk"), sdkChannel) + ]); + + var dependencies = new List(); + + _logger.LogInformation("Analyzing the dependency tree of repositories flowing to VMR..."); + + while (repositories.TryDequeue(out var node)) + { + _logger.LogInformation(" {mapping} / {branch} / {channel}", + node.Mapping.Name, + node.Channel.Branch, + node.Channel.Channel.Name); + dependencies.Add(node); + + var incomingSubscriptions = (await _pcsClient.Subscriptions + .ListSubscriptionsAsync(targetRepository: node.Channel.Repository, enabled: true)) + .Where(s => s.TargetBranch == node.Channel.Branch) + .ToList(); + + // Check all subscriptions going to the current repository + foreach (var incoming in incomingSubscriptions) + { + var mapping = sourceMappings.FirstOrDefault(m => m.DefaultRemote.Equals(incoming.SourceRepository, StringComparison.InvariantCultureIgnoreCase)); + if (mapping == null) + { + // VMR repos only + continue; + } + + if (dependencies.Any(n => n.Mapping.Name == mapping.Name) || repositories.Any(r => r.Mapping.Name == mapping.Name)) + { + // Already processed + continue; + } + + if (incoming.SourceRepository == "https://github.com/dotnet/arcade") + { + // Arcade will be handled separately + // It also publishes to the validation channel so the look-up below won't work + continue; + } + + // Find which branch publishes to the incoming subscription + List defaultChannels = await _pcsClient.DefaultChannels.ListAsync(repository: incoming.SourceRepository); + var matchingChannels = defaultChannels + .Where(c => c.Channel.Id == incoming.Channel.Id) + .ToList(); + DefaultChannel defaultChannel; + + switch (matchingChannels.Count) + { + case 0: + _logger.LogWarning( + " No {dependency} branch publishing to channel '{channel}' for dependency of {parent}. " + + "Using default branch {ref}", + mapping.Name, + incoming.Channel.Name, + node.Mapping.Name, + mapping.DefaultRef); + defaultChannel = new DefaultChannel(0, incoming.SourceRepository, true) + { + Branch = mapping.DefaultRef, + Channel = incoming.Channel, + }; + break; + + case 1: + defaultChannel = matchingChannels.Single(); + break; + + default: + if (matchingChannels.Any(c => c.Branch == mapping.DefaultRef)) + { + defaultChannel = matchingChannels.Single(c => c.Branch == mapping.DefaultRef); + _logger.LogWarning( + " Multiple {repo} branches publishing to channel '{channel}' for dependency of {parent}. " + + "Using the one that matches the default branch {ref}", + mapping.Name, + incoming.Channel.Name, + node.Mapping.Name, + mapping.DefaultRef); + } + else + { + defaultChannel = matchingChannels.First(); + _logger.LogWarning( + " Multiple {dependency} branches publishing to channel '{channel}' for dependency of {parent}. " + + "Using the first one", + mapping.Name, + incoming.Channel.Name, + node.Mapping.Name); + } + + break; + } + + repositories.Enqueue(new VmrDependency(mapping, defaultChannel)); + } + } + + _logger.LogInformation("Found {count} repositories flowing to VMR", dependencies.Count); + foreach (var missing in sourceMappings.Where(m => !dependencies.Any(d => d.Mapping.Name == m.Name))) + { + _logger.LogWarning("Repository {mapping} not found in the dependency tree", missing.Name); + } + + return dependencies; + } +} From eb4d650db70dc5f7d442175e2b9cc643f3a745df Mon Sep 17 00:00:00 2001 From: dkurepa Date: Fri, 7 Mar 2025 12:13:46 +0100 Subject: [PATCH 3/9] changes --- .../FlatFlowTestOperation.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs index 62f538ef49..f701711780 100644 --- a/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs +++ b/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs @@ -77,9 +77,10 @@ await PrepareVmrForkAsync( targetBranch: vmrTestBranch.Value, sourceDirectory: null, targetDirectory: vmrDependency.Mapping.Name, - skipCleanup: false); + skipCleanup: true); - await darcProcessManager.AddBuildToChannelAsync(localBuild.Id, channelName, false); + var testChannel = (await localPcsApi.Channels.ListChannelsAsync()).Where(channel => channel.Name == channelName).First(); + await localPcsApi.Channels.AddBuildToChannelAsync(localBuild.Id, testChannel!.Id); await TriggerSubscriptionAsync(testSubscription.Value); } From d886ae0a51d5a0d94b33bbda651b2004c3f50bb6 Mon Sep 17 00:00:00 2001 From: dkurepa Date: Wed, 12 Mar 2025 11:30:21 +0100 Subject: [PATCH 4/9] hmm --- .../FlatFlowTestOperation.cs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs index f701711780..cbbd2aa771 100644 --- a/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs +++ b/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs @@ -4,7 +4,6 @@ using Maestro.Data; using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.DarcLib.Helpers; -using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.DotNet.ProductConstructionService.Client.Models; @@ -43,38 +42,46 @@ internal async Task TestFlatFlow() "https://github.com/dotnet/sdk", "main"); + vmrDependencies = vmrDependencies.Where(d => d.Mapping.Name == "runtime").ToList(); + logger.LogInformation("Preparing VMR fork"); // Sync the VMR fork branch await SyncForkAsync("dotnet", "dotnet", "main"); // Check if the user has the forked VMR in local DB await AddRepositoryToBarIfMissingAsync(VmrForkUri); - var vmrTestBranch = await CreateTmpBranchAsync(VmrForkRepoName, "main", true); + //var vmrTestBranch = await CreateTmpBranchAsync(VmrForkRepoName, "main", true); + var vmrTestBranch = "repro/21283b94-b656-432d-95ce-cb603b39b353"; var channelName = $"repro-{Guid.NewGuid()}"; await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, true); foreach (var vmrDependency in vmrDependencies) - { + { var productRepoForkUri = $"{ProductRepoFormat}{vmrDependency.Mapping.Name}"; - var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrDependency.Mapping.DefaultRemote, productRepoForkUri, vmrDependency.Mapping.DefaultRef, false); - + if (vmrDependency.Mapping.Name == "nuget-client") + { + productRepoForkUri = $"{ProductRepoFormat}nuget.client"; + } var latestBuild = await prodBarClient.GetLatestBuildAsync(vmrDependency.Mapping.DefaultRemote, vmrDependency.Channel.Channel.Id); - var localBuild = CreateBuildAsync( + + var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrDependency.Mapping.DefaultRemote, productRepoForkUri, latestBuild.GetBranch(), false); + + var localBuild = await CreateBuildAsync( productRepoForkUri, productRepoTmpBranch.Value, latestBuild.Commit, []); await PrepareVmrForkAsync( - vmrTestBranch.Value, + vmrTestBranch, vmrDependency.Mapping.DefaultRemote, productRepoForkUri, true); await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( channel: channelName, sourceRepo: productRepoForkUri, targetRepo: VmrForkUri, - targetBranch: vmrTestBranch.Value, + targetBranch: vmrTestBranch, sourceDirectory: null, targetDirectory: vmrDependency.Mapping.Name, skipCleanup: true); @@ -192,6 +199,8 @@ private async Task> PrepareProductRepoForkAsync( { logger.LogInformation("Forking product repo {source} to fork {fork}", productRepoUri, productRepoForkUri); await ghClient.Repository.Forks.Create(org, name, new NewRepositoryFork { Organization = MaestroAuthTestOrgName }); + + await Task.Delay(TimeSpan.FromSeconds(15)); } await AddRepositoryToBarIfMissingAsync(productRepoForkUri); From 28eb736358450fc59e4e85113679ab9baccd339c Mon Sep 17 00:00:00 2001 From: dkurepa Date: Thu, 13 Mar 2025 13:38:34 +0100 Subject: [PATCH 5/9] Add Tools.Common lib, add flat flow test functionality to ReproTool --- arcade-services.sln | 15 + .../FlatFlowMigrationCli.csproj | 1 + tools/FlatFlowMigrationCli/MigrationLogger.cs | 1 + .../Operations/MigrateOperation.cs | 2 + tools/FlatFlowMigrationCli/Options/Options.cs | 1 + .../SubscriptionMigrator.cs | 1 + .../DarcProcessManager.cs | 15 +- .../Operations/FlatFlowTestOperation.cs | 75 ++++ .../Operations/FullBackflowTestOperation.cs | 79 ++++ .../Operation.cs} | 255 ++++++------- .../Operations/ReproOperation.cs | 158 ++++++++ .../Options/FlatFlowTestOptions.cs | 14 + .../Options/FullBackflowTestOptions.cs | 30 ++ .../Options.cs} | 30 +- .../ReproOptions.cs} | 16 +- ...roductConstructionService.ReproTool.csproj | 1 + .../Program.cs | 27 +- .../ReproTool.cs | 354 ------------------ .../VmrDependencyResolver.cs | 155 -------- .../Constants.cs | 4 +- tools/Tools.Common/Tools.Common.csproj | 15 + .../VmrDependencyResolver.cs | 11 +- 22 files changed, 580 insertions(+), 680 deletions(-) create mode 100644 tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs create mode 100644 tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs rename tools/ProductConstructionService.ReproTool/{FlatFlowTestOperation.cs => Operations/Operation.cs} (59%) create mode 100644 tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs create mode 100644 tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs create mode 100644 tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs rename tools/ProductConstructionService.ReproTool/{ReproToolConfiguration.cs => Options/Options.cs} (82%) rename tools/ProductConstructionService.ReproTool/{ReproToolOptions.cs => Options/ReproOptions.cs} (62%) delete mode 100644 tools/ProductConstructionService.ReproTool/ReproTool.cs delete mode 100644 tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs rename tools/{FlatFlowMigrationCli => Tools.Common}/Constants.cs (89%) create mode 100644 tools/Tools.Common/Tools.Common.csproj rename tools/{FlatFlowMigrationCli => Tools.Common}/VmrDependencyResolver.cs (95%) diff --git a/arcade-services.sln b/arcade-services.sln index db45720598..fe35956a9a 100644 --- a/arcade-services.sln +++ b/arcade-services.sln @@ -96,6 +96,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FlatFlowMigrationCli", "tools\FlatFlowMigrationCli\FlatFlowMigrationCli.csproj", "{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools.Common", "tools\Tools.Common\Tools.Common.csproj", "{6D710AEF-A872-4F96-97F1-A8B9845839AB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -514,6 +516,18 @@ Global {89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x64.Build.0 = Release|Any CPU {89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.ActiveCfg = Release|Any CPU {89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.Build.0 = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x64.ActiveCfg = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x64.Build.0 = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x86.ActiveCfg = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x86.Build.0 = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|Any CPU.Build.0 = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x64.ActiveCfg = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x64.Build.0 = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x86.ActiveCfg = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -556,6 +570,7 @@ Global {B95B12A8-FCE1-618A-CA77-134B59A5C050} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {261CB211-6023-8025-48E1-D11953F4C61C} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {89DF188B-B1FC-D3C4-A76E-019144ABA9CB} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} + {6D710AEF-A872-4F96-97F1-A8B9845839AB} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B} diff --git a/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj b/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj index f03c138338..493c75a9ef 100644 --- a/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj +++ b/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj @@ -20,5 +20,6 @@ + diff --git a/tools/FlatFlowMigrationCli/MigrationLogger.cs b/tools/FlatFlowMigrationCli/MigrationLogger.cs index 350624642d..481e2b2bac 100644 --- a/tools/FlatFlowMigrationCli/MigrationLogger.cs +++ b/tools/FlatFlowMigrationCli/MigrationLogger.cs @@ -5,6 +5,7 @@ using System.Text.Json.Serialization; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; +using Tools.Common; namespace FlatFlowMigrationCli; diff --git a/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs b/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs index 26746c9a04..2664b7c109 100644 --- a/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs +++ b/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs @@ -9,6 +9,8 @@ using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; +using Tools.Common; +using Constants = Tools.Common.Constants; namespace FlatFlowMigrationCli.Operations; diff --git a/tools/FlatFlowMigrationCli/Options/Options.cs b/tools/FlatFlowMigrationCli/Options/Options.cs index 5f4d4410c6..c3b49d6aa6 100644 --- a/tools/FlatFlowMigrationCli/Options/Options.cs +++ b/tools/FlatFlowMigrationCli/Options/Options.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Console; using ProductConstructionService.Common; +using Tools.Common; namespace FlatFlowMigrationCli.Options; diff --git a/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs b/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs index 3a1317f3bf..f61f8e6ef8 100644 --- a/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs +++ b/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs @@ -5,6 +5,7 @@ using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; +using Tools.Common; namespace FlatFlowMigrationCli; diff --git a/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs b/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs index f8ebf774d4..95337cc00f 100644 --- a/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs +++ b/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs @@ -52,7 +52,7 @@ internal async Task ExecuteAsync(IEnumerable arg _darcExePath, [ .. args, - "--bar-uri", ReproToolConfiguration.PcsLocalUri + "--bar-uri", Options.Options.PcsLocalUri ]); } @@ -135,7 +135,8 @@ public async Task> CreateSubscriptionAsync( string targetBranch, string? sourceDirectory, string? targetDirectory, - bool skipCleanup) + bool skipCleanup, + List? excludedAssets = null) { logger.LogInformation("Creating a test subscription"); @@ -143,6 +144,10 @@ public async Task> CreateSubscriptionAsync( ["--source-directory", sourceDirectory] : ["--target-directory", targetDirectory!]; + string[] excludedAssetsParameter = excludedAssets != null ? + [ "--excluded-assets", string.Join(';', excludedAssets) ] : + []; + var res = await ExecuteAsync([ "add-subscription", "--channel", channel, @@ -153,7 +158,8 @@ public async Task> CreateSubscriptionAsync( "--no-trigger", "--source-enabled", "true", "--update-frequency", "none", - .. directoryArg + .. directoryArg, + .. excludedAssetsParameter ]); Match match = Regex.Match(res.StandardOutput, "Successfully created new subscription with id '([a-f0-9-]+)'"); @@ -187,7 +193,8 @@ public async Task TriggerSubscriptionAsync(string subscr return await ExecuteAsync( [ "trigger-subscriptions", - "--ids", subscriptionId + "--ids", subscriptionId, + "-q" ]); } } diff --git a/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs new file mode 100644 index 0000000000..8fe8210991 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Tools.Common; +using GitHubClient = Octokit.GitHubClient; + +namespace ProductConstructionService.ReproTool.Operations; + +internal class FlatFlowTestOperation( + VmrDependencyResolver vmrDependencyResolver, + ILogger logger, + GitHubClient ghClient, + BuildAssetRegistryContext context, + DarcProcessManager darcProcessManager, + IBarApiClient prodBarClient, + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi) : Operation(logger, ghClient, context, localPcsApi) +{ + internal override async Task RunAsync() + { + await darcProcessManager.InitializeAsync(); + + var vmrRepos = await vmrDependencyResolver.GetVmrRepositoriesAsync( + "https://github.com/dotnet/dotnet", + "https://github.com/dotnet/sdk", + "main"); + + vmrRepos = vmrRepos.Where(d => d.Mapping.Name == "runtime").ToList(); + + var vmrTestBranch = await PrepareVmrForkAsync("main", skipCleanup: true); + + var channelName = $"repro-{Guid.NewGuid()}"; + await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, true); + + foreach (var vmrRepo in vmrRepos) + { + var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.Name}"; + if (vmrRepo.Mapping.Name == "nuget-client") + { + productRepoForkUri = $"{ProductRepoFormat}nuget.client"; + } + var latestBuild = await prodBarClient.GetLatestBuildAsync(vmrRepo.Mapping.DefaultRemote, vmrRepo.Channel.Channel.Id); + + var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrRepo.Mapping.DefaultRemote, productRepoForkUri, latestBuild.GetBranch(), false); + + var testBuild = await CreateBuildAsync( + productRepoForkUri, + productRepoTmpBranch.Value, + latestBuild.Commit, + []); + + await UpdateVmrSourceFiles( + vmrTestBranch.Value, + vmrRepo.Mapping.DefaultRemote, + productRepoForkUri); + + await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( + channel: channelName, + sourceRepo: productRepoForkUri, + targetRepo: VmrForkUri, + targetBranch: vmrTestBranch.Value, + sourceDirectory: null, + targetDirectory: vmrRepo.Mapping.Name, + skipCleanup: true); + + await darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, skipCleanup: true); + + await TriggerSubscriptionAsync(testSubscription.Value); + } + } +} diff --git a/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs new file mode 100644 index 0000000000..f8fc5ca131 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.DotNet.ProductConstructionService.Client.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ProductConstructionService.ReproTool.Options; +using Tools.Common; +using GitHubClient = Octokit.GitHubClient; + +namespace ProductConstructionService.ReproTool.Operations; +internal class FullBackflowTestOperation : Operation +{ + private readonly IBarApiClient _prodBarClient; + private readonly FullBackflowTestOptions _options; + private readonly DarcProcessManager _darcProcessManager; + private readonly VmrDependencyResolver _vmrDependencyResolver; + + public FullBackflowTestOperation( + ILogger logger, + GitHubClient ghClient, + BuildAssetRegistryContext context, + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi, + IBarApiClient prodBarClient, + FullBackflowTestOptions options, + DarcProcessManager darcProcessManager, + VmrDependencyResolver vmrDependencyResolver) + : base(logger, ghClient, context, localPcsApi) + { + _prodBarClient = prodBarClient; + _options = options; + _darcProcessManager = darcProcessManager; + _vmrDependencyResolver = vmrDependencyResolver; + } + + internal override async Task RunAsync() + { + await _darcProcessManager.InitializeAsync(); + Build vmrBuild = await _prodBarClient.GetBuildAsync(_options.BuildId); + + Build testBuild = await CreateBuildAsync( + VmrForkUri, + _options.VmrBranch, + _options.Commit, + [ ..CreateAssetDataFromBuild(vmrBuild).Take(1000)]); + + var channelName = $"repro-{Guid.NewGuid()}"; + await using var channel = await _darcProcessManager.CreateTestChannelAsync(channelName, skipCleanup: true); + await _darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, skipCleanup: true); + + var vmrRepos = (await _vmrDependencyResolver.GetVmrRepositoriesAsync( + "https://github.com/dotnet/dotnet", + "https://github.com/dotnet/sdk", + "main")); + + foreach (var vmrRepo in vmrRepos) + { + var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.Name}"; + if (vmrRepo.Mapping.Name == "nuget-client") + { + productRepoForkUri = $"{ProductRepoFormat}nuget.client"; + } + + var subscription = await _darcProcessManager.CreateSubscriptionAsync( + channel: channelName, + sourceRepo: VmrForkUri, + targetRepo: productRepoForkUri, + targetBranch: _options.TargetBranch, + sourceDirectory: vmrRepo.Mapping.Name, + targetDirectory: null, + skipCleanup: true); + + await _darcProcessManager.TriggerSubscriptionAsync(subscription.Value); + } + } +} diff --git a/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/Operation.cs similarity index 59% rename from tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs rename to tools/ProductConstructionService.ReproTool/Operations/Operation.cs index cbbd2aa771..10d698377b 100644 --- a/tools/ProductConstructionService.ReproTool/FlatFlowTestOperation.cs +++ b/tools/ProductConstructionService.ReproTool/Operations/Operation.cs @@ -2,110 +2,103 @@ // The .NET Foundation licenses this file to you under the MIT license. using Maestro.Data; -using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Octokit; -using GitHubClient = Octokit.GitHubClient; -namespace ProductConstructionService.ReproTool; - -internal class FlatFlowTestOperation( - VmrDependencyResolver vmrDependencyResolver, - ILogger logger, +namespace ProductConstructionService.ReproTool.Operations; +internal abstract class Operation( + ILogger logger, GitHubClient ghClient, BuildAssetRegistryContext context, - DarcProcessManager darcProcessManager, - IBarApiClient prodBarClient, - [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi) + IProductConstructionServiceApi localPcsApi) { - private const string MaestroAuthTestOrgName = "maestro-auth-test"; - private const string VmrForkRepoName = "dotnet"; - private const string VmrForkUri = $"https://github.com/{MaestroAuthTestOrgName}/{VmrForkRepoName}"; - private const string ProductRepoFormat = $"https://github.com/{MaestroAuthTestOrgName}/"; - private const long InstallationId = 289474; - private const string SourceMappingsPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceMappingsFileName}"; - private const string SourceManifestPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceManifestFileName}"; - private const string DarcPRBranchPrefix = "darc"; - - internal async Task TestFlatFlow() + protected const string MaestroAuthTestOrgName = "maestro-auth-test"; + protected const string VmrForkRepoName = "dotnet"; + protected const string VmrForkUri = $"https://github.com/{MaestroAuthTestOrgName}/{VmrForkRepoName}"; + protected const string ProductRepoFormat = $"https://github.com/{MaestroAuthTestOrgName}/"; + protected const long InstallationId = 289474; + protected const string SourceMappingsPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceMappingsFileName}"; + protected const string SourceManifestPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceManifestFileName}"; + protected const string DarcPRBranchPrefix = "darc"; + + internal abstract Task RunAsync(); + + protected async Task DeleteDarcPRBranchAsync(string repo, string targetBranch) { - await darcProcessManager.InitializeAsync(); - - var vmrDependencies = await vmrDependencyResolver.GetVmrDependenciesAsync( - "https://github.com/dotnet/dotnet", - "https://github.com/dotnet/sdk", - "main"); - - vmrDependencies = vmrDependencies.Where(d => d.Mapping.Name == "runtime").ToList(); + var branch = (await ghClient.Repository.Branch.GetAll(MaestroAuthTestOrgName, repo)) + .FirstOrDefault(branch => branch.Name.StartsWith($"{DarcPRBranchPrefix}-{targetBranch}")); - logger.LogInformation("Preparing VMR fork"); - // Sync the VMR fork branch - await SyncForkAsync("dotnet", "dotnet", "main"); - // Check if the user has the forked VMR in local DB - await AddRepositoryToBarIfMissingAsync(VmrForkUri); - - //var vmrTestBranch = await CreateTmpBranchAsync(VmrForkRepoName, "main", true); - var vmrTestBranch = "repro/21283b94-b656-432d-95ce-cb603b39b353"; + if (branch == null) + { + logger.LogWarning("Couldn't find darc PR branch targeting branch {targetBranch}", targetBranch); + } + else + { + await DeleteGitHubBranchAsync(repo, branch.Name); + } + } - var channelName = $"repro-{Guid.NewGuid()}"; - await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, true); + private async Task DeleteGitHubBranchAsync(string repo, string branch) => await ghClient.Git.Reference.Delete(MaestroAuthTestOrgName, repo, $"heads/{branch}"); - foreach (var vmrDependency in vmrDependencies) - { - var productRepoForkUri = $"{ProductRepoFormat}{vmrDependency.Mapping.Name}"; - if (vmrDependency.Mapping.Name == "nuget-client") + protected async Task AddRepositoryToBarIfMissingAsync(string repositoryName) + { + if (await context.Repositories.FirstOrDefaultAsync(repo => repo.RepositoryName == repositoryName) == null) + { + logger.LogInformation("Repo {repo} missing in local BAR. Adding an entry for it", repositoryName); + context.Repositories.Add(new Maestro.Data.Models.Repository { - productRepoForkUri = $"{ProductRepoFormat}nuget.client"; - } - var latestBuild = await prodBarClient.GetLatestBuildAsync(vmrDependency.Mapping.DefaultRemote, vmrDependency.Channel.Channel.Id); - - var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrDependency.Mapping.DefaultRemote, productRepoForkUri, latestBuild.GetBranch(), false); - - var localBuild = await CreateBuildAsync( - productRepoForkUri, - productRepoTmpBranch.Value, - latestBuild.Commit, - []); + RepositoryName = repositoryName, + InstallationId = InstallationId + }); + await context.SaveChangesAsync(); + } + } - await PrepareVmrForkAsync( - vmrTestBranch, - vmrDependency.Mapping.DefaultRemote, productRepoForkUri, true); + protected async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, List assets) + { + logger.LogInformation("Creating a test build"); - await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( - channel: channelName, - sourceRepo: productRepoForkUri, - targetRepo: VmrForkUri, - targetBranch: vmrTestBranch, - sourceDirectory: null, - targetDirectory: vmrDependency.Mapping.Name, - skipCleanup: true); + Build build = await localPcsApi.Builds.CreateAsync(new BuildData( + commit: commit, + azureDevOpsAccount: "test", + azureDevOpsProject: "test", + azureDevOpsBuildNumber: $"{DateTime.UtcNow:yyyyMMdd}.{new Random().Next(1, 75)}", + azureDevOpsRepository: repositoryUrl, + azureDevOpsBranch: branch, + released: false, + stable: false) + { + GitHubRepository = repositoryUrl, + GitHubBranch = branch, + Assets = assets + }); - var testChannel = (await localPcsApi.Channels.ListChannelsAsync()).Where(channel => channel.Name == channelName).First(); - await localPcsApi.Channels.AddBuildToChannelAsync(localBuild.Id, testChannel!.Id); + return build; + } - await TriggerSubscriptionAsync(testSubscription.Value); - } + protected async Task TriggerSubscriptionAsync(string subscriptionId) + { + logger.LogInformation("Triggering subscription {subscriptionId}", subscriptionId); + await localPcsApi.Subscriptions.TriggerSubscriptionAsync(default, Guid.Parse(subscriptionId)); } - private async Task SyncForkAsync(string originOrg, string repoName, string branch) + protected async Task> PrepareVmrForkAsync(string branch, bool skipCleanup) { - logger.LogInformation("Syncing fork {fork} branch {branch} with upstream repo {upstream}", $"{MaestroAuthTestOrgName}/{repoName}", branch, $"{originOrg}/{repoName}"); - var reference = $"heads/{branch}"; - var upstream = await ghClient.Git.Reference.Get(originOrg, repoName, reference); - await ghClient.Git.Reference.Update(MaestroAuthTestOrgName, repoName, reference, new ReferenceUpdate(upstream.Object.Sha, true)); + logger.LogInformation("Preparing VMR fork"); + // Sync the VMR fork branch + await SyncForkAsync("dotnet", "dotnet", branch); + // Check if the user has the forked VMR in local DB + await AddRepositoryToBarIfMissingAsync(VmrForkUri); + + return await CreateTmpBranchAsync(VmrForkRepoName, branch, skipCleanup); } - private async Task PrepareVmrForkAsync( - string branch, - string productRepoUri, - string productRepoForkUri, - bool skipCleanup) + protected async Task UpdateVmrSourceFiles(string branch, string productRepoUri, string productRepoForkUri) { // Fetch source mappings and source manifest files and replace the mapping for the repo we're testing on logger.LogInformation("Updating source mappings and source manifest files in VMR fork to replace original product repo mapping with fork mapping"); @@ -113,20 +106,6 @@ private async Task PrepareVmrForkAsync( await UpdateRemoteVmrForkFileAsync(branch, productRepoUri, productRepoForkUri, SourceManifestPath); } - private async Task AddRepositoryToBarIfMissingAsync(string repositoryName) - { - if ((await context.Repositories.FirstOrDefaultAsync(repo => repo.RepositoryName == repositoryName)) == null) - { - logger.LogInformation("Repo {repo} missing in local BAR. Adding an entry for it", repositoryName); - context.Repositories.Add(new Maestro.Data.Models.Repository - { - RepositoryName = repositoryName, - InstallationId = InstallationId - }); - await context.SaveChangesAsync(); - } - } - private async Task UpdateRemoteVmrForkFileAsync(string branch, string productRepoUri, string productRepoForkUri, string filePath) { logger.LogInformation("Updating file {file} on branch {branch} in the VMR fork", filePath, branch); @@ -155,29 +134,7 @@ await ghClient.Repository.Content.UpdateFile( update); } - private async Task> CreateTmpBranchAsync(string repoName, string originalBranch, bool skipCleanup) - { - var newBranchName = $"repro/{Guid.NewGuid()}"; - logger.LogInformation("Creating temporary branch {branch} in {repo}", newBranchName, $"{MaestroAuthTestOrgName}/{repoName}"); - - var baseBranch = await ghClient.Git.Reference.Get(MaestroAuthTestOrgName, repoName, $"heads/{originalBranch}"); - var newBranch = new NewReference($"refs/heads/{newBranchName}", baseBranch.Object.Sha); - await ghClient.Git.Reference.Create(MaestroAuthTestOrgName, repoName, newBranch); - - return AsyncDisposableValue.Create(newBranchName, async () => - { - if (skipCleanup) - { - return; - } - - logger.LogInformation("Cleaning up temporary branch {branchName}", newBranchName); - await Task.Delay(1); - return; - }); - } - - private async Task> PrepareProductRepoForkAsync( + protected async Task> PrepareProductRepoForkAsync( string productRepoUri, string productRepoForkUri, string productRepoBranch, @@ -200,6 +157,7 @@ private async Task> PrepareProductRepoForkAsync( logger.LogInformation("Forking product repo {source} to fork {fork}", productRepoUri, productRepoForkUri); await ghClient.Repository.Forks.Create(org, name, new NewRepositoryFork { Organization = MaestroAuthTestOrgName }); + // The Octokit client doesn't wait for the fork to actually be created, so we should wait a bit to make sure it's there await Task.Delay(TimeSpan.FromSeconds(15)); } await AddRepositoryToBarIfMissingAsync(productRepoForkUri); @@ -207,31 +165,60 @@ private async Task> PrepareProductRepoForkAsync( return await CreateTmpBranchAsync(name, productRepoBranch, skipCleanup); } - private async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, List assets) + protected async Task SyncForkAsync(string originOrg, string repoName, string branch) { - logger.LogInformation("Creating a test build"); + logger.LogInformation("Syncing fork {fork} branch {branch} with upstream repo {upstream}", $"{MaestroAuthTestOrgName}/{repoName}", branch, $"{originOrg}/{repoName}"); + var reference = $"heads/{branch}"; + var upstream = await ghClient.Git.Reference.Get(originOrg, repoName, reference); + await ghClient.Git.Reference.Update(MaestroAuthTestOrgName, repoName, reference, new ReferenceUpdate(upstream.Object.Sha, true)); + } - Build build = await localPcsApi.Builds.CreateAsync(new BuildData( - commit: commit, - azureDevOpsAccount: "test", - azureDevOpsProject: "test", - azureDevOpsBuildNumber: $"{DateTime.UtcNow:yyyyMMdd}.{new Random().Next(1, 75)}", - azureDevOpsRepository: repositoryUrl, - azureDevOpsBranch: branch, - released: false, - stable: false) + protected async Task> CreateTmpBranchAsync(string repoName, string originalBranch, bool skipCleanup) + { + var newBranchName = $"repro/{Guid.NewGuid()}"; + logger.LogInformation("Creating temporary branch {branch} in {repo}", newBranchName, $"{MaestroAuthTestOrgName}/{repoName}"); + + var baseBranch = await ghClient.Git.Reference.Get(MaestroAuthTestOrgName, repoName, $"heads/{originalBranch}"); + var newBranch = new NewReference($"refs/heads/{newBranchName}", baseBranch.Object.Sha); + await ghClient.Git.Reference.Create(MaestroAuthTestOrgName, repoName, newBranch); + + return AsyncDisposableValue.Create(newBranchName, async () => { - GitHubRepository = repositoryUrl, - GitHubBranch = branch, - Assets = assets + if (skipCleanup) + { + return; + } + + logger.LogInformation("Cleaning up temporary branch {branchName}", newBranchName); + try + { + await DeleteGitHubBranchAsync(repoName, newBranchName); + } + catch + { + // If this throws an exception the most likely cause is that the branch was already deleted + } }); + } - return build; + protected static List CreateAssetDataFromBuild(Build build) + { + return build.Assets + .Select(asset => new AssetData(false) + { + Name = asset.Name, + Version = asset.Version, + Locations = asset.Locations?.Select(location => new AssetLocationData(location.Type) { Location = location.Location }).ToList() + }) + .ToList(); } - private async Task TriggerSubscriptionAsync(string subscriptionId) + protected async Task GetLatestCommitInBranch(string owner, string repo, string branch) { - logger.LogInformation("Triggering subscription {subscriptionId}", subscriptionId); - await localPcsApi.Subscriptions.TriggerSubscriptionAsync(default, Guid.Parse(subscriptionId)); + var reference = await ghClient.Git.Reference.Get(owner, repo, $"heads/{branch}"); + return reference.Object.Sha; } + + protected async Task GetCommit(string sourceRepoOwner, string sourceRepoName, string commit) + => await ghClient.Repository.Commit.Get(sourceRepoOwner, sourceRepoName, commit); } diff --git a/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs new file mode 100644 index 0000000000..6d474cfeb3 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs @@ -0,0 +1,158 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Octokit; +using ProductConstructionService.ReproTool.Options; +using Build = Microsoft.DotNet.ProductConstructionService.Client.Models.Build; +using GitHubClient = Octokit.GitHubClient; + +namespace ProductConstructionService.ReproTool.Operations; + +internal class ReproOperation( + IBarApiClient prodBarClient, + ReproOptions options, + BuildAssetRegistryContext context, + DarcProcessManager darcProcessManager, + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi, + GitHubClient ghClient, + ILogger logger) : Operation(logger, ghClient, context, localPcsApi) +{ + internal override async Task RunAsync() + { + logger.LogInformation("Fetching {subscriptionId} subscription from BAR", + options.Subscription); + var subscription = await prodBarClient.GetSubscriptionAsync(options.Subscription); + + if (subscription == null) + { + throw new ArgumentException($"Couldn't find subscription with subscription id {options.Subscription}"); + } + + if (!subscription.SourceEnabled) + { + throw new ArgumentException($"Subscription {options.Subscription} is not a code flow subscription"); + } + + if (!string.IsNullOrEmpty(subscription.SourceDirectory) && !string.IsNullOrEmpty(subscription.TargetDirectory)) + { + throw new ArgumentException("Code flow subscription incorrectly configured: is missing SourceDirectory or TargetDirectory"); + } + + if (!string.IsNullOrEmpty(options.Commit) && options.BuildId != null) + { + throw new ArgumentException($"Only one of {nameof(ReproOptions.Commit)} and {nameof(ReproOptions.BuildId)} can be provided"); + } + + Build? build = null; + if (options.BuildId != null) + { + build = await prodBarClient.GetBuildAsync(options.BuildId.Value); + if (build.GitHubRepository != subscription.SourceRepository) + { + throw new ArgumentException($"Build {build.Id} repository {build.GitHubRepository} doesn't match the subscription source repository {subscription.SourceRepository}"); + } + } + await darcProcessManager.InitializeAsync(); + + var defaultChannel = (await prodBarClient.GetDefaultChannelsAsync(repository: subscription.SourceRepository, channel: subscription.Channel.Name)).First(); + + string vmrBranch, productRepoUri, productRepoBranch; + var isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); + if (isForwardFlow) + { + vmrBranch = subscription.TargetBranch; + productRepoUri = subscription.SourceRepository; + productRepoBranch = defaultChannel.Branch; + } + else + { + vmrBranch = defaultChannel.Branch; + productRepoUri = subscription.TargetRepository; + productRepoBranch = subscription.TargetBranch; + } + var productRepoForkUri = ProductRepoFormat + productRepoUri.Split('/', StringSplitOptions.RemoveEmptyEntries).Last(); + logger.LogInformation("Reproducing subscription from {sourceRepo} to {targetRepo}", + isForwardFlow ? productRepoForkUri : VmrForkUri, + isForwardFlow ? VmrForkUri : productRepoForkUri); + + await using var vmrTmpBranch = await PrepareVmrForkAsync(vmrBranch, options.SkipCleanup); + await UpdateVmrSourceFiles(vmrTmpBranch.Value, productRepoUri, productRepoForkUri); + + logger.LogInformation("Preparing product repo fork {productRepoFork}, branch {branch}", productRepoForkUri, productRepoBranch); + await using var productRepoTmpBranch = await PrepareProductRepoForkAsync(productRepoUri, productRepoForkUri, productRepoBranch, options.SkipCleanup); + + // Find the latest commit in the source repo to create a build from + string sourceRepoSha; + (var sourceRepoName, var sourceRepoOwner) = GitRepoUrlParser.GetRepoNameAndOwner(subscription.SourceRepository); + if (build != null) + { + sourceRepoSha = build.Commit; + } + else if (string.IsNullOrEmpty(options.Commit)) + { + sourceRepoSha = await GetLatestCommitInBranch(sourceRepoOwner, sourceRepoName, defaultChannel.Branch); + } + else + { + // Validate that the commit actually exists + try + { + await GetCommit(sourceRepoOwner, sourceRepoName, options.Commit); + } + catch (NotFoundException) + { + throw new ArgumentException($"Commit {options.Commit} doesn't exist in repo {subscription.SourceRepository}"); + } + sourceRepoSha = options.Commit; + } + + var channelName = $"repro-{Guid.NewGuid()}"; + await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, options.SkipCleanup); + + var testBuild = await CreateBuildAsync( + isForwardFlow ? productRepoForkUri : VmrForkUri, + isForwardFlow ? productRepoTmpBranch.Value : vmrTmpBranch.Value, + sourceRepoSha, + build != null ? CreateAssetDataFromBuild(build) : []); + + await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( + channel: channelName, + sourceRepo: isForwardFlow ? productRepoForkUri : VmrForkUri, + targetRepo: isForwardFlow ? VmrForkUri : productRepoForkUri, + targetBranch: isForwardFlow ? vmrTmpBranch.Value : productRepoTmpBranch.Value, + sourceDirectory: subscription.SourceDirectory, + targetDirectory: subscription.TargetDirectory, + skipCleanup: options.SkipCleanup); + + await darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, options.SkipCleanup); + + await TriggerSubscriptionAsync(testSubscription.Value); + + if (options.SkipCleanup) + { + logger.LogInformation("Skipping cleanup. If you want to re-trigger the reproduced subscription run \"darc trigger-subscriptions --ids {subscriptionId} --bar-uri {barUri}\"", + testSubscription.Value, + ProductConstructionServiceApiOptions.PcsLocalUri); + return; + } + + logger.LogInformation("Code flow successfully recreated. Press enter to finish and cleanup"); + Console.ReadLine(); + + // Cleanup + if (isForwardFlow) + { + await DeleteDarcPRBranchAsync(VmrForkRepoName, vmrTmpBranch.Value); + } + else + { + await DeleteDarcPRBranchAsync(productRepoUri.Split('/').Last(), productRepoTmpBranch.Value); + } + } +} diff --git a/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs b/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs new file mode 100644 index 0000000000..c67948e3cd --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using ProductConstructionService.ReproTool.Operations; + +namespace ProductConstructionService.ReproTool.Options; +[Verb("flat-flow-test", HelpText = "Test full flat flow in the maestro-auth-test org")] +internal class FlatFlowTestOptions : Options +{ + internal override Operation GetOperation(IServiceProvider sp) + => ActivatorUtilities.CreateInstance(sp); +} diff --git a/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs b/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs new file mode 100644 index 0000000000..3971e55324 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using ProductConstructionService.ReproTool.Operations; + +namespace ProductConstructionService.ReproTool.Options; + +[Verb("backflow-test", HelpText = "Flows an existing VMR build to all repos in maestro-auth-test")] +internal class FullBackflowTestOptions : Options +{ + [Option("build", HelpText = "Real VMR build from which we'll take assets from", Required = true)] + public required int BuildId { get; init; } + + [Option("target-branch", HelpText = "Target branch in all repos", Required = true)] + public required string TargetBranch { get; init; } + + [Option("vmr-branch", HelpText = "Vmr branch from which to backflow", Required = true)] + public required string VmrBranch { get; init; } + + [Option("commit", HelpText = "maestro-auth-test/dotnet commit to flow", Required = true)] + public required string Commit { get; init; } + + internal override Operation GetOperation(IServiceProvider sp) + => ActivatorUtilities.CreateInstance(sp, this); +} diff --git a/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs b/tools/ProductConstructionService.ReproTool/Options/Options.cs similarity index 82% rename from tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs rename to tools/ProductConstructionService.ReproTool/Options/Options.cs index 53a9cc931b..a1eb4b8445 100644 --- a/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs +++ b/tools/ProductConstructionService.ReproTool/Options/Options.cs @@ -1,35 +1,39 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using CommandLine; using Maestro.Data; -using Microsoft.DotNet.DarcLib.Helpers; +using Maestro.DataProviders; using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.DotNet.Kusto; +using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; -using Maestro.DataProviders; -using Microsoft.DotNet.Kusto; -using Microsoft.DotNet.ProductConstructionService.Client; -using Octokit; using Microsoft.Extensions.Logging.Console; +using Octokit; using ProductConstructionService.Common; +using ProductConstructionService.ReproTool.Operations; using GitHubClient = Octokit.GitHubClient; -namespace ProductConstructionService.ReproTool; -internal static class ReproToolConfiguration +namespace ProductConstructionService.ReproTool.Options; +internal abstract class Options { private const string LocalDbConnectionString = "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true"; private const string MaestroProdUri = "https://maestro.dot.net"; internal const string PcsLocalUri = "https://localhost:53180"; - internal static ServiceCollection RegisterServices( - this ServiceCollection services, - ReproToolOptions options) + [Option("github-token", HelpText = "GitHub token", Required = false)] + public string? GitHubToken { get; set; } + + internal abstract Operation GetOperation(IServiceProvider sp); + + public virtual IServiceCollection RegisterServices(IServiceCollection services) { - services.AddSingleton(options); services.AddLogging(b => b .AddConsole(o => o.FormatterName = CompactConsoleLoggerFormatter.FormatterName) .AddConsoleFormatter() @@ -44,10 +48,10 @@ internal static ServiceCollection RegisterServices( services.AddSingleton(sp => ActivatorUtilities.CreateInstance(sp, "git")); services.AddSingleton(); services.AddKeyedSingleton("local", PcsApiFactory.GetAnonymous(PcsLocalUri)); - services.AddKeyedSingleton("prod", PcsApiFactory.GetAuthenticated("https://maestro.dot.net/", null, null, false)); + services.AddSingleton(PcsApiFactory.GetAuthenticated(MaestroProdUri, null, null, false)); services.AddSingleton(_ => new GitHubClient(new ProductHeaderValue("repro-tool")) { - Credentials = new Credentials(options.GitHubToken) + Credentials = new Credentials(GitHubToken) }); services.TryAddTransient(); diff --git a/tools/ProductConstructionService.ReproTool/ReproToolOptions.cs b/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs similarity index 62% rename from tools/ProductConstructionService.ReproTool/ReproToolOptions.cs rename to tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs index f520e96a71..0ad02b6c14 100644 --- a/tools/ProductConstructionService.ReproTool/ReproToolOptions.cs +++ b/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs @@ -1,18 +1,21 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using ProductConstructionService.ReproTool.Operations; -namespace ProductConstructionService.ReproTool; +namespace ProductConstructionService.ReproTool.Options; -internal class ReproToolOptions +[Verb("repro", HelpText = "Locally reproduce a codeflow subscription in the maestro-auth-test org")] +internal class ReproOptions : Options { [Option('s', "subscription", HelpText = "Subscription that's getting reproduced", Required = true)] public required string Subscription { get; init; } - [Option("github-token", HelpText = "GitHub token", Required = false)] - public string? GitHubToken { get; set; } - [Option("commit", HelpText = "Commit to flow. Use when not flowing a build. If neither commit or build is specified, the latest commit in the subscription's source repository is flown", Required = false)] public string? Commit { get; init; } @@ -21,4 +24,7 @@ internal class ReproToolOptions [Option("skip-cleanup", HelpText = "Don't delete the created resources if they're needed for further testing. This includes the channel, subscription and PR branches. False by default", Required = false)] public bool SkipCleanup { get; init; } = false; + + internal override Operation GetOperation(IServiceProvider sp) + => ActivatorUtilities.CreateInstance(sp, this); } diff --git a/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj b/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj index 237741067c..c7686e4502 100644 --- a/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj +++ b/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj @@ -21,5 +21,6 @@ + diff --git a/tools/ProductConstructionService.ReproTool/Program.cs b/tools/ProductConstructionService.ReproTool/Program.cs index 9331969b9b..d896b6db9f 100644 --- a/tools/ProductConstructionService.ReproTool/Program.cs +++ b/tools/ProductConstructionService.ReproTool/Program.cs @@ -2,16 +2,26 @@ // The .NET Foundation licenses this file to you under the MIT license. using CommandLine; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.ReproTool; -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using ProductConstructionService.ReproTool.Operations; +using ProductConstructionService.ReproTool.Options; +using Tools.Common; -Parser.Default.ParseArguments(args) - .WithParsed(o => +Type[] options = +[ + typeof(ReproOptions), + typeof(FlatFlowTestOptions), + typeof(FullBackflowTestOptions) +]; + +Parser.Default.ParseArguments(args, options) + .MapResult((Options o) => { IConfiguration userSecrets = new ConfigurationBuilder() - .AddUserSecrets() + .AddUserSecrets() .Build(); o.GitHubToken ??= userSecrets["GITHUB_TOKEN"]; o.GitHubToken ??= Environment.GetEnvironmentVariable("GITHUB_TOKEN"); @@ -19,12 +29,15 @@ var services = new ServiceCollection(); - services.RegisterServices(o); + o.RegisterServices(services); services.AddSingleton(); services.AddMultiVmrSupport(Path.GetTempPath()); var provider = services.BuildServiceProvider(); - ActivatorUtilities.CreateInstance(provider).TestFlatFlow().GetAwaiter().GetResult(); - }); + o.GetOperation(provider).RunAsync().GetAwaiter().GetResult(); + + return 0; + }, + (_) => -1); diff --git a/tools/ProductConstructionService.ReproTool/ReproTool.cs b/tools/ProductConstructionService.ReproTool/ReproTool.cs deleted file mode 100644 index 9faf87dd98..0000000000 --- a/tools/ProductConstructionService.ReproTool/ReproTool.cs +++ /dev/null @@ -1,354 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Maestro.Data; -using Microsoft.DotNet.DarcLib; -using Microsoft.DotNet.DarcLib.Helpers; -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; -using Microsoft.DotNet.ProductConstructionService.Client; -using Microsoft.DotNet.ProductConstructionService.Client.Models; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using Octokit; -using Build = Microsoft.DotNet.ProductConstructionService.Client.Models.Build; -using BuildData = Microsoft.DotNet.ProductConstructionService.Client.Models.BuildData; -using GitHubClient = Octokit.GitHubClient; - -namespace ProductConstructionService.ReproTool; - -internal class ReproTool( - IBarApiClient prodBarClient, - ReproToolOptions options, - BuildAssetRegistryContext context, - DarcProcessManager darcProcessManager, - IProductConstructionServiceApi localPcsApi, - GitHubClient ghClient, - ILogger logger) -{ - private const string MaestroAuthTestOrgName = "maestro-auth-test"; - private const string VmrForkRepoName = "dotnet"; - private const string VmrForkUri = $"https://github.com/{MaestroAuthTestOrgName}/{VmrForkRepoName}"; - private const string ProductRepoFormat = $"https://github.com/{MaestroAuthTestOrgName}/"; - private const long InstallationId = 289474; - private const string SourceMappingsPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceMappingsFileName}"; - private const string SourceManifestPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceManifestFileName}"; - private const string DarcPRBranchPrefix = "darc"; - - internal async Task ReproduceCodeFlow() - { - logger.LogInformation("Fetching {subscriptionId} subscription from BAR", - options.Subscription); - var subscription = await prodBarClient.GetSubscriptionAsync(options.Subscription); - - if (subscription == null) - { - throw new ArgumentException($"Couldn't find subscription with subscription id {options.Subscription}"); - } - - if (!subscription.SourceEnabled) - { - throw new ArgumentException($"Subscription {options.Subscription} is not a code flow subscription"); - } - - if (!string.IsNullOrEmpty(subscription.SourceDirectory) && !string.IsNullOrEmpty(subscription.TargetDirectory)) - { - throw new ArgumentException("Code flow subscription incorrectly configured: is missing SourceDirectory or TargetDirectory"); - } - - if (!string.IsNullOrEmpty(options.Commit) && options.BuildId != null) - { - throw new ArgumentException($"Only one of {nameof(ReproToolOptions.Commit)} and {nameof(ReproToolOptions.BuildId)} can be provided"); - } - - Build? build = null; - if (options.BuildId != null) - { - build = await prodBarClient.GetBuildAsync(options.BuildId.Value); - if (build.GitHubRepository != subscription.SourceRepository) - { - throw new ArgumentException($"Build {build.Id} repository {build.GitHubRepository} doesn't match the subscription source repository {subscription.SourceRepository}"); - } - } - await darcProcessManager.InitializeAsync(); - - var defaultChannel = (await prodBarClient.GetDefaultChannelsAsync(repository: subscription.SourceRepository, channel: subscription.Channel.Name)).First(); - - string vmrBranch, productRepoUri, productRepoBranch; - bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); - if (isForwardFlow) - { - vmrBranch = subscription.TargetBranch; - productRepoUri = subscription.SourceRepository; - productRepoBranch = defaultChannel.Branch; - } - else - { - vmrBranch = defaultChannel.Branch; - productRepoUri = subscription.TargetRepository; - productRepoBranch = subscription.TargetBranch; - } - var productRepoForkUri = ProductRepoFormat + productRepoUri.Split('/', StringSplitOptions.RemoveEmptyEntries).Last(); - logger.LogInformation("Reproducing subscription from {sourceRepo} to {targetRepo}", - isForwardFlow ? productRepoForkUri : VmrForkUri, - isForwardFlow ? VmrForkUri : productRepoForkUri); - - await using var vmrTmpBranch = await PrepareVmrForkAsync(vmrBranch, productRepoUri, productRepoForkUri, options.SkipCleanup); - - logger.LogInformation("Preparing product repo fork {productRepoFork}, branch {branch}", productRepoForkUri, productRepoBranch); - await using var productRepoTmpBranch = await PrepareProductRepoForkAsync(productRepoUri, productRepoForkUri, productRepoBranch, options.SkipCleanup); - - // Find the latest commit in the source repo to create a build from - string sourceRepoSha; - (string sourceRepoName, string sourceRepoOwner) = GitRepoUrlParser.GetRepoNameAndOwner(subscription.SourceRepository); - if (build != null) - { - sourceRepoSha = build.Commit; - } - else if (string.IsNullOrEmpty(options.Commit)) - { - var res = await ghClient.Git.Reference.Get(sourceRepoOwner, sourceRepoName, $"heads/{defaultChannel.Branch}"); - sourceRepoSha = res.Object.Sha; - } - else - { - // Validate that the commit actually exists - try - { - await ghClient.Repository.Commit.Get(sourceRepoOwner, sourceRepoName, options.Commit); - } - catch (NotFoundException) - { - throw new ArgumentException($"Commit {options.Commit} doesn't exist in repo {subscription.SourceRepository}"); - } - sourceRepoSha = options.Commit; - } - - var channelName = $"repro-{Guid.NewGuid()}"; - await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, options.SkipCleanup); - - var testBuild = await CreateBuildAsync( - isForwardFlow ? productRepoForkUri : VmrForkUri, - isForwardFlow ? productRepoTmpBranch.Value : vmrTmpBranch.Value, - sourceRepoSha, - build != null ? CreateAssetDataFromBuild(build) : []); - - await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( - channel: channelName, - sourceRepo: isForwardFlow ? productRepoForkUri : VmrForkUri, - targetRepo: isForwardFlow ? VmrForkUri : productRepoForkUri, - targetBranch: isForwardFlow ? vmrTmpBranch.Value : productRepoTmpBranch.Value, - sourceDirectory: subscription.SourceDirectory, - targetDirectory: subscription.TargetDirectory, - skipCleanup: options.SkipCleanup); - - await darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, options.SkipCleanup); - - await TriggerSubscriptionAsync(testSubscription.Value); - - if (options.SkipCleanup) - { - logger.LogInformation("Skipping cleanup. If you want to re-trigger the reproduced subscription run \"darc trigger-subscriptions --ids {subscriptionId} --bar-uri {barUri}\"", - testSubscription.Value, - ProductConstructionServiceApiOptions.PcsLocalUri); - return; - } - - logger.LogInformation("Code flow successfully recreated. Press enter to finish and cleanup"); - Console.ReadLine(); - - // Cleanup - if (isForwardFlow) - { - await DeleteDarcPRBranchAsync(VmrForkRepoName, vmrTmpBranch.Value); - } - else - { - await DeleteDarcPRBranchAsync(productRepoUri.Split('/').Last(), productRepoTmpBranch.Value); - } - } - - private async Task DeleteDarcPRBranchAsync(string repo, string targetBranch) - { - var branch = (await ghClient.Repository.Branch.GetAll(MaestroAuthTestOrgName, repo)) - .FirstOrDefault(branch => branch.Name.StartsWith($"{DarcPRBranchPrefix}-{targetBranch}")); - - if (branch == null) - { - logger.LogWarning("Couldn't find darc PR branch targeting branch {targetBranch}", targetBranch); - } - else - { - await DeleteGitHubBranchAsync(repo, branch.Name); - } - } - - private async Task AddRepositoryToBarIfMissingAsync(string repositoryName) - { - if ((await context.Repositories.FirstOrDefaultAsync(repo => repo.RepositoryName == repositoryName)) == null) - { - logger.LogInformation("Repo {repo} missing in local BAR. Adding an entry for it", repositoryName); - context.Repositories.Add(new Maestro.Data.Models.Repository - { - RepositoryName = repositoryName, - InstallationId = InstallationId - }); - await context.SaveChangesAsync(); - } - } - - private async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, List assets) - { - logger.LogInformation("Creating a test build"); - - Build build = await localPcsApi.Builds.CreateAsync(new BuildData( - commit: commit, - azureDevOpsAccount: "test", - azureDevOpsProject: "test", - azureDevOpsBuildNumber: $"{DateTime.UtcNow:yyyyMMdd}.{new Random().Next(1, 75)}", - azureDevOpsRepository: repositoryUrl, - azureDevOpsBranch: branch, - released: false, - stable: false) - { - GitHubRepository = repositoryUrl, - GitHubBranch = branch, - Assets = assets - }); - - return build; - } - - private static List CreateAssetDataFromBuild(Build build) - { - return build.Assets - .Select(asset => new AssetData(false) - { - Name = asset.Name, - Version = asset.Version, - Locations = asset.Locations?.Select(location => new AssetLocationData(location.Type) { Location = location.Location}).ToList() - }) - .ToList(); - } - - private async Task TriggerSubscriptionAsync(string subscriptionId) - { - logger.LogInformation("Triggering subscription {subscriptionId}", subscriptionId); - await localPcsApi.Subscriptions.TriggerSubscriptionAsync(default, Guid.Parse(subscriptionId)); - } - - private async Task> PrepareVmrForkAsync( - string branch, - string productRepoUri, - string productRepoForkUri, - bool skipCleanup) - { - logger.LogInformation("Preparing VMR fork"); - // Sync the VMR fork branch - await SyncForkAsync("dotnet", "dotnet", branch); - // Check if the user has the forked VMR in local DB - await AddRepositoryToBarIfMissingAsync(VmrForkUri); - - var newBranch = await CreateTmpBranchAsync(VmrForkRepoName, branch, skipCleanup); - - // Fetch source mappings and source manifest files and replace the mapping for the repo we're testing on - logger.LogInformation("Updating source mappings and source manifest files in VMR fork to replace original product repo mapping with fork mapping"); - await UpdateRemoteVmrForkFileAsync(newBranch.Value, productRepoUri, productRepoForkUri, SourceMappingsPath); - await UpdateRemoteVmrForkFileAsync(newBranch.Value, productRepoUri, productRepoForkUri, SourceManifestPath); - - return newBranch; - } - - private async Task DeleteGitHubBranchAsync(string repo, string branch) => await ghClient.Git.Reference.Delete(MaestroAuthTestOrgName, repo, $"heads/{branch}"); - - private async Task UpdateRemoteVmrForkFileAsync(string branch, string productRepoUri, string productRepoForkUri, string filePath) - { - logger.LogInformation("Updating file {file} on branch {branch} in the VMR fork", filePath, branch); - // Fetch remote file and replace the product repo URI with the repo we're testing on - var sourceMappingsFile = (await ghClient.Repository.Content.GetAllContentsByRef( - MaestroAuthTestOrgName, - VmrForkRepoName, - filePath, - branch)) - .FirstOrDefault() - ?? throw new Exception($"Failed to find file {SourceMappingsPath} in {MaestroAuthTestOrgName}" + - $"/{VmrForkRepoName} on branch {SourceMappingsPath}"); - - // Replace the product repo uri with the forked one - var updatedSourceMappings = sourceMappingsFile.Content.Replace(productRepoUri, productRepoForkUri); - UpdateFileRequest update = new( - $"Update {productRepoUri} source mapping", - updatedSourceMappings, - sourceMappingsFile.Sha, - branch); - - await ghClient.Repository.Content.UpdateFile( - MaestroAuthTestOrgName, - VmrForkRepoName, - filePath, - update); - } - - private async Task> PrepareProductRepoForkAsync( - string productRepoUri, - string productRepoForkUri, - string productRepoBranch, - bool skipCleanup) - { - logger.LogInformation("Preparing product repo {repo} fork", productRepoUri); - (var name, var org) = GitRepoUrlParser.GetRepoNameAndOwner(productRepoUri); - // Check if the product repo fork already exists - var allRepos = await ghClient.Repository.GetAllForOrg(MaestroAuthTestOrgName); - - // If we already have a fork in maestro-auth-test, sync the branch we need with the source - if (allRepos.FirstOrDefault(repo => repo.HtmlUrl == productRepoForkUri) != null) - { - logger.LogInformation("Product repo fork {fork} already exists, syncing branch {branch} with source", productRepoForkUri, productRepoBranch); - await SyncForkAsync(org, name, productRepoBranch); - } - // If we don't, create a fork - else - { - logger.LogInformation("Forking product repo {source} to fork {fork}", productRepoUri, productRepoForkUri); - await ghClient.Repository.Forks.Create(org, name, new NewRepositoryFork { Organization = MaestroAuthTestOrgName }); - } - await AddRepositoryToBarIfMissingAsync(productRepoForkUri); - - return await CreateTmpBranchAsync(name, productRepoBranch, skipCleanup); - } - - private async Task SyncForkAsync(string originOrg, string repoName, string branch) - { - logger.LogInformation("Syncing fork {fork} branch {branch} with upstream repo {upstream}", $"{MaestroAuthTestOrgName}/{repoName}", branch, $"{originOrg}/{repoName}"); - var reference = $"heads/{branch}"; - var upstream = await ghClient.Git.Reference.Get(originOrg, repoName, reference); - await ghClient.Git.Reference.Update(MaestroAuthTestOrgName, repoName, reference, new ReferenceUpdate(upstream.Object.Sha, true)); - } - - private async Task> CreateTmpBranchAsync(string repoName, string originalBranch, bool skipCleanup) - { - var newBranchName = $"repro/{Guid.NewGuid()}"; - logger.LogInformation("Creating temporary branch {branch} in {repo}", newBranchName, $"{MaestroAuthTestOrgName}/{repoName}"); - - var baseBranch = await ghClient.Git.Reference.Get(MaestroAuthTestOrgName, repoName, $"heads/{originalBranch}"); - var newBranch = new NewReference($"refs/heads/{newBranchName}", baseBranch.Object.Sha); - await ghClient.Git.Reference.Create(MaestroAuthTestOrgName, repoName, newBranch); - - return AsyncDisposableValue.Create(newBranchName, async () => - { - if (skipCleanup) - { - return; - } - - logger.LogInformation("Cleaning up temporary branch {branchName}", newBranchName); - try - { - await DeleteGitHubBranchAsync(repoName, newBranchName); - } - catch - { - // If this throws an exception the most likely cause is that the branch was already deleted - } - }); - } -} diff --git a/tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs b/tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs deleted file mode 100644 index 941d2909f5..0000000000 --- a/tools/ProductConstructionService.ReproTool/VmrDependencyResolver.cs +++ /dev/null @@ -1,155 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.DotNet.DarcLib; -using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; -using Microsoft.DotNet.ProductConstructionService.Client; -using Microsoft.DotNet.ProductConstructionService.Client.Models; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace ProductConstructionService.ReproTool; - -internal record VmrDependency(SourceMapping Mapping, DefaultChannel Channel); - -internal class VmrDependencyResolver -{ - private readonly IProductConstructionServiceApi _pcsClient; - private readonly IGitRepoFactory _gitRepoFactory; - private readonly ISourceMappingParser _sourceMappingParser; - private readonly ILogger _logger; - - public VmrDependencyResolver( - [FromKeyedServices("prod")] IProductConstructionServiceApi pcsClient, - IGitRepoFactory gitRepoFactory, - ISourceMappingParser sourceMappingParser, - ILogger logger) - { - _pcsClient = pcsClient; - _logger = logger; - _gitRepoFactory = gitRepoFactory; - _sourceMappingParser = sourceMappingParser; - } - - public async Task> GetVmrDependenciesAsync(string vmrUri, string rootRepoUri, string branch) - { - IGitRepo vmr = _gitRepoFactory.CreateClient(vmrUri); - var sourceMappingsJson = await vmr.GetFileContentsAsync(VmrInfo.DefaultRelativeSourceMappingsPath, vmrUri, "main"); - IReadOnlyCollection sourceMappings = _sourceMappingParser.ParseMappingsFromJson(sourceMappingsJson); - - DefaultChannel sdkChannel = (await _pcsClient.DefaultChannels.ListAsync(repository: rootRepoUri, branch: branch)) - .Single(); - - var repositories = new Queue( - [ - new VmrDependency(sourceMappings.First(m => m.Name == "sdk"), sdkChannel) - ]); - - var dependencies = new List(); - - _logger.LogInformation("Analyzing the dependency tree of repositories flowing to VMR..."); - - while (repositories.TryDequeue(out var node)) - { - _logger.LogInformation(" {mapping} / {branch} / {channel}", - node.Mapping.Name, - node.Channel.Branch, - node.Channel.Channel.Name); - dependencies.Add(node); - - var incomingSubscriptions = (await _pcsClient.Subscriptions - .ListSubscriptionsAsync(targetRepository: node.Channel.Repository, enabled: true)) - .Where(s => s.TargetBranch == node.Channel.Branch) - .ToList(); - - // Check all subscriptions going to the current repository - foreach (var incoming in incomingSubscriptions) - { - var mapping = sourceMappings.FirstOrDefault(m => m.DefaultRemote.Equals(incoming.SourceRepository, StringComparison.InvariantCultureIgnoreCase)); - if (mapping == null) - { - // VMR repos only - continue; - } - - if (dependencies.Any(n => n.Mapping.Name == mapping.Name) || repositories.Any(r => r.Mapping.Name == mapping.Name)) - { - // Already processed - continue; - } - - if (incoming.SourceRepository == "https://github.com/dotnet/arcade") - { - // Arcade will be handled separately - // It also publishes to the validation channel so the look-up below won't work - continue; - } - - // Find which branch publishes to the incoming subscription - List defaultChannels = await _pcsClient.DefaultChannels.ListAsync(repository: incoming.SourceRepository); - var matchingChannels = defaultChannels - .Where(c => c.Channel.Id == incoming.Channel.Id) - .ToList(); - DefaultChannel defaultChannel; - - switch (matchingChannels.Count) - { - case 0: - _logger.LogWarning( - " No {dependency} branch publishing to channel '{channel}' for dependency of {parent}. " + - "Using default branch {ref}", - mapping.Name, - incoming.Channel.Name, - node.Mapping.Name, - mapping.DefaultRef); - defaultChannel = new DefaultChannel(0, incoming.SourceRepository, true) - { - Branch = mapping.DefaultRef, - Channel = incoming.Channel, - }; - break; - - case 1: - defaultChannel = matchingChannels.Single(); - break; - - default: - if (matchingChannels.Any(c => c.Branch == mapping.DefaultRef)) - { - defaultChannel = matchingChannels.Single(c => c.Branch == mapping.DefaultRef); - _logger.LogWarning( - " Multiple {repo} branches publishing to channel '{channel}' for dependency of {parent}. " + - "Using the one that matches the default branch {ref}", - mapping.Name, - incoming.Channel.Name, - node.Mapping.Name, - mapping.DefaultRef); - } - else - { - defaultChannel = matchingChannels.First(); - _logger.LogWarning( - " Multiple {dependency} branches publishing to channel '{channel}' for dependency of {parent}. " + - "Using the first one", - mapping.Name, - incoming.Channel.Name, - node.Mapping.Name); - } - - break; - } - - repositories.Enqueue(new VmrDependency(mapping, defaultChannel)); - } - } - - _logger.LogInformation("Found {count} repositories flowing to VMR", dependencies.Count); - foreach (var missing in sourceMappings.Where(m => !dependencies.Any(d => d.Mapping.Name == m.Name))) - { - _logger.LogWarning("Repository {mapping} not found in the dependency tree", missing.Name); - } - - return dependencies; - } -} diff --git a/tools/FlatFlowMigrationCli/Constants.cs b/tools/Tools.Common/Constants.cs similarity index 89% rename from tools/FlatFlowMigrationCli/Constants.cs rename to tools/Tools.Common/Constants.cs index 0ae82321e3..e866797948 100644 --- a/tools/FlatFlowMigrationCli/Constants.cs +++ b/tools/Tools.Common/Constants.cs @@ -1,9 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace FlatFlowMigrationCli; +namespace Tools.Common; -internal static class Constants +public static class Constants { public const string VmrUri = "https://github.com/dotnet/dotnet"; public const string ArcadeRepoUri = "https://github.com/dotnet/arcade"; diff --git a/tools/Tools.Common/Tools.Common.csproj b/tools/Tools.Common/Tools.Common.csproj new file mode 100644 index 0000000000..c04a57cb85 --- /dev/null +++ b/tools/Tools.Common/Tools.Common.csproj @@ -0,0 +1,15 @@ + + + + net8.0 + enable + enable + false + + + + + + + + diff --git a/tools/FlatFlowMigrationCli/VmrDependencyResolver.cs b/tools/Tools.Common/VmrDependencyResolver.cs similarity index 95% rename from tools/FlatFlowMigrationCli/VmrDependencyResolver.cs rename to tools/Tools.Common/VmrDependencyResolver.cs index da3cedfcc4..a4a8589c97 100644 --- a/tools/FlatFlowMigrationCli/VmrDependencyResolver.cs +++ b/tools/Tools.Common/VmrDependencyResolver.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using FlatFlowMigrationCli.Operations; using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.DarcLib.VirtualMonoRepo; @@ -9,22 +8,22 @@ using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; -namespace FlatFlowMigrationCli; +namespace Tools.Common; -internal record VmrRepository(SourceMapping Mapping, DefaultChannel Channel); +public record VmrRepository(SourceMapping Mapping, DefaultChannel Channel); -internal class VmrDependencyResolver +public class VmrDependencyResolver { private readonly IProductConstructionServiceApi _pcsClient; private readonly IGitRepoFactory _gitRepoFactory; private readonly ISourceMappingParser _sourceMappingParser; - private readonly ILogger _logger; + private readonly ILogger _logger; public VmrDependencyResolver( IProductConstructionServiceApi pcsClient, IGitRepoFactory gitRepoFactory, ISourceMappingParser sourceMappingParser, - ILogger logger) + ILogger logger) { _pcsClient = pcsClient; _logger = logger; From 24cd5b5684a04527a873e15f7ac8d410bb881dbd Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Thu, 13 Mar 2025 16:30:20 +0100 Subject: [PATCH 6/9] Update tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Options/FullBackflowTestOptions.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs b/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs index 3971e55324..bfc44ba5f6 100644 --- a/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs +++ b/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs @@ -1,9 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - using CommandLine; using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.ReproTool.Operations; From 5b1ae2a83f4d66418c5151608b1b498b9c2496cb Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:23:50 +0100 Subject: [PATCH 7/9] Update tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Přemek Vysoký --- .../Options/ReproOptions.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs b/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs index 0ad02b6c14..2eddeea300 100644 --- a/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs +++ b/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs @@ -1,9 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - using CommandLine; using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.ReproTool.Operations; From 881e8521d2a746c6b8877e41de74b702bc45f32c Mon Sep 17 00:00:00 2001 From: dkurepa Date: Mon, 17 Mar 2025 12:21:08 +0100 Subject: [PATCH 8/9] Address PR feedback --- .../Operations/FlatFlowTestOperation.cs | 5 +---- .../Operations/FullBackflowTestOperation.cs | 3 +-- .../Operations/Operation.cs | 20 ------------------- .../Operations/ReproOperation.cs | 3 +-- .../Options/FlatFlowTestOptions.cs | 2 +- .../Options/Options.cs | 1 - 6 files changed, 4 insertions(+), 30 deletions(-) diff --git a/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs index 8fe8210991..f245ec749a 100644 --- a/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs +++ b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs @@ -15,10 +15,9 @@ internal class FlatFlowTestOperation( VmrDependencyResolver vmrDependencyResolver, ILogger logger, GitHubClient ghClient, - BuildAssetRegistryContext context, DarcProcessManager darcProcessManager, IBarApiClient prodBarClient, - [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi) : Operation(logger, ghClient, context, localPcsApi) + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi) : Operation(logger, ghClient, localPcsApi) { internal override async Task RunAsync() { @@ -29,8 +28,6 @@ internal override async Task RunAsync() "https://github.com/dotnet/sdk", "main"); - vmrRepos = vmrRepos.Where(d => d.Mapping.Name == "runtime").ToList(); - var vmrTestBranch = await PrepareVmrForkAsync("main", skipCleanup: true); var channelName = $"repro-{Guid.NewGuid()}"; diff --git a/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs index f8fc5ca131..4f09449bf2 100644 --- a/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs +++ b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs @@ -22,13 +22,12 @@ internal class FullBackflowTestOperation : Operation public FullBackflowTestOperation( ILogger logger, GitHubClient ghClient, - BuildAssetRegistryContext context, [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi, IBarApiClient prodBarClient, FullBackflowTestOptions options, DarcProcessManager darcProcessManager, VmrDependencyResolver vmrDependencyResolver) - : base(logger, ghClient, context, localPcsApi) + : base(logger, ghClient, localPcsApi) { _prodBarClient = prodBarClient; _options = options; diff --git a/tools/ProductConstructionService.ReproTool/Operations/Operation.cs b/tools/ProductConstructionService.ReproTool/Operations/Operation.cs index 10d698377b..d9c6782262 100644 --- a/tools/ProductConstructionService.ReproTool/Operations/Operation.cs +++ b/tools/ProductConstructionService.ReproTool/Operations/Operation.cs @@ -1,12 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Maestro.Data; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.DotNet.ProductConstructionService.Client.Models; -using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Octokit; @@ -14,7 +12,6 @@ namespace ProductConstructionService.ReproTool.Operations; internal abstract class Operation( ILogger logger, GitHubClient ghClient, - BuildAssetRegistryContext context, IProductConstructionServiceApi localPcsApi) { protected const string MaestroAuthTestOrgName = "maestro-auth-test"; @@ -45,20 +42,6 @@ protected async Task DeleteDarcPRBranchAsync(string repo, string targetBranch) private async Task DeleteGitHubBranchAsync(string repo, string branch) => await ghClient.Git.Reference.Delete(MaestroAuthTestOrgName, repo, $"heads/{branch}"); - protected async Task AddRepositoryToBarIfMissingAsync(string repositoryName) - { - if (await context.Repositories.FirstOrDefaultAsync(repo => repo.RepositoryName == repositoryName) == null) - { - logger.LogInformation("Repo {repo} missing in local BAR. Adding an entry for it", repositoryName); - context.Repositories.Add(new Maestro.Data.Models.Repository - { - RepositoryName = repositoryName, - InstallationId = InstallationId - }); - await context.SaveChangesAsync(); - } - } - protected async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, List assets) { logger.LogInformation("Creating a test build"); @@ -92,8 +75,6 @@ protected async Task> PrepareVmrForkAsync(string br logger.LogInformation("Preparing VMR fork"); // Sync the VMR fork branch await SyncForkAsync("dotnet", "dotnet", branch); - // Check if the user has the forked VMR in local DB - await AddRepositoryToBarIfMissingAsync(VmrForkUri); return await CreateTmpBranchAsync(VmrForkRepoName, branch, skipCleanup); } @@ -160,7 +141,6 @@ protected async Task> PrepareProductRepoForkAsync( // The Octokit client doesn't wait for the fork to actually be created, so we should wait a bit to make sure it's there await Task.Delay(TimeSpan.FromSeconds(15)); } - await AddRepositoryToBarIfMissingAsync(productRepoForkUri); return await CreateTmpBranchAsync(name, productRepoBranch, skipCleanup); } diff --git a/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs index 6d474cfeb3..409c6c8de9 100644 --- a/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs +++ b/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs @@ -17,11 +17,10 @@ namespace ProductConstructionService.ReproTool.Operations; internal class ReproOperation( IBarApiClient prodBarClient, ReproOptions options, - BuildAssetRegistryContext context, DarcProcessManager darcProcessManager, [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi, GitHubClient ghClient, - ILogger logger) : Operation(logger, ghClient, context, localPcsApi) + ILogger logger) : Operation(logger, ghClient, localPcsApi) { internal override async Task RunAsync() { diff --git a/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs b/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs index c67948e3cd..3f6b44fdbb 100644 --- a/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs +++ b/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs @@ -6,7 +6,7 @@ using ProductConstructionService.ReproTool.Operations; namespace ProductConstructionService.ReproTool.Options; -[Verb("flat-flow-test", HelpText = "Test full flat flow in the maestro-auth-test org")] +[Verb("forward-flow-test", HelpText = "Test full flat flow in the maestro-auth-test org")] internal class FlatFlowTestOptions : Options { internal override Operation GetOperation(IServiceProvider sp) diff --git a/tools/ProductConstructionService.ReproTool/Options/Options.cs b/tools/ProductConstructionService.ReproTool/Options/Options.cs index a1eb4b8445..a11f529fdd 100644 --- a/tools/ProductConstructionService.ReproTool/Options/Options.cs +++ b/tools/ProductConstructionService.ReproTool/Options/Options.cs @@ -27,7 +27,6 @@ internal abstract class Options private const string MaestroProdUri = "https://maestro.dot.net"; internal const string PcsLocalUri = "https://localhost:53180"; - [Option("github-token", HelpText = "GitHub token", Required = false)] public string? GitHubToken { get; set; } internal abstract Operation GetOperation(IServiceProvider sp); From 485c1b064d4c8d42367bfd2ccac5456b7dbc0f94 Mon Sep 17 00:00:00 2001 From: dkurepa Date: Mon, 17 Mar 2025 12:24:32 +0100 Subject: [PATCH 9/9] Get repo name from DefaultRemote --- .../Operations/FlatFlowTestOperation.cs | 8 ++------ .../Operations/FullBackflowTestOperation.cs | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs index f245ec749a..87fb23f0aa 100644 --- a/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs +++ b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs @@ -34,12 +34,8 @@ internal override async Task RunAsync() await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, true); foreach (var vmrRepo in vmrRepos) - { - var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.Name}"; - if (vmrRepo.Mapping.Name == "nuget-client") - { - productRepoForkUri = $"{ProductRepoFormat}nuget.client"; - } + { + var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.DefaultRemote.Split('/', StringSplitOptions.RemoveEmptyEntries).Last()}"; var latestBuild = await prodBarClient.GetLatestBuildAsync(vmrRepo.Mapping.DefaultRemote, vmrRepo.Channel.Channel.Id); var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrRepo.Mapping.DefaultRemote, productRepoForkUri, latestBuild.GetBranch(), false); diff --git a/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs index 4f09449bf2..24b08e0971 100644 --- a/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs +++ b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs @@ -57,11 +57,7 @@ internal override async Task RunAsync() foreach (var vmrRepo in vmrRepos) { - var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.Name}"; - if (vmrRepo.Mapping.Name == "nuget-client") - { - productRepoForkUri = $"{ProductRepoFormat}nuget.client"; - } + var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.DefaultRemote.Split('/', StringSplitOptions.RemoveEmptyEntries).Last()}"; var subscription = await _darcProcessManager.CreateSubscriptionAsync( channel: channelName,