diff --git a/arcade-services.sln b/arcade-services.sln index db45720598..fe35956a9a 100644 --- a/arcade-services.sln +++ b/arcade-services.sln @@ -96,6 +96,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FlatFlowMigrationCli", "tools\FlatFlowMigrationCli\FlatFlowMigrationCli.csproj", "{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools.Common", "tools\Tools.Common\Tools.Common.csproj", "{6D710AEF-A872-4F96-97F1-A8B9845839AB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -514,6 +516,18 @@ Global {89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x64.Build.0 = Release|Any CPU {89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.ActiveCfg = Release|Any CPU {89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.Build.0 = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x64.ActiveCfg = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x64.Build.0 = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x86.ActiveCfg = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x86.Build.0 = Debug|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|Any CPU.Build.0 = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x64.ActiveCfg = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x64.Build.0 = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x86.ActiveCfg = Release|Any CPU + {6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -556,6 +570,7 @@ Global {B95B12A8-FCE1-618A-CA77-134B59A5C050} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {261CB211-6023-8025-48E1-D11953F4C61C} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {89DF188B-B1FC-D3C4-A76E-019144ABA9CB} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} + {6D710AEF-A872-4F96-97F1-A8B9845839AB} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B} diff --git a/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj b/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj index f03c138338..493c75a9ef 100644 --- a/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj +++ b/tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj @@ -20,5 +20,6 @@ + diff --git a/tools/FlatFlowMigrationCli/MigrationLogger.cs b/tools/FlatFlowMigrationCli/MigrationLogger.cs index 350624642d..481e2b2bac 100644 --- a/tools/FlatFlowMigrationCli/MigrationLogger.cs +++ b/tools/FlatFlowMigrationCli/MigrationLogger.cs @@ -5,6 +5,7 @@ using System.Text.Json.Serialization; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; +using Tools.Common; namespace FlatFlowMigrationCli; diff --git a/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs b/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs index 26746c9a04..2664b7c109 100644 --- a/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs +++ b/tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs @@ -9,6 +9,8 @@ using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; +using Tools.Common; +using Constants = Tools.Common.Constants; namespace FlatFlowMigrationCli.Operations; diff --git a/tools/FlatFlowMigrationCli/Options/Options.cs b/tools/FlatFlowMigrationCli/Options/Options.cs index 5f4d4410c6..c3b49d6aa6 100644 --- a/tools/FlatFlowMigrationCli/Options/Options.cs +++ b/tools/FlatFlowMigrationCli/Options/Options.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Console; using ProductConstructionService.Common; +using Tools.Common; namespace FlatFlowMigrationCli.Options; diff --git a/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs b/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs index 3a1317f3bf..f61f8e6ef8 100644 --- a/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs +++ b/tools/FlatFlowMigrationCli/SubscriptionMigrator.cs @@ -5,6 +5,7 @@ using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; +using Tools.Common; namespace FlatFlowMigrationCli; diff --git a/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs b/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs index f8ebf774d4..95337cc00f 100644 --- a/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs +++ b/tools/ProductConstructionService.ReproTool/DarcProcessManager.cs @@ -52,7 +52,7 @@ internal async Task ExecuteAsync(IEnumerable arg _darcExePath, [ .. args, - "--bar-uri", ReproToolConfiguration.PcsLocalUri + "--bar-uri", Options.Options.PcsLocalUri ]); } @@ -135,7 +135,8 @@ public async Task> CreateSubscriptionAsync( string targetBranch, string? sourceDirectory, string? targetDirectory, - bool skipCleanup) + bool skipCleanup, + List? excludedAssets = null) { logger.LogInformation("Creating a test subscription"); @@ -143,6 +144,10 @@ public async Task> CreateSubscriptionAsync( ["--source-directory", sourceDirectory] : ["--target-directory", targetDirectory!]; + string[] excludedAssetsParameter = excludedAssets != null ? + [ "--excluded-assets", string.Join(';', excludedAssets) ] : + []; + var res = await ExecuteAsync([ "add-subscription", "--channel", channel, @@ -153,7 +158,8 @@ public async Task> CreateSubscriptionAsync( "--no-trigger", "--source-enabled", "true", "--update-frequency", "none", - .. directoryArg + .. directoryArg, + .. excludedAssetsParameter ]); Match match = Regex.Match(res.StandardOutput, "Successfully created new subscription with id '([a-f0-9-]+)'"); @@ -187,7 +193,8 @@ public async Task TriggerSubscriptionAsync(string subscr return await ExecuteAsync( [ "trigger-subscriptions", - "--ids", subscriptionId + "--ids", subscriptionId, + "-q" ]); } } diff --git a/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs new file mode 100644 index 0000000000..87fb23f0aa --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Operations/FlatFlowTestOperation.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Tools.Common; +using GitHubClient = Octokit.GitHubClient; + +namespace ProductConstructionService.ReproTool.Operations; + +internal class FlatFlowTestOperation( + VmrDependencyResolver vmrDependencyResolver, + ILogger logger, + GitHubClient ghClient, + DarcProcessManager darcProcessManager, + IBarApiClient prodBarClient, + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi) : Operation(logger, ghClient, localPcsApi) +{ + internal override async Task RunAsync() + { + await darcProcessManager.InitializeAsync(); + + var vmrRepos = await vmrDependencyResolver.GetVmrRepositoriesAsync( + "https://github.com/dotnet/dotnet", + "https://github.com/dotnet/sdk", + "main"); + + var vmrTestBranch = await PrepareVmrForkAsync("main", skipCleanup: true); + + var channelName = $"repro-{Guid.NewGuid()}"; + await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, true); + + foreach (var vmrRepo in vmrRepos) + { + var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.DefaultRemote.Split('/', StringSplitOptions.RemoveEmptyEntries).Last()}"; + var latestBuild = await prodBarClient.GetLatestBuildAsync(vmrRepo.Mapping.DefaultRemote, vmrRepo.Channel.Channel.Id); + + var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrRepo.Mapping.DefaultRemote, productRepoForkUri, latestBuild.GetBranch(), false); + + var testBuild = await CreateBuildAsync( + productRepoForkUri, + productRepoTmpBranch.Value, + latestBuild.Commit, + []); + + await UpdateVmrSourceFiles( + vmrTestBranch.Value, + vmrRepo.Mapping.DefaultRemote, + productRepoForkUri); + + await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( + channel: channelName, + sourceRepo: productRepoForkUri, + targetRepo: VmrForkUri, + targetBranch: vmrTestBranch.Value, + sourceDirectory: null, + targetDirectory: vmrRepo.Mapping.Name, + skipCleanup: true); + + await darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, skipCleanup: true); + + await TriggerSubscriptionAsync(testSubscription.Value); + } + } +} diff --git a/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs new file mode 100644 index 0000000000..24b08e0971 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Operations/FullBackflowTestOperation.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.DotNet.ProductConstructionService.Client.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ProductConstructionService.ReproTool.Options; +using Tools.Common; +using GitHubClient = Octokit.GitHubClient; + +namespace ProductConstructionService.ReproTool.Operations; +internal class FullBackflowTestOperation : Operation +{ + private readonly IBarApiClient _prodBarClient; + private readonly FullBackflowTestOptions _options; + private readonly DarcProcessManager _darcProcessManager; + private readonly VmrDependencyResolver _vmrDependencyResolver; + + public FullBackflowTestOperation( + ILogger logger, + GitHubClient ghClient, + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi, + IBarApiClient prodBarClient, + FullBackflowTestOptions options, + DarcProcessManager darcProcessManager, + VmrDependencyResolver vmrDependencyResolver) + : base(logger, ghClient, localPcsApi) + { + _prodBarClient = prodBarClient; + _options = options; + _darcProcessManager = darcProcessManager; + _vmrDependencyResolver = vmrDependencyResolver; + } + + internal override async Task RunAsync() + { + await _darcProcessManager.InitializeAsync(); + Build vmrBuild = await _prodBarClient.GetBuildAsync(_options.BuildId); + + Build testBuild = await CreateBuildAsync( + VmrForkUri, + _options.VmrBranch, + _options.Commit, + [ ..CreateAssetDataFromBuild(vmrBuild).Take(1000)]); + + var channelName = $"repro-{Guid.NewGuid()}"; + await using var channel = await _darcProcessManager.CreateTestChannelAsync(channelName, skipCleanup: true); + await _darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, skipCleanup: true); + + var vmrRepos = (await _vmrDependencyResolver.GetVmrRepositoriesAsync( + "https://github.com/dotnet/dotnet", + "https://github.com/dotnet/sdk", + "main")); + + foreach (var vmrRepo in vmrRepos) + { + var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.DefaultRemote.Split('/', StringSplitOptions.RemoveEmptyEntries).Last()}"; + + var subscription = await _darcProcessManager.CreateSubscriptionAsync( + channel: channelName, + sourceRepo: VmrForkUri, + targetRepo: productRepoForkUri, + targetBranch: _options.TargetBranch, + sourceDirectory: vmrRepo.Mapping.Name, + targetDirectory: null, + skipCleanup: true); + + await _darcProcessManager.TriggerSubscriptionAsync(subscription.Value); + } + } +} diff --git a/tools/ProductConstructionService.ReproTool/Operations/Operation.cs b/tools/ProductConstructionService.ReproTool/Operations/Operation.cs new file mode 100644 index 0000000000..d9c6782262 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Operations/Operation.cs @@ -0,0 +1,204 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.DotNet.ProductConstructionService.Client.Models; +using Microsoft.Extensions.Logging; +using Octokit; + +namespace ProductConstructionService.ReproTool.Operations; +internal abstract class Operation( + ILogger logger, + GitHubClient ghClient, + IProductConstructionServiceApi localPcsApi) +{ + protected const string MaestroAuthTestOrgName = "maestro-auth-test"; + protected const string VmrForkRepoName = "dotnet"; + protected const string VmrForkUri = $"https://github.com/{MaestroAuthTestOrgName}/{VmrForkRepoName}"; + protected const string ProductRepoFormat = $"https://github.com/{MaestroAuthTestOrgName}/"; + protected const long InstallationId = 289474; + protected const string SourceMappingsPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceMappingsFileName}"; + protected const string SourceManifestPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceManifestFileName}"; + protected const string DarcPRBranchPrefix = "darc"; + + internal abstract Task RunAsync(); + + protected async Task DeleteDarcPRBranchAsync(string repo, string targetBranch) + { + var branch = (await ghClient.Repository.Branch.GetAll(MaestroAuthTestOrgName, repo)) + .FirstOrDefault(branch => branch.Name.StartsWith($"{DarcPRBranchPrefix}-{targetBranch}")); + + if (branch == null) + { + logger.LogWarning("Couldn't find darc PR branch targeting branch {targetBranch}", targetBranch); + } + else + { + await DeleteGitHubBranchAsync(repo, branch.Name); + } + } + + private async Task DeleteGitHubBranchAsync(string repo, string branch) => await ghClient.Git.Reference.Delete(MaestroAuthTestOrgName, repo, $"heads/{branch}"); + + protected async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, List assets) + { + logger.LogInformation("Creating a test build"); + + Build build = await localPcsApi.Builds.CreateAsync(new BuildData( + commit: commit, + azureDevOpsAccount: "test", + azureDevOpsProject: "test", + azureDevOpsBuildNumber: $"{DateTime.UtcNow:yyyyMMdd}.{new Random().Next(1, 75)}", + azureDevOpsRepository: repositoryUrl, + azureDevOpsBranch: branch, + released: false, + stable: false) + { + GitHubRepository = repositoryUrl, + GitHubBranch = branch, + Assets = assets + }); + + return build; + } + + protected async Task TriggerSubscriptionAsync(string subscriptionId) + { + logger.LogInformation("Triggering subscription {subscriptionId}", subscriptionId); + await localPcsApi.Subscriptions.TriggerSubscriptionAsync(default, Guid.Parse(subscriptionId)); + } + + protected async Task> PrepareVmrForkAsync(string branch, bool skipCleanup) + { + logger.LogInformation("Preparing VMR fork"); + // Sync the VMR fork branch + await SyncForkAsync("dotnet", "dotnet", branch); + + return await CreateTmpBranchAsync(VmrForkRepoName, branch, skipCleanup); + } + + protected async Task UpdateVmrSourceFiles(string branch, string productRepoUri, string productRepoForkUri) + { + // Fetch source mappings and source manifest files and replace the mapping for the repo we're testing on + logger.LogInformation("Updating source mappings and source manifest files in VMR fork to replace original product repo mapping with fork mapping"); + await UpdateRemoteVmrForkFileAsync(branch, productRepoUri, productRepoForkUri, SourceMappingsPath); + await UpdateRemoteVmrForkFileAsync(branch, productRepoUri, productRepoForkUri, SourceManifestPath); + } + + private async Task UpdateRemoteVmrForkFileAsync(string branch, string productRepoUri, string productRepoForkUri, string filePath) + { + logger.LogInformation("Updating file {file} on branch {branch} in the VMR fork", filePath, branch); + // Fetch remote file and replace the product repo URI with the repo we're testing on + var sourceMappingsFile = (await ghClient.Repository.Content.GetAllContentsByRef( + MaestroAuthTestOrgName, + VmrForkRepoName, + filePath, + branch)) + .FirstOrDefault() + ?? throw new Exception($"Failed to find file {SourceMappingsPath} in {MaestroAuthTestOrgName}" + + $"/{VmrForkRepoName} on branch {SourceMappingsPath}"); + + // Replace the product repo uri with the forked one + var updatedSourceMappings = sourceMappingsFile.Content.Replace(productRepoUri, productRepoForkUri); + UpdateFileRequest update = new( + $"Update {productRepoUri} source mapping", + updatedSourceMappings, + sourceMappingsFile.Sha, + branch); + + await ghClient.Repository.Content.UpdateFile( + MaestroAuthTestOrgName, + VmrForkRepoName, + filePath, + update); + } + + protected async Task> PrepareProductRepoForkAsync( + string productRepoUri, + string productRepoForkUri, + string productRepoBranch, + bool skipCleanup) + { + logger.LogInformation("Preparing product repo {repo} fork", productRepoUri); + (var name, var org) = GitRepoUrlParser.GetRepoNameAndOwner(productRepoUri); + // Check if the product repo fork already exists + var allRepos = await ghClient.Repository.GetAllForOrg(MaestroAuthTestOrgName); + + // If we already have a fork in maestro-auth-test, sync the branch we need with the source + if (allRepos.FirstOrDefault(repo => repo.HtmlUrl == productRepoForkUri) != null) + { + logger.LogInformation("Product repo fork {fork} already exists, syncing branch {branch} with source", productRepoForkUri, productRepoBranch); + await SyncForkAsync(org, name, productRepoBranch); + } + // If we don't, create a fork + else + { + logger.LogInformation("Forking product repo {source} to fork {fork}", productRepoUri, productRepoForkUri); + await ghClient.Repository.Forks.Create(org, name, new NewRepositoryFork { Organization = MaestroAuthTestOrgName }); + + // The Octokit client doesn't wait for the fork to actually be created, so we should wait a bit to make sure it's there + await Task.Delay(TimeSpan.FromSeconds(15)); + } + + return await CreateTmpBranchAsync(name, productRepoBranch, skipCleanup); + } + + protected async Task SyncForkAsync(string originOrg, string repoName, string branch) + { + logger.LogInformation("Syncing fork {fork} branch {branch} with upstream repo {upstream}", $"{MaestroAuthTestOrgName}/{repoName}", branch, $"{originOrg}/{repoName}"); + var reference = $"heads/{branch}"; + var upstream = await ghClient.Git.Reference.Get(originOrg, repoName, reference); + await ghClient.Git.Reference.Update(MaestroAuthTestOrgName, repoName, reference, new ReferenceUpdate(upstream.Object.Sha, true)); + } + + protected async Task> CreateTmpBranchAsync(string repoName, string originalBranch, bool skipCleanup) + { + var newBranchName = $"repro/{Guid.NewGuid()}"; + logger.LogInformation("Creating temporary branch {branch} in {repo}", newBranchName, $"{MaestroAuthTestOrgName}/{repoName}"); + + var baseBranch = await ghClient.Git.Reference.Get(MaestroAuthTestOrgName, repoName, $"heads/{originalBranch}"); + var newBranch = new NewReference($"refs/heads/{newBranchName}", baseBranch.Object.Sha); + await ghClient.Git.Reference.Create(MaestroAuthTestOrgName, repoName, newBranch); + + return AsyncDisposableValue.Create(newBranchName, async () => + { + if (skipCleanup) + { + return; + } + + logger.LogInformation("Cleaning up temporary branch {branchName}", newBranchName); + try + { + await DeleteGitHubBranchAsync(repoName, newBranchName); + } + catch + { + // If this throws an exception the most likely cause is that the branch was already deleted + } + }); + } + + protected static List CreateAssetDataFromBuild(Build build) + { + return build.Assets + .Select(asset => new AssetData(false) + { + Name = asset.Name, + Version = asset.Version, + Locations = asset.Locations?.Select(location => new AssetLocationData(location.Type) { Location = location.Location }).ToList() + }) + .ToList(); + } + + protected async Task GetLatestCommitInBranch(string owner, string repo, string branch) + { + var reference = await ghClient.Git.Reference.Get(owner, repo, $"heads/{branch}"); + return reference.Object.Sha; + } + + protected async Task GetCommit(string sourceRepoOwner, string sourceRepoName, string commit) + => await ghClient.Repository.Commit.Get(sourceRepoOwner, sourceRepoName, commit); +} diff --git a/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs b/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs new file mode 100644 index 0000000000..409c6c8de9 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Operations/ReproOperation.cs @@ -0,0 +1,157 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.ProductConstructionService.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Octokit; +using ProductConstructionService.ReproTool.Options; +using Build = Microsoft.DotNet.ProductConstructionService.Client.Models.Build; +using GitHubClient = Octokit.GitHubClient; + +namespace ProductConstructionService.ReproTool.Operations; + +internal class ReproOperation( + IBarApiClient prodBarClient, + ReproOptions options, + DarcProcessManager darcProcessManager, + [FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi, + GitHubClient ghClient, + ILogger logger) : Operation(logger, ghClient, localPcsApi) +{ + internal override async Task RunAsync() + { + logger.LogInformation("Fetching {subscriptionId} subscription from BAR", + options.Subscription); + var subscription = await prodBarClient.GetSubscriptionAsync(options.Subscription); + + if (subscription == null) + { + throw new ArgumentException($"Couldn't find subscription with subscription id {options.Subscription}"); + } + + if (!subscription.SourceEnabled) + { + throw new ArgumentException($"Subscription {options.Subscription} is not a code flow subscription"); + } + + if (!string.IsNullOrEmpty(subscription.SourceDirectory) && !string.IsNullOrEmpty(subscription.TargetDirectory)) + { + throw new ArgumentException("Code flow subscription incorrectly configured: is missing SourceDirectory or TargetDirectory"); + } + + if (!string.IsNullOrEmpty(options.Commit) && options.BuildId != null) + { + throw new ArgumentException($"Only one of {nameof(ReproOptions.Commit)} and {nameof(ReproOptions.BuildId)} can be provided"); + } + + Build? build = null; + if (options.BuildId != null) + { + build = await prodBarClient.GetBuildAsync(options.BuildId.Value); + if (build.GitHubRepository != subscription.SourceRepository) + { + throw new ArgumentException($"Build {build.Id} repository {build.GitHubRepository} doesn't match the subscription source repository {subscription.SourceRepository}"); + } + } + await darcProcessManager.InitializeAsync(); + + var defaultChannel = (await prodBarClient.GetDefaultChannelsAsync(repository: subscription.SourceRepository, channel: subscription.Channel.Name)).First(); + + string vmrBranch, productRepoUri, productRepoBranch; + var isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); + if (isForwardFlow) + { + vmrBranch = subscription.TargetBranch; + productRepoUri = subscription.SourceRepository; + productRepoBranch = defaultChannel.Branch; + } + else + { + vmrBranch = defaultChannel.Branch; + productRepoUri = subscription.TargetRepository; + productRepoBranch = subscription.TargetBranch; + } + var productRepoForkUri = ProductRepoFormat + productRepoUri.Split('/', StringSplitOptions.RemoveEmptyEntries).Last(); + logger.LogInformation("Reproducing subscription from {sourceRepo} to {targetRepo}", + isForwardFlow ? productRepoForkUri : VmrForkUri, + isForwardFlow ? VmrForkUri : productRepoForkUri); + + await using var vmrTmpBranch = await PrepareVmrForkAsync(vmrBranch, options.SkipCleanup); + await UpdateVmrSourceFiles(vmrTmpBranch.Value, productRepoUri, productRepoForkUri); + + logger.LogInformation("Preparing product repo fork {productRepoFork}, branch {branch}", productRepoForkUri, productRepoBranch); + await using var productRepoTmpBranch = await PrepareProductRepoForkAsync(productRepoUri, productRepoForkUri, productRepoBranch, options.SkipCleanup); + + // Find the latest commit in the source repo to create a build from + string sourceRepoSha; + (var sourceRepoName, var sourceRepoOwner) = GitRepoUrlParser.GetRepoNameAndOwner(subscription.SourceRepository); + if (build != null) + { + sourceRepoSha = build.Commit; + } + else if (string.IsNullOrEmpty(options.Commit)) + { + sourceRepoSha = await GetLatestCommitInBranch(sourceRepoOwner, sourceRepoName, defaultChannel.Branch); + } + else + { + // Validate that the commit actually exists + try + { + await GetCommit(sourceRepoOwner, sourceRepoName, options.Commit); + } + catch (NotFoundException) + { + throw new ArgumentException($"Commit {options.Commit} doesn't exist in repo {subscription.SourceRepository}"); + } + sourceRepoSha = options.Commit; + } + + var channelName = $"repro-{Guid.NewGuid()}"; + await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, options.SkipCleanup); + + var testBuild = await CreateBuildAsync( + isForwardFlow ? productRepoForkUri : VmrForkUri, + isForwardFlow ? productRepoTmpBranch.Value : vmrTmpBranch.Value, + sourceRepoSha, + build != null ? CreateAssetDataFromBuild(build) : []); + + await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( + channel: channelName, + sourceRepo: isForwardFlow ? productRepoForkUri : VmrForkUri, + targetRepo: isForwardFlow ? VmrForkUri : productRepoForkUri, + targetBranch: isForwardFlow ? vmrTmpBranch.Value : productRepoTmpBranch.Value, + sourceDirectory: subscription.SourceDirectory, + targetDirectory: subscription.TargetDirectory, + skipCleanup: options.SkipCleanup); + + await darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, options.SkipCleanup); + + await TriggerSubscriptionAsync(testSubscription.Value); + + if (options.SkipCleanup) + { + logger.LogInformation("Skipping cleanup. If you want to re-trigger the reproduced subscription run \"darc trigger-subscriptions --ids {subscriptionId} --bar-uri {barUri}\"", + testSubscription.Value, + ProductConstructionServiceApiOptions.PcsLocalUri); + return; + } + + logger.LogInformation("Code flow successfully recreated. Press enter to finish and cleanup"); + Console.ReadLine(); + + // Cleanup + if (isForwardFlow) + { + await DeleteDarcPRBranchAsync(VmrForkRepoName, vmrTmpBranch.Value); + } + else + { + await DeleteDarcPRBranchAsync(productRepoUri.Split('/').Last(), productRepoTmpBranch.Value); + } + } +} diff --git a/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs b/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs new file mode 100644 index 0000000000..3f6b44fdbb --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Options/FlatFlowTestOptions.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using ProductConstructionService.ReproTool.Operations; + +namespace ProductConstructionService.ReproTool.Options; +[Verb("forward-flow-test", HelpText = "Test full flat flow in the maestro-auth-test org")] +internal class FlatFlowTestOptions : Options +{ + internal override Operation GetOperation(IServiceProvider sp) + => ActivatorUtilities.CreateInstance(sp); +} diff --git a/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs b/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs new file mode 100644 index 0000000000..bfc44ba5f6 --- /dev/null +++ b/tools/ProductConstructionService.ReproTool/Options/FullBackflowTestOptions.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using ProductConstructionService.ReproTool.Operations; + +namespace ProductConstructionService.ReproTool.Options; + +[Verb("backflow-test", HelpText = "Flows an existing VMR build to all repos in maestro-auth-test")] +internal class FullBackflowTestOptions : Options +{ + [Option("build", HelpText = "Real VMR build from which we'll take assets from", Required = true)] + public required int BuildId { get; init; } + + [Option("target-branch", HelpText = "Target branch in all repos", Required = true)] + public required string TargetBranch { get; init; } + + [Option("vmr-branch", HelpText = "Vmr branch from which to backflow", Required = true)] + public required string VmrBranch { get; init; } + + [Option("commit", HelpText = "maestro-auth-test/dotnet commit to flow", Required = true)] + public required string Commit { get; init; } + + internal override Operation GetOperation(IServiceProvider sp) + => ActivatorUtilities.CreateInstance(sp, this); +} diff --git a/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs b/tools/ProductConstructionService.ReproTool/Options/Options.cs similarity index 81% rename from tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs rename to tools/ProductConstructionService.ReproTool/Options/Options.cs index d373de62aa..a11f529fdd 100644 --- a/tools/ProductConstructionService.ReproTool/ReproToolConfiguration.cs +++ b/tools/ProductConstructionService.ReproTool/Options/Options.cs @@ -1,35 +1,38 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using CommandLine; using Maestro.Data; -using Microsoft.DotNet.DarcLib.Helpers; +using Maestro.DataProviders; using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.DotNet.Kusto; +using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; -using Maestro.DataProviders; -using Microsoft.DotNet.Kusto; -using Microsoft.DotNet.ProductConstructionService.Client; -using Octokit; using Microsoft.Extensions.Logging.Console; +using Octokit; using ProductConstructionService.Common; +using ProductConstructionService.ReproTool.Operations; using GitHubClient = Octokit.GitHubClient; -namespace ProductConstructionService.ReproTool; -internal static class ReproToolConfiguration +namespace ProductConstructionService.ReproTool.Options; +internal abstract class Options { private const string LocalDbConnectionString = "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true"; private const string MaestroProdUri = "https://maestro.dot.net"; internal const string PcsLocalUri = "https://localhost:53180"; - internal static ServiceCollection RegisterServices( - this ServiceCollection services, - ReproToolOptions options) + public string? GitHubToken { get; set; } + + internal abstract Operation GetOperation(IServiceProvider sp); + + public virtual IServiceCollection RegisterServices(IServiceCollection services) { - services.AddSingleton(options); services.AddLogging(b => b .AddConsole(o => o.FormatterName = CompactConsoleLoggerFormatter.FormatterName) .AddConsoleFormatter() @@ -43,10 +46,11 @@ internal static ServiceCollection RegisterServices( MaestroProdUri)); services.AddSingleton(sp => ActivatorUtilities.CreateInstance(sp, "git")); services.AddSingleton(); - services.AddSingleton(PcsApiFactory.GetAnonymous(PcsLocalUri)); + services.AddKeyedSingleton("local", PcsApiFactory.GetAnonymous(PcsLocalUri)); + services.AddSingleton(PcsApiFactory.GetAuthenticated(MaestroProdUri, null, null, false)); services.AddSingleton(_ => new GitHubClient(new ProductHeaderValue("repro-tool")) { - Credentials = new Credentials(options.GitHubToken) + Credentials = new Credentials(GitHubToken) }); services.TryAddTransient(); diff --git a/tools/ProductConstructionService.ReproTool/ReproToolOptions.cs b/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs similarity index 68% rename from tools/ProductConstructionService.ReproTool/ReproToolOptions.cs rename to tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs index f520e96a71..2eddeea300 100644 --- a/tools/ProductConstructionService.ReproTool/ReproToolOptions.cs +++ b/tools/ProductConstructionService.ReproTool/Options/ReproOptions.cs @@ -2,17 +2,17 @@ // The .NET Foundation licenses this file to you under the MIT license. using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using ProductConstructionService.ReproTool.Operations; -namespace ProductConstructionService.ReproTool; +namespace ProductConstructionService.ReproTool.Options; -internal class ReproToolOptions +[Verb("repro", HelpText = "Locally reproduce a codeflow subscription in the maestro-auth-test org")] +internal class ReproOptions : Options { [Option('s', "subscription", HelpText = "Subscription that's getting reproduced", Required = true)] public required string Subscription { get; init; } - [Option("github-token", HelpText = "GitHub token", Required = false)] - public string? GitHubToken { get; set; } - [Option("commit", HelpText = "Commit to flow. Use when not flowing a build. If neither commit or build is specified, the latest commit in the subscription's source repository is flown", Required = false)] public string? Commit { get; init; } @@ -21,4 +21,7 @@ internal class ReproToolOptions [Option("skip-cleanup", HelpText = "Don't delete the created resources if they're needed for further testing. This includes the channel, subscription and PR branches. False by default", Required = false)] public bool SkipCleanup { get; init; } = false; + + internal override Operation GetOperation(IServiceProvider sp) + => ActivatorUtilities.CreateInstance(sp, this); } diff --git a/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj b/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj index 237741067c..c7686e4502 100644 --- a/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj +++ b/tools/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj @@ -21,5 +21,6 @@ + diff --git a/tools/ProductConstructionService.ReproTool/Program.cs b/tools/ProductConstructionService.ReproTool/Program.cs index c5c907161f..d896b6db9f 100644 --- a/tools/ProductConstructionService.ReproTool/Program.cs +++ b/tools/ProductConstructionService.ReproTool/Program.cs @@ -2,15 +2,26 @@ // The .NET Foundation licenses this file to you under the MIT license. using CommandLine; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.ReproTool; +using ProductConstructionService.ReproTool.Operations; +using ProductConstructionService.ReproTool.Options; +using Tools.Common; -Parser.Default.ParseArguments(args) - .WithParsed(o => +Type[] options = +[ + typeof(ReproOptions), + typeof(FlatFlowTestOptions), + typeof(FullBackflowTestOptions) +]; + +Parser.Default.ParseArguments(args, options) + .MapResult((Options o) => { IConfiguration userSecrets = new ConfigurationBuilder() - .AddUserSecrets() + .AddUserSecrets() .Build(); o.GitHubToken ??= userSecrets["GITHUB_TOKEN"]; o.GitHubToken ??= Environment.GetEnvironmentVariable("GITHUB_TOKEN"); @@ -18,9 +29,15 @@ var services = new ServiceCollection(); - services.RegisterServices(o); + o.RegisterServices(services); + services.AddSingleton(); + + services.AddMultiVmrSupport(Path.GetTempPath()); var provider = services.BuildServiceProvider(); - ActivatorUtilities.CreateInstance(provider).ReproduceCodeFlow().GetAwaiter().GetResult(); - }); + o.GetOperation(provider).RunAsync().GetAwaiter().GetResult(); + + return 0; + }, + (_) => -1); diff --git a/tools/ProductConstructionService.ReproTool/ReproTool.cs b/tools/ProductConstructionService.ReproTool/ReproTool.cs deleted file mode 100644 index 9faf87dd98..0000000000 --- a/tools/ProductConstructionService.ReproTool/ReproTool.cs +++ /dev/null @@ -1,354 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Maestro.Data; -using Microsoft.DotNet.DarcLib; -using Microsoft.DotNet.DarcLib.Helpers; -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; -using Microsoft.DotNet.ProductConstructionService.Client; -using Microsoft.DotNet.ProductConstructionService.Client.Models; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using Octokit; -using Build = Microsoft.DotNet.ProductConstructionService.Client.Models.Build; -using BuildData = Microsoft.DotNet.ProductConstructionService.Client.Models.BuildData; -using GitHubClient = Octokit.GitHubClient; - -namespace ProductConstructionService.ReproTool; - -internal class ReproTool( - IBarApiClient prodBarClient, - ReproToolOptions options, - BuildAssetRegistryContext context, - DarcProcessManager darcProcessManager, - IProductConstructionServiceApi localPcsApi, - GitHubClient ghClient, - ILogger logger) -{ - private const string MaestroAuthTestOrgName = "maestro-auth-test"; - private const string VmrForkRepoName = "dotnet"; - private const string VmrForkUri = $"https://github.com/{MaestroAuthTestOrgName}/{VmrForkRepoName}"; - private const string ProductRepoFormat = $"https://github.com/{MaestroAuthTestOrgName}/"; - private const long InstallationId = 289474; - private const string SourceMappingsPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceMappingsFileName}"; - private const string SourceManifestPath = $"{VmrInfo.SourceDirName}/{VmrInfo.SourceManifestFileName}"; - private const string DarcPRBranchPrefix = "darc"; - - internal async Task ReproduceCodeFlow() - { - logger.LogInformation("Fetching {subscriptionId} subscription from BAR", - options.Subscription); - var subscription = await prodBarClient.GetSubscriptionAsync(options.Subscription); - - if (subscription == null) - { - throw new ArgumentException($"Couldn't find subscription with subscription id {options.Subscription}"); - } - - if (!subscription.SourceEnabled) - { - throw new ArgumentException($"Subscription {options.Subscription} is not a code flow subscription"); - } - - if (!string.IsNullOrEmpty(subscription.SourceDirectory) && !string.IsNullOrEmpty(subscription.TargetDirectory)) - { - throw new ArgumentException("Code flow subscription incorrectly configured: is missing SourceDirectory or TargetDirectory"); - } - - if (!string.IsNullOrEmpty(options.Commit) && options.BuildId != null) - { - throw new ArgumentException($"Only one of {nameof(ReproToolOptions.Commit)} and {nameof(ReproToolOptions.BuildId)} can be provided"); - } - - Build? build = null; - if (options.BuildId != null) - { - build = await prodBarClient.GetBuildAsync(options.BuildId.Value); - if (build.GitHubRepository != subscription.SourceRepository) - { - throw new ArgumentException($"Build {build.Id} repository {build.GitHubRepository} doesn't match the subscription source repository {subscription.SourceRepository}"); - } - } - await darcProcessManager.InitializeAsync(); - - var defaultChannel = (await prodBarClient.GetDefaultChannelsAsync(repository: subscription.SourceRepository, channel: subscription.Channel.Name)).First(); - - string vmrBranch, productRepoUri, productRepoBranch; - bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); - if (isForwardFlow) - { - vmrBranch = subscription.TargetBranch; - productRepoUri = subscription.SourceRepository; - productRepoBranch = defaultChannel.Branch; - } - else - { - vmrBranch = defaultChannel.Branch; - productRepoUri = subscription.TargetRepository; - productRepoBranch = subscription.TargetBranch; - } - var productRepoForkUri = ProductRepoFormat + productRepoUri.Split('/', StringSplitOptions.RemoveEmptyEntries).Last(); - logger.LogInformation("Reproducing subscription from {sourceRepo} to {targetRepo}", - isForwardFlow ? productRepoForkUri : VmrForkUri, - isForwardFlow ? VmrForkUri : productRepoForkUri); - - await using var vmrTmpBranch = await PrepareVmrForkAsync(vmrBranch, productRepoUri, productRepoForkUri, options.SkipCleanup); - - logger.LogInformation("Preparing product repo fork {productRepoFork}, branch {branch}", productRepoForkUri, productRepoBranch); - await using var productRepoTmpBranch = await PrepareProductRepoForkAsync(productRepoUri, productRepoForkUri, productRepoBranch, options.SkipCleanup); - - // Find the latest commit in the source repo to create a build from - string sourceRepoSha; - (string sourceRepoName, string sourceRepoOwner) = GitRepoUrlParser.GetRepoNameAndOwner(subscription.SourceRepository); - if (build != null) - { - sourceRepoSha = build.Commit; - } - else if (string.IsNullOrEmpty(options.Commit)) - { - var res = await ghClient.Git.Reference.Get(sourceRepoOwner, sourceRepoName, $"heads/{defaultChannel.Branch}"); - sourceRepoSha = res.Object.Sha; - } - else - { - // Validate that the commit actually exists - try - { - await ghClient.Repository.Commit.Get(sourceRepoOwner, sourceRepoName, options.Commit); - } - catch (NotFoundException) - { - throw new ArgumentException($"Commit {options.Commit} doesn't exist in repo {subscription.SourceRepository}"); - } - sourceRepoSha = options.Commit; - } - - var channelName = $"repro-{Guid.NewGuid()}"; - await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, options.SkipCleanup); - - var testBuild = await CreateBuildAsync( - isForwardFlow ? productRepoForkUri : VmrForkUri, - isForwardFlow ? productRepoTmpBranch.Value : vmrTmpBranch.Value, - sourceRepoSha, - build != null ? CreateAssetDataFromBuild(build) : []); - - await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync( - channel: channelName, - sourceRepo: isForwardFlow ? productRepoForkUri : VmrForkUri, - targetRepo: isForwardFlow ? VmrForkUri : productRepoForkUri, - targetBranch: isForwardFlow ? vmrTmpBranch.Value : productRepoTmpBranch.Value, - sourceDirectory: subscription.SourceDirectory, - targetDirectory: subscription.TargetDirectory, - skipCleanup: options.SkipCleanup); - - await darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, options.SkipCleanup); - - await TriggerSubscriptionAsync(testSubscription.Value); - - if (options.SkipCleanup) - { - logger.LogInformation("Skipping cleanup. If you want to re-trigger the reproduced subscription run \"darc trigger-subscriptions --ids {subscriptionId} --bar-uri {barUri}\"", - testSubscription.Value, - ProductConstructionServiceApiOptions.PcsLocalUri); - return; - } - - logger.LogInformation("Code flow successfully recreated. Press enter to finish and cleanup"); - Console.ReadLine(); - - // Cleanup - if (isForwardFlow) - { - await DeleteDarcPRBranchAsync(VmrForkRepoName, vmrTmpBranch.Value); - } - else - { - await DeleteDarcPRBranchAsync(productRepoUri.Split('/').Last(), productRepoTmpBranch.Value); - } - } - - private async Task DeleteDarcPRBranchAsync(string repo, string targetBranch) - { - var branch = (await ghClient.Repository.Branch.GetAll(MaestroAuthTestOrgName, repo)) - .FirstOrDefault(branch => branch.Name.StartsWith($"{DarcPRBranchPrefix}-{targetBranch}")); - - if (branch == null) - { - logger.LogWarning("Couldn't find darc PR branch targeting branch {targetBranch}", targetBranch); - } - else - { - await DeleteGitHubBranchAsync(repo, branch.Name); - } - } - - private async Task AddRepositoryToBarIfMissingAsync(string repositoryName) - { - if ((await context.Repositories.FirstOrDefaultAsync(repo => repo.RepositoryName == repositoryName)) == null) - { - logger.LogInformation("Repo {repo} missing in local BAR. Adding an entry for it", repositoryName); - context.Repositories.Add(new Maestro.Data.Models.Repository - { - RepositoryName = repositoryName, - InstallationId = InstallationId - }); - await context.SaveChangesAsync(); - } - } - - private async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, List assets) - { - logger.LogInformation("Creating a test build"); - - Build build = await localPcsApi.Builds.CreateAsync(new BuildData( - commit: commit, - azureDevOpsAccount: "test", - azureDevOpsProject: "test", - azureDevOpsBuildNumber: $"{DateTime.UtcNow:yyyyMMdd}.{new Random().Next(1, 75)}", - azureDevOpsRepository: repositoryUrl, - azureDevOpsBranch: branch, - released: false, - stable: false) - { - GitHubRepository = repositoryUrl, - GitHubBranch = branch, - Assets = assets - }); - - return build; - } - - private static List CreateAssetDataFromBuild(Build build) - { - return build.Assets - .Select(asset => new AssetData(false) - { - Name = asset.Name, - Version = asset.Version, - Locations = asset.Locations?.Select(location => new AssetLocationData(location.Type) { Location = location.Location}).ToList() - }) - .ToList(); - } - - private async Task TriggerSubscriptionAsync(string subscriptionId) - { - logger.LogInformation("Triggering subscription {subscriptionId}", subscriptionId); - await localPcsApi.Subscriptions.TriggerSubscriptionAsync(default, Guid.Parse(subscriptionId)); - } - - private async Task> PrepareVmrForkAsync( - string branch, - string productRepoUri, - string productRepoForkUri, - bool skipCleanup) - { - logger.LogInformation("Preparing VMR fork"); - // Sync the VMR fork branch - await SyncForkAsync("dotnet", "dotnet", branch); - // Check if the user has the forked VMR in local DB - await AddRepositoryToBarIfMissingAsync(VmrForkUri); - - var newBranch = await CreateTmpBranchAsync(VmrForkRepoName, branch, skipCleanup); - - // Fetch source mappings and source manifest files and replace the mapping for the repo we're testing on - logger.LogInformation("Updating source mappings and source manifest files in VMR fork to replace original product repo mapping with fork mapping"); - await UpdateRemoteVmrForkFileAsync(newBranch.Value, productRepoUri, productRepoForkUri, SourceMappingsPath); - await UpdateRemoteVmrForkFileAsync(newBranch.Value, productRepoUri, productRepoForkUri, SourceManifestPath); - - return newBranch; - } - - private async Task DeleteGitHubBranchAsync(string repo, string branch) => await ghClient.Git.Reference.Delete(MaestroAuthTestOrgName, repo, $"heads/{branch}"); - - private async Task UpdateRemoteVmrForkFileAsync(string branch, string productRepoUri, string productRepoForkUri, string filePath) - { - logger.LogInformation("Updating file {file} on branch {branch} in the VMR fork", filePath, branch); - // Fetch remote file and replace the product repo URI with the repo we're testing on - var sourceMappingsFile = (await ghClient.Repository.Content.GetAllContentsByRef( - MaestroAuthTestOrgName, - VmrForkRepoName, - filePath, - branch)) - .FirstOrDefault() - ?? throw new Exception($"Failed to find file {SourceMappingsPath} in {MaestroAuthTestOrgName}" + - $"/{VmrForkRepoName} on branch {SourceMappingsPath}"); - - // Replace the product repo uri with the forked one - var updatedSourceMappings = sourceMappingsFile.Content.Replace(productRepoUri, productRepoForkUri); - UpdateFileRequest update = new( - $"Update {productRepoUri} source mapping", - updatedSourceMappings, - sourceMappingsFile.Sha, - branch); - - await ghClient.Repository.Content.UpdateFile( - MaestroAuthTestOrgName, - VmrForkRepoName, - filePath, - update); - } - - private async Task> PrepareProductRepoForkAsync( - string productRepoUri, - string productRepoForkUri, - string productRepoBranch, - bool skipCleanup) - { - logger.LogInformation("Preparing product repo {repo} fork", productRepoUri); - (var name, var org) = GitRepoUrlParser.GetRepoNameAndOwner(productRepoUri); - // Check if the product repo fork already exists - var allRepos = await ghClient.Repository.GetAllForOrg(MaestroAuthTestOrgName); - - // If we already have a fork in maestro-auth-test, sync the branch we need with the source - if (allRepos.FirstOrDefault(repo => repo.HtmlUrl == productRepoForkUri) != null) - { - logger.LogInformation("Product repo fork {fork} already exists, syncing branch {branch} with source", productRepoForkUri, productRepoBranch); - await SyncForkAsync(org, name, productRepoBranch); - } - // If we don't, create a fork - else - { - logger.LogInformation("Forking product repo {source} to fork {fork}", productRepoUri, productRepoForkUri); - await ghClient.Repository.Forks.Create(org, name, new NewRepositoryFork { Organization = MaestroAuthTestOrgName }); - } - await AddRepositoryToBarIfMissingAsync(productRepoForkUri); - - return await CreateTmpBranchAsync(name, productRepoBranch, skipCleanup); - } - - private async Task SyncForkAsync(string originOrg, string repoName, string branch) - { - logger.LogInformation("Syncing fork {fork} branch {branch} with upstream repo {upstream}", $"{MaestroAuthTestOrgName}/{repoName}", branch, $"{originOrg}/{repoName}"); - var reference = $"heads/{branch}"; - var upstream = await ghClient.Git.Reference.Get(originOrg, repoName, reference); - await ghClient.Git.Reference.Update(MaestroAuthTestOrgName, repoName, reference, new ReferenceUpdate(upstream.Object.Sha, true)); - } - - private async Task> CreateTmpBranchAsync(string repoName, string originalBranch, bool skipCleanup) - { - var newBranchName = $"repro/{Guid.NewGuid()}"; - logger.LogInformation("Creating temporary branch {branch} in {repo}", newBranchName, $"{MaestroAuthTestOrgName}/{repoName}"); - - var baseBranch = await ghClient.Git.Reference.Get(MaestroAuthTestOrgName, repoName, $"heads/{originalBranch}"); - var newBranch = new NewReference($"refs/heads/{newBranchName}", baseBranch.Object.Sha); - await ghClient.Git.Reference.Create(MaestroAuthTestOrgName, repoName, newBranch); - - return AsyncDisposableValue.Create(newBranchName, async () => - { - if (skipCleanup) - { - return; - } - - logger.LogInformation("Cleaning up temporary branch {branchName}", newBranchName); - try - { - await DeleteGitHubBranchAsync(repoName, newBranchName); - } - catch - { - // If this throws an exception the most likely cause is that the branch was already deleted - } - }); - } -} diff --git a/tools/FlatFlowMigrationCli/Constants.cs b/tools/Tools.Common/Constants.cs similarity index 89% rename from tools/FlatFlowMigrationCli/Constants.cs rename to tools/Tools.Common/Constants.cs index 0ae82321e3..e866797948 100644 --- a/tools/FlatFlowMigrationCli/Constants.cs +++ b/tools/Tools.Common/Constants.cs @@ -1,9 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace FlatFlowMigrationCli; +namespace Tools.Common; -internal static class Constants +public static class Constants { public const string VmrUri = "https://github.com/dotnet/dotnet"; public const string ArcadeRepoUri = "https://github.com/dotnet/arcade"; diff --git a/tools/Tools.Common/Tools.Common.csproj b/tools/Tools.Common/Tools.Common.csproj new file mode 100644 index 0000000000..c04a57cb85 --- /dev/null +++ b/tools/Tools.Common/Tools.Common.csproj @@ -0,0 +1,15 @@ + + + + net8.0 + enable + enable + false + + + + + + + + diff --git a/tools/FlatFlowMigrationCli/VmrDependencyResolver.cs b/tools/Tools.Common/VmrDependencyResolver.cs similarity index 95% rename from tools/FlatFlowMigrationCli/VmrDependencyResolver.cs rename to tools/Tools.Common/VmrDependencyResolver.cs index da3cedfcc4..a4a8589c97 100644 --- a/tools/FlatFlowMigrationCli/VmrDependencyResolver.cs +++ b/tools/Tools.Common/VmrDependencyResolver.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using FlatFlowMigrationCli.Operations; using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.DarcLib.VirtualMonoRepo; @@ -9,22 +8,22 @@ using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; -namespace FlatFlowMigrationCli; +namespace Tools.Common; -internal record VmrRepository(SourceMapping Mapping, DefaultChannel Channel); +public record VmrRepository(SourceMapping Mapping, DefaultChannel Channel); -internal class VmrDependencyResolver +public class VmrDependencyResolver { private readonly IProductConstructionServiceApi _pcsClient; private readonly IGitRepoFactory _gitRepoFactory; private readonly ISourceMappingParser _sourceMappingParser; - private readonly ILogger _logger; + private readonly ILogger _logger; public VmrDependencyResolver( IProductConstructionServiceApi pcsClient, IGitRepoFactory gitRepoFactory, ISourceMappingParser sourceMappingParser, - ILogger logger) + ILogger logger) { _pcsClient = pcsClient; _logger = logger;