Skip to content

Simplify readme generator test services and remove noisy test logging #12021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
benbp merged 1 commit into from
Sep 9, 2025
Merged

Simplify readme generator test services and remove noisy test logging #12021

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions ...i.Tests/Mocks/Helpers/MockOutputHelper.cs → ...Cli.Tests/TestHelpers/TestOutputHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
namespace Azure.Sdk.Tools.Cli.Tests.Mocks.Helpers;

/// <summary>
/// A really simple mock that just accumulates all the outputs into a List, which you can assert on
/// An IOutputHelper implementation that can be used in test.
/// It accumulates all the outputs into a List, which you can assert on
/// after the test is complete.
/// </summary>
internal class MockOutputHelper : IOutputHelper
internal class TestOutputHelper : IOutputHelper
{
/// <summary>
/// All the collected outputs, in the order they were added.
Expand All @@ -15,7 +16,7 @@ internal class MockOutputHelper : IOutputHelper
public IEnumerable<(string Method, object OutputValue)> Outputs => outputs;
private readonly List<(string Method, object OutputValue)> outputs;

public MockOutputHelper()
public TestOutputHelper()
{
outputs = [];
}
Expand Down
101 changes: 34 additions & 67 deletions tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/Generators/ReadMeGeneratorToolTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,42 +1,56 @@
using System.CommandLine;
using System.CommandLine.Parsing;
using Azure.AI.OpenAI;
using Azure.Sdk.Tools.Cli.Helpers;
using Moq;
using Azure.Sdk.Tools.Cli.Microagents;
using Azure.Sdk.Tools.Cli.Services;
using Azure.Sdk.Tools.Cli.Tests.Mocks.Helpers;
using Azure.Sdk.Tools.Cli.Tests.TestHelpers;
using Azure.Sdk.Tools.Cli.Tools.Package;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Moq;
using OpenAI.Chat;

namespace Azure.Sdk.Tools.Cli.Tests.Tools.Generators
{
internal class ReadMeGeneratorToolTests
{
private ReadMeGeneratorTool tool;
private TestOutputHelper outputHelper;
private Mock<IMicroagentHostService>? mockMicroAgentService;

[SetUp]
public void Setup()
{
outputHelper = new TestOutputHelper();
mockMicroAgentService = new Mock<IMicroagentHostService>();

tool = new ReadMeGeneratorTool(
new TestLogger<ReadMeGeneratorTool>(),
outputHelper,
mockMicroAgentService.Object
);
}

[Test]
public async Task TestReadmeGeneratorTool()
{
var testClients = SetupMocks();
var readmeContents = "This is a test response for the readme generation.";
mockMicroAgentService?.Setup(svc => svc.RunAgentToCompletion(
It.IsAny<Microagent<ReadmeGenerator.ReadmeContents>>(), It.IsAny<CancellationToken>())
).Returns(() => Task.FromResult(new ReadmeGenerator.ReadmeContents(readmeContents)));

(DirectoryInfo root, string packagePath) = await CreateFakeLanguageRepo();

var readmeOutputPath = Path.GetTempFileName();
var readmeTemplatePath = Path.Combine(AppContext.BaseDirectory, "TestAssets", "README-template.go.md");

try
{
var tool = ActivatorUtilities.CreateInstance<ReadMeGeneratorTool>(testClients.ServiceProvider);
var command = tool.GetCommand();

int exitCode = command.Invoke($"--output-path \"{readmeOutputPath}\" --service-url \"https://learn.microsoft.com/azure/service-bus-messaging\" --template-path {readmeTemplatePath} --package-path {packagePath}");

Assert.Multiple(() =>
{
Assert.That(exitCode, Is.EqualTo(0), "Command should execute successfully");
Assert.That(testClients.OutputHelper.Outputs.First().Method, Is.EqualTo("Output"));
Assert.That(testClients.OutputHelper.Outputs.First().OutputValue, Is.EqualTo($"Readme written to {readmeOutputPath}"));
Assert.That(outputHelper.Outputs.First().Method, Is.EqualTo("Output"));
Assert.That(outputHelper.Outputs.First().OutputValue, Is.EqualTo($"Readme written to {readmeOutputPath}"));
});

Assert.That(File.Exists(readmeOutputPath), Is.True, "Readme output file should be created");
Expand Down Expand Up @@ -64,9 +78,6 @@ public void TestReadmeGeneratorToolLive()
Assert.Ignore("Skipping test as AZURE_SDK_FOR_GO_PATH is not set");
}

var (sp, OutputHelperMock) = CreateServiceProvider();

var tool = ActivatorUtilities.CreateInstance<ReadMeGeneratorTool>(sp);
var command = tool.GetCommand();
var readmeOutputPath = Path.GetTempFileName();
var readmeTemplatePath = Path.Combine(AppContext.BaseDirectory, "TestAssets", "README-template.go.md");
Expand All @@ -78,13 +89,13 @@ public void TestReadmeGeneratorToolLive()

Assert.Multiple(() =>
{
Assert.That(OutputHelperMock.Outputs.First().Method, Is.EqualTo("Output"));
Assert.That(OutputHelperMock.Outputs.First().OutputValue, Is.EqualTo($"Readme written to {readmeOutputPath}"));
Assert.That(outputHelper.Outputs.First().Method, Is.EqualTo("Output"));
Assert.That(outputHelper.Outputs.First().OutputValue, Is.EqualTo($"Readme written to {readmeOutputPath}"));
});
}

/// <summary>
/// These tests all correspond to "LLM needs to do more work" type of returns from the tool, like removing
/// These tests all correspond to "LLM needs to do more work" type of returns from the tool, like removing
/// hardcoded locales, like 'en-us', or having it regenerate if some of the templates tokens make it through
/// and weren't replaced.
/// </summary>
Expand All @@ -100,24 +111,26 @@ public void TestReadmeGeneratorToolLive()
"The readme contains placeholders ((package path)) that should be removed and replaced with a proper package name")]
public async Task TestBadReadmeContent(string readmeContent, string expectedFeedback)
{
var testClients = SetupMocks(readmeContent);
mockMicroAgentService?.Setup(svc => svc.RunAgentToCompletion(
It.IsAny<Microagent<ReadmeGenerator.ReadmeContents>>(), It.IsAny<CancellationToken>())
).Returns(() => Task.FromResult(new ReadmeGenerator.ReadmeContents(readmeContent)));

(DirectoryInfo root, string packagePath) = await CreateFakeLanguageRepo();

var readmeOutputPath = Path.GetTempFileName();
var readmeTemplatePath = Path.Combine(AppContext.BaseDirectory, "TestAssets", "README-template.go.md");

try
{
var tool = ActivatorUtilities.CreateInstance<ReadMeGeneratorTool>(testClients.ServiceProvider);
var command = tool.GetCommand();

int exitCode = command.Invoke($"--output-path \"{readmeOutputPath}\" --service-url \"https://learn.microsoft.com/azure/service-bus-messaging\" --template-path {readmeTemplatePath} --package-path {packagePath}");

Assert.Multiple(() =>
{
Assert.That(exitCode, Is.EqualTo(1), "Command should fail, as the final readme doesn't pass validation");
Assert.That(testClients.OutputHelper.Outputs.First().Method, Is.EqualTo("OutputError"));
Assert.That(testClients.OutputHelper.Outputs.First().OutputValue, Is.EqualTo($"ReadmeGenerator failed with validation errors: {expectedFeedback}"));
Assert.That(outputHelper.Outputs.First().Method, Is.EqualTo("OutputError"));
Assert.That(outputHelper.Outputs.First().OutputValue, Is.EqualTo($"ReadmeGenerator failed with validation errors: {expectedFeedback}"));
});

Assert.That(File.Exists(readmeOutputPath), Is.True, "Readme output file should be created");
Expand All @@ -128,45 +141,6 @@ public async Task TestBadReadmeContent(string readmeContent, string expectedFeed
}
}

/// <summary>
/// Creates a service provider in the same way as the normal program. Can be used to instantiate real clients.
/// </summary>
/// <param name="customizeServices">Optional Action you can use to register your own client instances</param>
/// <returns></returns>
private static (ServiceProvider, MockOutputHelper) CreateServiceProvider(Action<ServiceCollection>? customizeServices = default)
Comment thread
richardpark-msft marked this conversation as resolved.
{
var serviceCollection = new ServiceCollection();
var OutputHelper = new MockOutputHelper();

serviceCollection.AddLogging(); // so our ILogger<T>'s will be available.

ServiceRegistrations.RegisterCommonServices(serviceCollection);
serviceCollection.AddSingleton<IOutputHelper>(OutputHelper);

customizeServices?.Invoke(serviceCollection);

var sp = serviceCollection.BuildServiceProvider();
return (sp, OutputHelper);
}

private static TestClients SetupMocks(string readmeContents = "This is a test response for the readme generation.")
{
var (openAIClientMock, chatClientMock) = OpenAIMockHelper.Create("gpt-4.1");

var serviceMock = new Mock<IMicroagentHostService>();
serviceMock.Setup(svc => svc.RunAgentToCompletion(
It.IsAny<Microagent<ReadmeGenerator.ReadmeContents>>(), It.IsAny<CancellationToken>())
).Returns(() => Task.FromResult(new ReadmeGenerator.ReadmeContents(readmeContents)));

var (serviceProvider, OutputHelper) = CreateServiceProvider((sc) =>
{
sc.AddLogging((lb) => lb.AddConsole());
sc.AddSingleton(serviceMock.Object);
});

return new TestClients(openAIClientMock, chatClientMock, serviceProvider, OutputHelper);
}

private static async Task<(DirectoryInfo root, string packagePath)> CreateFakeLanguageRepo()
{
var root = Directory.CreateTempSubdirectory("readme-generator-tool-test");
Expand All @@ -179,12 +153,5 @@ private static (ServiceProvider, MockOutputHelper) CreateServiceProvider(Action<
await File.WriteAllTextAsync(Path.Join(scriptsDir, "Verify-Links.ps1"), "Write-Host 'passed'");
return (root, packagePath);
}

private record TestClients(
Mock<AzureOpenAIClient> OpenAIClient,
Mock<ChatClient> ChatClient,
ServiceProvider ServiceProvider,
MockOutputHelper OutputHelper
);
}
}
Loading