Skip to content

Commit d72c699

Browse files
maorlegermaorleger
committed
add verification loop
1 parent 8e2454c commit d72c699

10 files changed

Lines changed: 1752 additions & 4 deletions

File tree

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Services/DockerServiceTests.cs‎

Lines changed: 487 additions & 0 deletions
Large diffs are not rendered by default.

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Services/Languages/TypeScriptTypecheckerTests.cs‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Sdk.Tools.Cli.Helpers;
5+
using Azure.Sdk.Tools.Cli.Services;
6+
using Azure.Sdk.Tools.Cli.Services.Languages;
7+
using Microsoft.Extensions.Logging.Abstractions;
8+
using Moq;
9+
10+
namespace Azure.Sdk.Tools.Cli.Tests.Services.Languages
11+
{
12+
[TestFixture]
13+
public class TypeScriptTypecheckerTests
14+
{
15+
private Mock<IDockerService> mockDockerService;
16+
private TypeScriptTypechecker typechecker;
17+
18+
[SetUp]
19+
public void SetUp()
20+
{
21+
mockDockerService = new Mock<IDockerService>();
22+
typechecker = new TypeScriptTypechecker(mockDockerService.Object, NullLogger<TypeScriptTypechecker>.Instance);
23+
}
24+
25+
[TearDown]
26+
public async Task TearDown()
27+
{
28+
if (typechecker != null)
29+
{
30+
// Setup mock for container removal during disposal
31+
mockDockerService.Setup(x => x.RemoveContainerAsync(
32+
It.IsAny<string>(), true, It.IsAny<CancellationToken>()))
33+
.ReturnsAsync(new ProcessResult { ExitCode = 0 });
34+
35+
await typechecker.DisposeAsync();
36+
}
37+
}
38+
39+
[Test]
40+
public void Constructor_WithNullDockerService_ThrowsArgumentNullException()
41+
{
42+
Assert.Throws<ArgumentNullException>(() =>
43+
new TypeScriptTypechecker(null!, NullLogger<TypeScriptTypechecker>.Instance));
44+
}
45+
46+
[Test]
47+
public void Constructor_WithNullLogger_ThrowsArgumentNullException()
48+
{
49+
Assert.Throws<ArgumentNullException>(() =>
50+
new TypeScriptTypechecker(mockDockerService.Object, null!));
51+
}
52+
53+
[Test]
54+
public async Task TypecheckAsync_WithValidCode_ReturnsSuccess()
55+
{
56+
// Arrange
57+
var code = "const message: string = 'Hello, World!';";
58+
var parameters = new TypeCheckRequest(
59+
code,
60+
"sample.ts",
61+
"/path/to/azure-sdk-for-js/sdk/keyvault/keyvault-keys",
62+
"/path/to/azure-sdk-for-js");
63+
64+
SetupSuccessfulDockerInteractions();
65+
66+
// Act
67+
var result = await typechecker.TypecheckAsync(parameters, CancellationToken.None);
68+
69+
// Assert
70+
Assert.That(result.Succeeded, Is.True);
71+
VerifyDockerInteractionSequence();
72+
}
73+
74+
[Test]
75+
public async Task TypecheckAsync_WithFailedTypeCheck_ReturnsFailure()
76+
{
77+
// Arrange
78+
var code = "const message: string = 123; // Type error";
79+
var parameters = new TypeCheckRequest(
80+
code,
81+
"sample.ts",
82+
"/path/to/azure-sdk-for-js/sdk/keyvault/keyvault-keys",
83+
"/path/to/azure-sdk-for-js");
84+
85+
SetupFailedTypecheckInteraction();
86+
87+
// Act
88+
var result = await typechecker.TypecheckAsync(parameters, CancellationToken.None);
89+
90+
// Assert
91+
Assert.That(result.Succeeded, Is.False);
92+
Assert.That(result.Output, Contains.Substring("Type error"));
93+
}
94+
95+
private void SetupSuccessfulDockerInteractions()
96+
{
97+
// Container creation and startup
98+
var createResult = new ProcessResult { ExitCode = 0 };
99+
createResult.AppendStdout("container-id");
100+
mockDockerService.Setup(x => x.CreateContainerAsync(
101+
"node:20",
102+
It.IsAny<string>(),
103+
It.IsAny<Dictionary<string, string>>(), // environmentVars
104+
null, // workingDirectory
105+
It.IsAny<Dictionary<string, string>>(), // volumeMounts
106+
It.IsAny<CancellationToken>()))
107+
.ReturnsAsync(createResult);
108+
109+
mockDockerService.Setup(x => x.StartContainerAsync(
110+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
111+
.ReturnsAsync(new ProcessResult { ExitCode = 0 });
112+
113+
mockDockerService.Setup(x => x.IsContainerRunningAsync(
114+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
115+
.ReturnsAsync(false); // Force container creation
116+
117+
// pnpm install globally
118+
mockDockerService.Setup(x => x.RunCommandInContainerAsync(
119+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "npm", "install", "-g", "pnpm" })),
120+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
121+
.ReturnsAsync(new ProcessResult { ExitCode = 0 });
122+
123+
// turbo install globally
124+
mockDockerService.Setup(x => x.RunCommandInContainerAsync(
125+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "npm", "install", "-g", "turbo" })),
126+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
127+
.ReturnsAsync(new ProcessResult { ExitCode = 0 });
128+
129+
// File operations
130+
mockDockerService.Setup(x => x.RunCommandInContainerAsync(
131+
It.IsAny<string>(), It.Is<string[]>(args => args.Contains("mkdir")),
132+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
133+
.ReturnsAsync(new ProcessResult { ExitCode = 0 });
134+
135+
mockDockerService.Setup(x => x.CopyToContainerAsync(
136+
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
137+
.ReturnsAsync(new ProcessResult { ExitCode = 0 });
138+
139+
// pnpm install in monorepo
140+
var pnpmInstallResult = new ProcessResult { ExitCode = 0 };
141+
pnpmInstallResult.AppendStdout("pnpm install completed");
142+
mockDockerService.Setup(x => x.RunCommandInContainerAsync(
143+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "pnpm", "install" })),
144+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
145+
.ReturnsAsync(pnpmInstallResult);
146+
147+
// turbo run build for package
148+
var turboBuildResult = new ProcessResult { ExitCode = 0 };
149+
turboBuildResult.AppendStdout("turbo build completed");
150+
mockDockerService.Setup(x => x.RunCommandInContainerAsync(
151+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "turbo", "run", "build" })),
152+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
153+
.ReturnsAsync(turboBuildResult);
154+
155+
// build:samples
156+
var buildResult = new ProcessResult { ExitCode = 0 };
157+
buildResult.AppendStdout("Compilation successful");
158+
mockDockerService.Setup(x => x.RunCommandInContainerAsync(
159+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "pnpm", "run", "build:samples" })),
160+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
161+
.ReturnsAsync(buildResult);
162+
}
163+
164+
private void SetupFailedTypecheckInteraction()
165+
{
166+
SetupSuccessfulDockerInteractions();
167+
168+
// Override build:samples to fail
169+
var failedBuildResult = new ProcessResult { ExitCode = 1 };
170+
failedBuildResult.AppendStderr("Type error: string is not assignable to number");
171+
mockDockerService.Setup(x => x.RunCommandInContainerAsync(
172+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "pnpm", "run", "build:samples" })),
173+
It.IsAny<string>(), It.IsAny<CancellationToken>()))
174+
.ReturnsAsync(failedBuildResult);
175+
}
176+
177+
private void VerifyDockerInteractionSequence()
178+
{
179+
// Verify container creation with volume mounts and environment vars
180+
mockDockerService.Verify(x => x.CreateContainerAsync(
181+
"node:20",
182+
It.IsAny<string>(),
183+
It.IsAny<Dictionary<string, string>>(), // environmentVars
184+
null, // workingDirectory
185+
It.IsAny<Dictionary<string, string>>(), // volumeMounts
186+
It.IsAny<CancellationToken>()),
187+
Times.Once);
188+
189+
// Verify pnpm install globally
190+
mockDockerService.Verify(x => x.RunCommandInContainerAsync(
191+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "npm", "install", "-g", "pnpm" })),
192+
It.IsAny<string>(), It.IsAny<CancellationToken>()),
193+
Times.Once);
194+
195+
// Verify turbo install globally
196+
mockDockerService.Verify(x => x.RunCommandInContainerAsync(
197+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "npm", "install", "-g", "turbo" })),
198+
It.IsAny<string>(), It.IsAny<CancellationToken>()),
199+
Times.Once);
200+
201+
// Verify pnpm install in monorepo
202+
mockDockerService.Verify(x => x.RunCommandInContainerAsync(
203+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "pnpm", "install" })),
204+
It.IsAny<string>(), It.IsAny<CancellationToken>()),
205+
Times.Once);
206+
207+
// Verify turbo run build
208+
mockDockerService.Verify(x => x.RunCommandInContainerAsync(
209+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "turbo", "run", "build" })),
210+
It.IsAny<string>(), It.IsAny<CancellationToken>()),
211+
Times.Once);
212+
213+
// Verify build:samples
214+
mockDockerService.Verify(x => x.RunCommandInContainerAsync(
215+
It.IsAny<string>(), It.Is<string[]>(args => args.SequenceEqual(new[] { "pnpm", "run", "build:samples" })),
216+
It.IsAny<string>(), It.IsAny<CancellationToken>()),
217+
Times.Once);
218+
}
219+
}
220+
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Services/SampleVerificationServiceTests.cs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Sdk.Tools.Cli.Microagents;
5+
using Azure.Sdk.Tools.Cli.Services;
6+
using Azure.Sdk.Tools.Cli.Tools.Package;
7+
using Microsoft.Extensions.Logging;
8+
using Microsoft.Extensions.Logging.Abstractions;
9+
using Moq;
10+
11+
namespace Azure.Sdk.Tools.Cli.Tests.Services
12+
{
13+
[TestFixture]
14+
public class SampleVerificationTests
15+
{
16+
private Mock<IDockerService> mockDockerService;
17+
private Mock<IMicroagentHostService> mockMicroagentService;
18+
private ILogger<SampleVerificationTests> logger;
19+
20+
[SetUp]
21+
public void SetUp()
22+
{
23+
mockDockerService = new Mock<IDockerService>();
24+
mockMicroagentService = new Mock<IMicroagentHostService>();
25+
logger = NullLogger<SampleVerificationTests>.Instance;
26+
}
27+
28+
[Test]
29+
public async Task VerifyAndFixSampleAsync_WithDockerNotAvailable_ReturnsFailureResult()
30+
{
31+
// Arrange
32+
var sample = new GeneratedSample("test.ts", "const x = 1;");
33+
mockDockerService.Setup(x => x.IsDockerAvailableAsync(It.IsAny<CancellationToken>()))
34+
.ReturnsAsync(false);
35+
36+
// Act
37+
var result = await SampleVerification.VerifyAndFixSampleAsync(
38+
sample, "typescript", mockDockerService.Object, mockMicroagentService.Object,
39+
logger, "/path/to/package", "/path/to/repo", CancellationToken.None);
40+
41+
// Assert
42+
Assert.That(result.Succeeded, Is.False);
43+
Assert.That(result.Content, Is.EqualTo(sample.Content));
44+
Assert.That(result.AttemptsMade, Is.EqualTo(0));
45+
Assert.That(result.Attempts.Count, Is.EqualTo(1));
46+
Assert.That(result.Attempts[0].TypeCheckOutput, Contains.Substring("Docker is not available"));
47+
}
48+
49+
[Test]
50+
public async Task VerifyAndFixSampleAsync_WithUnsupportedLanguage_ReturnsFailureResult()
51+
{
52+
// Arrange
53+
var sample = new GeneratedSample("test.unsupported", "const x = 1;");
54+
mockDockerService.Setup(x => x.IsDockerAvailableAsync(It.IsAny<CancellationToken>()))
55+
.ReturnsAsync(true);
56+
57+
// Act
58+
var result = await SampleVerification.VerifyAndFixSampleAsync(
59+
sample, "unsupported", mockDockerService.Object, mockMicroagentService.Object,
60+
logger, "/path/to/package", "/path/to/repo", CancellationToken.None);
61+
62+
// Assert
63+
Assert.That(result.Succeeded, Is.False);
64+
Assert.That(result.Content, Is.EqualTo(sample.Content));
65+
Assert.That(result.AttemptsMade, Is.EqualTo(0));
66+
Assert.That(result.Attempts.Count, Is.EqualTo(1));
67+
Assert.That(result.Attempts[0].TypeCheckOutput, Contains.Substring("not supported for verification"));
68+
}
69+
}
70+
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli/SampleGeneration/ILanguageTypechecker.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Azure.Sdk.Tools.Cli.Services.Languages
5+
{
6+
/// <summary>
7+
/// Interface for language-specific type checkers.
8+
/// </summary>
9+
internal interface ILanguageTypechecker
10+
{
11+
/// <summary>
12+
/// Performs type checking on the provided code.
13+
/// </summary>
14+
/// <param name="parameters">Type check parameters</param>
15+
/// <param name="ct">Cancellation token</param>
16+
/// <returns>Type check result</returns>
17+
Task<TypeCheckResult> TypecheckAsync(TypeCheckRequest parameters, CancellationToken ct);
18+
}
19+
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli/SampleGeneration/LanguageSupport.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4+
using Azure.Sdk.Tools.Cli.Services;
5+
using Azure.Sdk.Tools.Cli.Services.Languages;
6+
47
namespace Azure.Sdk.Tools.Cli.SampleGeneration
58
{
69
/// <summary>
@@ -113,5 +116,30 @@ public static bool IsLanguageSupported(string language)
113116
var supportedLanguages = GetSupportedLanguages();
114117
return supportedLanguages.Contains(language.ToLowerInvariant());
115118
}
119+
120+
/// <summary>
121+
/// Creates a typechecker for the specified language.
122+
/// </summary>
123+
public static ILanguageTypechecker CreateTypechecker(string language, IDockerService dockerService, ILogger logger)
124+
{
125+
return language.ToLowerInvariant() switch
126+
{
127+
"typescript" => new TypeScriptTypechecker(dockerService, logger),
128+
_ => throw new ArgumentException($"Language '{language}' sample verification is not yet implemented.", nameof(language))
129+
};
130+
}
131+
132+
public static string GetTypecheckingInstructions(string language)
133+
{
134+
return language.ToLowerInvariant() switch
135+
{
136+
"dotnet" => "Ensure proper using statements, namespace declarations, and type safety. Fix any compilation errors.",
137+
"typescript" => "Fix import statements, type annotations, and ensure strict TypeScript compliance. Resolve any tsc errors.",
138+
"python" => "Fix import statements, type hints, and ensure mypy and flake8 compliance. Resolve type and lint errors.",
139+
"java" => "Fix import statements, class declarations, and ensure javac compilation. Resolve compilation errors.",
140+
"go" => "Fix import statements, package declarations, and ensure go build and golint compliance. Resolve build and lint errors.",
141+
_ => "Fix syntax and type errors according to language best practices."
142+
};
143+
}
116144
}
117145
}

0 commit comments

Comments
 (0)