From e2c16ca5c3999658f4d823ed389061f65f625c8b Mon Sep 17 00:00:00 2001 From: Sameeksha Vaity Date: Thu, 12 Mar 2026 13:32:34 -0700 Subject: [PATCH 1/4] Updtae tool fixes for operation status --- .../Package/CustomizedCodeUpdateResponse.cs | 8 +++- .../TypeSpec/CustomizedCodeUpdateTool.cs | 45 +++++++++++++------ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs index 78f83468591..788ae0eec71 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs @@ -29,8 +29,7 @@ public class CustomizedCodeUpdateResponse : PackageResponseBase public List? AppliedPatches { get; set; } /// - /// Raw build error output. Only set when Success = false. - /// The classifier uses this to determine next steps. + /// Raw build output. Set on both success and failure to provide context in MCP responses. /// [JsonPropertyName("buildResult")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -77,6 +76,11 @@ protected override string Format() { sb.AppendLine($"ErrorCode: {ErrorCode}"); } + if (!string.IsNullOrWhiteSpace(BuildResult)) + { + sb.AppendLine("Build Output:"); + sb.AppendLine(BuildResult); + } if (TypeSpecChangesSummary is { Count: > 0 }) { sb.AppendLine("TypeSpec Changes:"); diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs index 2e54736f284..7199f0abac7 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs @@ -111,6 +111,7 @@ public override async Task HandleCommand(ParseResult parseResul return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = $"Customized code update failed: {ex.Message}", Message = $"Customized code update failed: {ex.Message}", BuildResult = ex.Message, ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.UnexpectedError @@ -147,6 +148,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = $"Package path does not exist: {packagePath}", Message = $"Package path does not exist: {packagePath}", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.InvalidInput, BuildResult = $"Package path does not exist: {packagePath}" @@ -158,6 +160,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = $"TypeSpec project path does not exist: {tspProjectPath}", Message = $"TypeSpec project path does not exist: {tspProjectPath}", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.InvalidInput, BuildResult = $"TypeSpec project path does not exist: {tspProjectPath}" @@ -169,6 +172,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = $"Invalid TypeSpec project path: {tspProjectPath}. Directory must exist and contain tspconfig.yaml.", Message = $"Invalid TypeSpec project path: {tspProjectPath}. Directory must exist and contain tspconfig.yaml.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.InvalidInput, BuildResult = $"Invalid TypeSpec project path: {tspProjectPath}. Directory must exist and contain tspconfig.yaml." @@ -192,6 +196,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = "No feedback items provided. Please supply a customization request or API review URL.", Message = "No feedback items provided. Please supply a customization request or API review URL.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.InvalidInput, BuildResult = "No feedback items to process." @@ -215,6 +220,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = "Feedback could not be classified.", Message = "Feedback could not be classified.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.InvalidInput, BuildResult = "Feedback could not be classified." @@ -323,6 +329,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = $"Failed to apply any TypeSpec customizations: {tspFixFailedReasons}", Message = $"Failed to apply any TypeSpec customizations: {tspFixFailedReasons}", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.TypeSpecCustomizationFailed }; @@ -364,19 +371,23 @@ private async Task RunUpdateAsync(string packagePa } } - logger.LogDebug("Building {packagePath}", packagePath); - var (success, error, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); + // Only build if we regenerated (code changed) or don't yet have build error context + if (tspFixSucceeded > 0 || buildError == null) + { + logger.LogDebug("Building {packagePath}", packagePath); + var (success, error, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); - buildSucceeded = success; - buildError = error; + buildSucceeded = success; + buildError = error; - // Append build result context to all remaining feedback items for the next iteration - if (tries + 1 < maxTries) - { - var buildContext = success ? "Build succeeded." : (error ?? "Build failed with unknown error."); - foreach (var item in feedbackDictionary.Values) + // Append build result context to all remaining feedback items for the next iteration + if (tries + 1 < maxTries) { - item.AppendContext(buildContext, "Build Result"); + var buildContext = success ? "Build succeeded." : (error ?? "Build failed with unknown error."); + foreach (var item in feedbackDictionary.Values) + { + item.AppendContext(buildContext, "Build Result"); + } } } @@ -402,6 +413,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = "Language service does not support customized code updates.", Message = "Language service does not support customized code updates.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.NoLanguageService, BuildResult = "No language service available for this package type." @@ -416,6 +428,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = "Build failed but no customization files found to repair.", Message = "Build failed but no customization files found to repair.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.BuildNoCustomizationsFailed, BuildResult = buildError @@ -438,6 +451,7 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = "No patches could be applied - automated repair found nothing to fix.", Message = "No patches could be applied - automated repair found nothing to fix.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.PatchesFailed, BuildResult = buildError @@ -448,13 +462,14 @@ private async Task RunUpdateAsync(string packagePa if (languageService.Language == SdkLanguage.Java) { logger.LogInformation("Regenerating code after patches (Java)..."); - var regenResult = await tspClientHelper.UpdateGenerationAsync(packagePath, commitSha: null, isCli: false, localSpecRepoPath: tspProjectPath, ct); + var regenResult = await tspClientHelper.UpdateGenerationAsync(packagePath, localSpecRepoPath: tspProjectPath, isCli: false, ct: ct); if (!regenResult.IsSuccessful) { logger.LogWarning("Regeneration failed: {Error}", regenResult.ResponseError); return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = $"Regeneration failed after patches: {regenResult.ResponseError}", Message = $"Regeneration failed after patches: {regenResult.ResponseError}", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.RegenerateAfterPatchesFailed, BuildResult = regenResult.ResponseError, @@ -464,8 +479,8 @@ private async Task RunUpdateAsync(string packagePa } } - // Step 6: Final build to validate - logger.LogInformation("Running final build to validate..."); + // Step 6: Final build to validate patches + logger.LogInformation("Running final build to validate patches..."); var (finalBuildSuccess, finalBuildError, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); if (finalBuildSuccess) @@ -475,16 +490,18 @@ private async Task RunUpdateAsync(string packagePa { Success = true, Message = "Build passed after repairs.", + BuildResult = finalBuildError, TypeSpecChangesSummary = changesMade, AppliedPatches = patches }; } - // Build still failing + // Build still failing after patches logger.LogInformation("Build still failing after patches."); return new CustomizedCodeUpdateResponse { Success = false, + ResponseError = "Patches applied but build still failing.", Message = "Patches applied but build still failing.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.BuildAfterPatchesFailed, BuildResult = finalBuildError, From 53d27e3b56f04cce77a03576a30fb88bb1f89f96 Mon Sep 17 00:00:00 2001 From: Sameeksha Vaity Date: Fri, 13 Mar 2026 13:30:52 -0700 Subject: [PATCH 2/4] update buidl logic --- .../TypeSpec/CustomizedCodeUpdateToolTests.cs | 2 + .../Package/CustomizedCodeUpdateResponse.cs | 3 +- .../TypeSpec/CustomizedCodeUpdateTool.cs | 68 ++++++++++++------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs index 438853f233f..74e75affe60 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs @@ -431,8 +431,10 @@ public async Task TspCustomization_PartialFailure_ProceedsToBuild() public async Task TspRegeneration_Fails_ContinuesLoopAndAppendsContext() { var classifyCalls = 0; + var svc = new ConfigurableLanguageService(buildFunc: () => (false, "error: regen never succeeded", null)); var failingTsp = new MockTspHelper(updateSuccess: false, updateError: "tsp-client failed: exit code 1"); var (tool, _) = CreateTool( + languageService: svc, tspHelper: failingTsp, configureClassifier: c => c.Setup(x => x.ClassifyItemsAsync( diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs index 788ae0eec71..755aa308122 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/Package/CustomizedCodeUpdateResponse.cs @@ -29,7 +29,8 @@ public class CustomizedCodeUpdateResponse : PackageResponseBase public List? AppliedPatches { get; set; } /// - /// Raw build output. Set on both success and failure to provide context in MCP responses. + /// Raw build error output. Only set when Success = false. + /// The classifier uses this to determine next steps. /// [JsonPropertyName("buildResult")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs index 7199f0abac7..e99eec20a51 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs @@ -6,11 +6,13 @@ using Azure.Sdk.Tools.Cli.Commands; using Azure.Sdk.Tools.Cli.Helpers; using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.Responses; using Azure.Sdk.Tools.Cli.Models.Responses.Package; using Azure.Sdk.Tools.Cli.Services; using Azure.Sdk.Tools.Cli.Services.Languages; using Azure.Sdk.Tools.Cli.Services.TypeSpec; using Azure.Sdk.Tools.Cli.Tools.Core; +using Microsoft.Graph.Models; using ModelContextProtocol.Server; namespace Azure.Sdk.Tools.Cli.Tools.TypeSpec; @@ -335,18 +337,22 @@ private async Task RunUpdateAsync(string packagePa }; } - // All items are code customizations — no TSP changes were made, skip regen - // but still build to get error context for the patch agent + // All items are code customizations — no TSP changes were made, skip regen. + // Build will happen after the loop to get error context for the patch agent. if (tspApplicable == 0 && codeCustomizations > 0) { - logger.LogInformation("All items classified as CODE_CUSTOMIZATION — skipping regen, building for error context."); - var (codeCustSuccess, codeCustError, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); - buildSucceeded = codeCustSuccess; - buildError = codeCustError; + logger.LogInformation("All items classified as CODE_CUSTOMIZATION — skipping regen."); break; } } + // No TSP progress on subsequent iterations — stop looping + if (tries > 0 && tspFixSucceeded == 0) + { + logger.LogInformation("No TSP fixes succeeded on iteration {Iteration}, stopping.", tries + 1); + break; + } + // Don't waste time regenerating if no TSP fixes were successfully applied if (tspFixSucceeded > 0) { @@ -365,35 +371,42 @@ private async Task RunUpdateAsync(string packagePa } buildSucceeded = false; - buildError = regenContext; + // Reset buildError — no code was generated, nothing new to build + buildError = null; tries++; continue; } } - // Only build if we regenerated (code changed) or don't yet have build error context - if (tspFixSucceeded > 0 || buildError == null) - { - logger.LogDebug("Building {packagePath}", packagePath); - var (success, error, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); + // Always build after regen to get fresh error context for the classifier + logger.LogDebug("Building {packagePath}", packagePath); + var (success, error, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); - buildSucceeded = success; - buildError = error; + buildSucceeded = success; + buildError = error; - // Append build result context to all remaining feedback items for the next iteration - if (tries + 1 < maxTries) + // Append build result context to all remaining feedback items for the next iteration + if (tries + 1 < maxTries) + { + var buildContext = success ? "Build succeeded." : (error ?? "Build failed with unknown error."); + foreach (var item in feedbackDictionary.Values) { - var buildContext = success ? "Build succeeded." : (error ?? "Build failed with unknown error."); - foreach (var item in feedbackDictionary.Values) - { - item.AppendContext(buildContext, "Build Result"); - } + item.AppendContext(buildContext, "Build Result"); } } tries++; } while (!buildSucceeded && tries < maxTries); + // Build for error context if no build happened in the loop (pure CODE_CUSTOMIZATION path) + if (!buildSucceeded && buildError == null) + { + logger.LogInformation("Building for error context..."); + var (s, e, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); + buildSucceeded = s; + buildError = e; + } + if (buildSucceeded) { logger.LogInformation("Build passed after Typespec customizations."); @@ -428,7 +441,9 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, - ResponseError = "Build failed but no customization files found to repair.", + ResponseError = string.IsNullOrWhiteSpace(buildError) + ? "Build failed but no customization files found to repair." + : $"Build failed but no customization files found to repair.\n{buildError}", Message = "Build failed but no customization files found to repair.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.BuildNoCustomizationsFailed, BuildResult = buildError @@ -451,7 +466,9 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, - ResponseError = "No patches could be applied - automated repair found nothing to fix.", + ResponseError = string.IsNullOrWhiteSpace(buildError) + ? "No patches could be applied - automated repair found nothing to fix." + : $"No patches could be applied - automated repair found nothing to fix.\n{buildError}", Message = "No patches could be applied - automated repair found nothing to fix.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.PatchesFailed, BuildResult = buildError @@ -490,7 +507,6 @@ private async Task RunUpdateAsync(string packagePa { Success = true, Message = "Build passed after repairs.", - BuildResult = finalBuildError, TypeSpecChangesSummary = changesMade, AppliedPatches = patches }; @@ -501,7 +517,9 @@ private async Task RunUpdateAsync(string packagePa return new CustomizedCodeUpdateResponse { Success = false, - ResponseError = "Patches applied but build still failing.", + ResponseError = string.IsNullOrWhiteSpace(finalBuildError) + ? "Patches applied but build still failing." + : $"Patches applied but build still failing.\n{finalBuildError}", Message = "Patches applied but build still failing.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.BuildAfterPatchesFailed, BuildResult = finalBuildError, From 48c31797530b2b9bfa975572ec02ab52e416d514 Mon Sep 17 00:00:00 2001 From: Sameeksha Vaity Date: Tue, 17 Mar 2026 13:40:09 -0700 Subject: [PATCH 3/4] switch no-loop design with 2-pass deterministic flow, pass 1 applies fixes and builds, pass 2 re-evaluates with error context --- .../TypeSpec/CustomizedCodeUpdateToolTests.cs | 250 +++++++++++--- .../TypeSpec/CustomizedCodeUpdateTool.cs | 322 +++++++++--------- 2 files changed, 349 insertions(+), 223 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs index 74e75affe60..a2eb44a4ad6 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs @@ -133,16 +133,20 @@ public async Task TspFix_BuildPassesFirstIteration_ReturnsSuccess() } [Test] - public async Task TspFix_BuildFailsFirstTry_PassesOnRetry_ReturnsSuccess() + public async Task TspFix_BuildFailsAfterRegen_FallsThroughToPatching() { var buildCalls = 0; - var svc = new ConfigurableLanguageService(buildFunc: () => - { - buildCalls++; - return buildCalls == 1 - ? (false, "error: missing import", null) - : (true, null, null); - }); + var svc = new ConfigurableLanguageService( + buildFunc: () => + { + buildCalls++; + // First build (after regen) fails, second build (error context / final) passes + return buildCalls <= 1 + ? (false, "error: missing import", null) + : (true, null, null); + }, + hasCustomizations: true, + patchesFunc: () => [new AppliedPatch("test.java", "Fixed import", 1)]); var (tool, _) = CreateTool(languageService: svc); var pkg = CreateTempDir(); @@ -151,8 +155,7 @@ public async Task TspFix_BuildFailsFirstTry_PassesOnRetry_ReturnsSuccess() var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); Assert.That(result.Success, Is.True); - Assert.That(result.Message, Does.Contain("Build passed")); - Assert.That(buildCalls, Is.EqualTo(2), "Should have attempted build twice"); + Assert.That(result.Message, Does.Contain("Build passed after repairs")); } // ======================================================================== @@ -286,9 +289,10 @@ public async Task Classification_OnlyNonTspItems_ReturnsSuccess() } [Test] - public async Task Classification_EmptyOnSecondIteration_BreaksLoop() + public async Task Classification_EmptyOnSecondPass_StillBuildsForContext() { - // First call returns TSP_APPLICABLE, second call returns empty (nothing more to fix) + // First pass returns TSP_APPLICABLE, regen succeeds, build fails. + // Second pass (re-classify with build errors) returns empty — no further action. var classifyCalls = 0; var buildCalls = 0; var svc = new ConfigurableLanguageService(buildFunc: () => @@ -336,7 +340,8 @@ public async Task Classification_EmptyOnSecondIteration_BreaksLoop() var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); Assert.That(result.Success, Is.False); - Assert.That(buildCalls, Is.EqualTo(1), "Should have only built once before second classify returned empty"); + Assert.That(classifyCalls, Is.EqualTo(2), "Should classify twice: initial pass + second pass with build context"); + Assert.That(buildCalls, Is.EqualTo(1), "Should build once after regen"); } // ======================================================================== @@ -344,30 +349,72 @@ public async Task Classification_EmptyOnSecondIteration_BreaksLoop() // ======================================================================== [Test] - public async Task TspCustomization_AllFail_FirstIteration_ReturnsTypeSpecCustomizationFailed() + public async Task TspCustomization_AllFail_ReclassifiedOnSecondPass() { - var (tool, _) = CreateTool(configureTspCustomization: t => - t.Setup(x => x.ApplyCustomizationAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .ReturnsAsync(new TypeSpecCustomizationServiceResult - { - Success = false, - ChangesSummary = [], - FailureReason = "Could not parse TypeSpec project" - })); + var classifyCalls = 0; + var (tool, _) = CreateTool( + configureTspCustomization: t => + t.Setup(x => x.ApplyCustomizationAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new TypeSpecCustomizationServiceResult + { + Success = false, + ChangesSummary = [], + FailureReason = "Could not parse TypeSpec project" + }), + configureClassifier: c => + c.Setup(x => x.ClassifyItemsAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns, string, string, string?, string?, int?, CancellationToken>( + (items, _, _, _, _, _, _) => + { + classifyCalls++; + var actualId = items.FirstOrDefault()?.Id ?? "1"; + if (classifyCalls == 1) + { + return Task.FromResult(new FeedbackClassificationResponse + { + Classifications = + [ + new FeedbackClassificationResponse.ItemClassificationDetails + { + ItemId = actualId, Classification = "TSP_APPLICABLE", + Reason = "fixable", Text = "rename X" + } + ] + }); + } + // Second pass: classifier sees failure context and flags manual intervention + return Task.FromResult(new FeedbackClassificationResponse + { + Classifications = + [ + new FeedbackClassificationResponse.ItemClassificationDetails + { + ItemId = actualId, Classification = "REQUIRES_MANUAL_INTERVENTION", + Reason = "TSP customization failed, manual fix needed", Text = "rename X" + } + ] + }); + })); var pkg = CreateTempDir(); var tspDir = CreateTempDir(); var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); - Assert.That(result.Success, Is.False); - Assert.That(result.ErrorCode, Is.EqualTo(CustomizedCodeUpdateResponse.KnownErrorCodes.TypeSpecCustomizationFailed)); - Assert.That(result.Message, Does.Contain("Could not parse TypeSpec project")); + Assert.That(classifyCalls, Is.EqualTo(2), "Should classify twice: pass 1 + pass 2 after TSP failure"); + Assert.That(result.NextSteps, Is.Not.Null.And.Count.GreaterThan(0), "Should include manual intervention from second pass"); } [Test] @@ -428,13 +475,11 @@ public async Task TspCustomization_PartialFailure_ProceedsToBuild() } [Test] - public async Task TspRegeneration_Fails_ContinuesLoopAndAppendsContext() + public async Task TspRegeneration_Fails_ReclassifiesOnSecondPass() { var classifyCalls = 0; - var svc = new ConfigurableLanguageService(buildFunc: () => (false, "error: regen never succeeded", null)); var failingTsp = new MockTspHelper(updateSuccess: false, updateError: "tsp-client failed: exit code 1"); var (tool, _) = CreateTool( - languageService: svc, tspHelper: failingTsp, configureClassifier: c => c.Setup(x => x.ClassifyItemsAsync( @@ -450,14 +495,29 @@ public async Task TspRegeneration_Fails_ContinuesLoopAndAppendsContext() { classifyCalls++; var actualId = items.FirstOrDefault()?.Id ?? "1"; + if (classifyCalls == 1) + { + return Task.FromResult(new FeedbackClassificationResponse + { + Classifications = + [ + new FeedbackClassificationResponse.ItemClassificationDetails + { + ItemId = actualId, Classification = "TSP_APPLICABLE", + Reason = "fixable", Text = "rename X" + } + ] + }); + } + // Second pass: reclassify as manual intervention (emitter issue) return Task.FromResult(new FeedbackClassificationResponse { Classifications = [ new FeedbackClassificationResponse.ItemClassificationDetails { - ItemId = actualId, Classification = "TSP_APPLICABLE", - Reason = "fixable", Text = "rename X" + ItemId = actualId, Classification = "REQUIRES_MANUAL_INTERVENTION", + Reason = "Emitter issue, cannot be patched", Text = "rename X" } ] }); @@ -468,9 +528,95 @@ public async Task TspRegeneration_Fails_ContinuesLoopAndAppendsContext() var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); - // Regen failure should not short-circuit; loop continues for re-classification - Assert.That(result.Success, Is.False); - Assert.That(classifyCalls, Is.EqualTo(2), "Should classify twice (maxTries) even when regen fails"); + // Regen failure → second pass reclassifies as manual intervention + Assert.That(classifyCalls, Is.EqualTo(2), "Should classify twice: initial + second pass after regen failure"); + Assert.That(result.NextSteps, Is.Not.Null.And.Count.GreaterThan(0), "Should include manual intervention steps"); + } + + [Test] + public async Task RegenFails_TspCompiled_EmitterIssue_ClassifierGivesManualGuidance() + { + // TSP fix succeeds (compiles), but regen fails — likely an emitter issue we can't patch. + // Second pass classifier should see the regen failure context and flag manual intervention + // without retrying, since retries won't fix an emitter problem. + var classifyCalls = 0; + var buildCalls = 0; + string? secondPassContext = null; + + var failingTsp = new MockTspHelper(updateSuccess: false, updateError: "emitter @azure-tools/typespec-java failed: unexpected token"); + var svc = new ConfigurableLanguageService(buildFunc: () => + { + buildCalls++; + return (true, null, null); + }); + + var (tool, _) = CreateTool( + languageService: svc, + tspHelper: failingTsp, + configureClassifier: c => + c.Setup(x => x.ClassifyItemsAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns, string, string, string?, string?, int?, CancellationToken>( + (items, _, _, _, _, _, _) => + { + classifyCalls++; + var actualId = items.FirstOrDefault()?.Id ?? "1"; + if (classifyCalls == 1) + { + return Task.FromResult(new FeedbackClassificationResponse + { + Classifications = + [ + new FeedbackClassificationResponse.ItemClassificationDetails + { + ItemId = actualId, Classification = "TSP_APPLICABLE", + Reason = "rename operation", Text = "rename FooClient to BarClient" + } + ] + }); + } + + // Second pass: classifier sees regen failure and determines manual intervention + secondPassContext = items.FirstOrDefault()?.Context; + return Task.FromResult(new FeedbackClassificationResponse + { + Classifications = + [ + new FeedbackClassificationResponse.ItemClassificationDetails + { + ItemId = actualId, Classification = "REQUIRES_MANUAL_INTERVENTION", + Reason = "Emitter issue — cannot resolve via TSP or patching", + Text = "rename FooClient to BarClient" + } + ] + }); + })); + + var pkg = CreateTempDir(); + var tspDir = CreateTempDir(); + + var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, + customizationRequest: "Rename FooClient to BarClient", ct: CancellationToken.None); + + // Verified: exactly 2 classifier calls (pass 1 + pass 2), no retry loop + Assert.That(classifyCalls, Is.EqualTo(2), "Should classify twice: pass 1 + pass 2 after regen failure"); + + // Regen failed so no build in the regen block, but a "build for error context" call should happen + Assert.That(buildCalls, Is.EqualTo(1), "Should build once for error context since regen failed"); + + // Second pass should have the regen failure context + Assert.That(secondPassContext, Does.Contain("Regeneration failed"), "Second pass should see regen failure"); + Assert.That(secondPassContext, Does.Contain("emitter @azure-tools/typespec-java failed"), "Should include the emitter error details"); + + // Result flags manual intervention + Assert.That(result.NextSteps, Is.Not.Null.And.Count.GreaterThan(0), "Should include manual intervention steps"); + Assert.That(result.NextSteps![0], Does.Contain("Emitter issue")); } // ======================================================================== @@ -535,12 +681,12 @@ public async Task BuildFails_NoPatchesApplied_ReturnsPatchesFailed() public async Task BuildFails_PatchesApplied_FinalBuildSucceeds_ReturnsSuccess() { var buildCalls = 0; - // TSP loop: 2 builds fail, then falls through to patch pipeline, final build (3rd call) passes + // Pass 1: build fails (1st call), falls through to patch pipeline, final build (2nd call) passes var svc = new ConfigurableLanguageService( buildFunc: () => { buildCalls++; - return buildCalls <= 2 + return buildCalls <= 1 ? (false, "error: variable already defined", null) : (true, null, null); }, @@ -674,8 +820,8 @@ public async Task Java_RegenAfterPatches_Fails_ReturnsRegenerateAfterPatchesFail patchesFunc: () => [new AppliedPatch("test.java", "patch", 1)], language: SdkLanguage.Java); - // TSP loop regen calls succeed (2x), Java regen-after-patches (3rd) fails - var failingTspForJavaRegen = new CallCountMockTspHelper(failAfterCall: 2, failError: "regen failed: tsp-client error"); + // Pass 1 regen call succeeds (1st call), Java regen-after-patches (2nd) fails + var failingTspForJavaRegen = new CallCountMockTspHelper(failAfterCall: 1, failError: "regen failed: tsp-client error"); var (tool, _) = CreateTool(languageService: svc, tspHelper: failingTspForJavaRegen); var pkg = CreateTempDir(); var tspDir = CreateTempDir(); @@ -731,20 +877,14 @@ public async Task CustomizationRequest_FlowsToClassifier() } [Test] - public async Task RetryLoop_SecondIteration_FeedbackIncludesContextFromFirstIteration() + public async Task SecondPass_FeedbackIncludesBuildErrorContext() { - // Build fails first time, so second iteration should include context - var buildCalls = 0; + // TSP fix applied, regen succeeds, build fails → second pass should see build error context var classifyCalls = 0; string? secondCallContext = null; var svc = new ConfigurableLanguageService(buildFunc: () => - { - buildCalls++; - return buildCalls == 1 - ? (false, "error CS0246: type 'FooClient' not found", null) - : (true, null, null); - }); + (false, "error CS0246: type 'FooClient' not found", null)); var (tool, _) = CreateTool( languageService: svc, @@ -785,7 +925,6 @@ await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "Rename FooClient to BarClient", ct: CancellationToken.None); Assert.That(classifyCalls, Is.EqualTo(2)); - Assert.That(secondCallContext, Does.Contain("Iteration 1")); Assert.That(secondCallContext, Does.Contain("Typespec changes applied")); Assert.That(secondCallContext, Does.Contain("Renamed FooClient to BarClient")); Assert.That(secondCallContext, Does.Contain("Build Result")); @@ -793,8 +932,9 @@ await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, } [Test] - public async Task RetryLoop_MaxTriesRespected_DoesNotExceedTwoIterations() + public async Task TwoPass_ClassifiesExactlyTwiceAndBuildsOnce() { + // Build fails after regen → second pass reclassifies with error context var buildCalls = 0; var classifyCalls = 0; var svc = new ConfigurableLanguageService( @@ -839,8 +979,8 @@ public async Task RetryLoop_MaxTriesRespected_DoesNotExceedTwoIterations() var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); Assert.That(result.Success, Is.False); - Assert.That(classifyCalls, Is.EqualTo(2), "Should classify exactly 2 times (maxTries)"); - Assert.That(buildCalls, Is.EqualTo(2), "Should build exactly 2 times (maxTries)"); + Assert.That(classifyCalls, Is.EqualTo(2), "Should classify exactly 2 times (pass 1 + pass 2)"); + Assert.That(buildCalls, Is.EqualTo(1), "Should build exactly once after regen"); } [Test] diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs index e99eec20a51..40219f0fc2b 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs @@ -186,12 +186,6 @@ private async Task RunUpdateAsync(string packagePa // TODO - do this once we add API view option // var language = await feedbackService.GetLanguageAsync(apiViewUrl, ct); - // Step 1: try tsp fixes - var tries = 0; - var maxTries = 2; - bool buildSucceeded = false; - string? buildError = null; - var feedbackItems = await GetFeedbackItems(plainTextFeedback: customizationRequest, ct: ct); if (feedbackItems.Count == 0) { @@ -210,195 +204,185 @@ private async Task RunUpdateAsync(string packagePa List manualInterventions = new(); StringBuilder classifierAnalysis = new(); StringBuilder tspFixFailedReasons = new(); - do - { - // TODO - need to update this to avoid casting to/from list - var response = await _classifierService.ClassifyItemsAsync([.. feedbackDictionary.Values], globalContext: string.Join(";", changesMade), tspProjectPath, ct: ct); + bool buildSucceeded = false; + string? buildError = null; + + // ── Pass 1: Classify feedback and apply TSP fixes ── + // TODO - need to update this to avoid casting to/from list + var response = await _classifierService.ClassifyItemsAsync([.. feedbackDictionary.Values], globalContext: string.Join(";", changesMade), tspProjectPath, ct: ct); - if (response.Classifications == null || response.Classifications.Count == 0) + if (response.Classifications == null || response.Classifications.Count == 0) + { + return new CustomizedCodeUpdateResponse { - if (tries == 0) - { - return new CustomizedCodeUpdateResponse - { - Success = false, - ResponseError = "Feedback could not be classified.", - Message = "Feedback could not be classified.", - ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.InvalidInput, - BuildResult = "Feedback could not be classified." - }; - } + Success = false, + ResponseError = "Feedback could not be classified.", + Message = "Feedback could not be classified.", + ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.InvalidInput, + BuildResult = "Feedback could not be classified." + }; + } - // On subsequent iterations, no classifications means nothing more to fix via TSP - logger.LogInformation("No further TSP-applicable classifications found on iteration {Iteration}.", tries + 1); - break; - } + var tspFixFailed = 0; + var tspFixSucceeded = 0; + var tspApplicable = 0; + var codeCustomizations = 0; + var manualChanges = 0; + var noChanges = 0; - var tspFixFailed = 0; - var tspFixSucceeded = 0; - var tspApplicable = 0; - var codeCustomizations = 0; - var manualChanges = 0; - var noChanges = 0; + foreach (var itemDetails in response.Classifications) + { + feedbackDictionary.TryGetValue(itemDetails.ItemId, out var feedbackItem); - foreach (var itemDetails in response.Classifications) + if (feedbackItem == null) { - feedbackDictionary.TryGetValue(itemDetails.ItemId, out var feedbackItem); + logger.LogWarning("Classifier returned non-existent feedback item ID '{ItemId}', skipping.", itemDetails.ItemId); + continue; + } - if (feedbackItem == null) - { - logger.LogWarning("Classifier returned non-existent feedback item ID '{ItemId}', skipping.", itemDetails.ItemId); - continue; - } + if (itemDetails.Classification == ClassificationTspApplicable) + { + tspApplicable++; + logger.LogDebug("Applying tsp customization for: {feedback}", itemDetails.Text); + var tspCustomizationResult = await typeSpecCustomizationService.ApplyCustomizationAsync(tspProjectPath, itemDetails.Text, ct: ct); - if (itemDetails.Classification == ClassificationTspApplicable) + if (tspCustomizationResult.Success) { - tspApplicable++; - feedbackItem.AppendContext($"Iteration {tries+1}"); - logger.LogDebug("Applying tsp customization for: {feedback}", itemDetails.Text); - var tspCustomizationResult = await typeSpecCustomizationService.ApplyCustomizationAsync(tspProjectPath, itemDetails.Text, ct: ct); - - if (tspCustomizationResult.Success) - { - var changes = string.Join("; ", tspCustomizationResult.ChangesSummary); - logger.LogInformation("Successfully applied tsp customization changes, changes applied: {changes}", changes); - feedbackItem.AppendContext(changes, "Typespec changes applied"); - changesMade.AddRange(tspCustomizationResult.ChangesSummary); - tspFixSucceeded++; - } - else - { - logger.LogWarning("Some customizations failed to apply: {FailureReasons}", tspCustomizationResult.FailureReason); - tspFixFailedReasons.Append(tspCustomizationResult.FailureReason); - tspFixFailedReasons.Append("; "); - tspFixFailed++; - } + var changes = string.Join("; ", tspCustomizationResult.ChangesSummary); + logger.LogInformation("Successfully applied tsp customization changes, changes applied: {changes}", changes); + feedbackItem.AppendContext(changes, "Typespec changes applied"); + changesMade.AddRange(tspCustomizationResult.ChangesSummary); + tspFixSucceeded++; } - else if (itemDetails.Classification == ClassificationCodeCustomization) + else { - codeCustomizations++; - logger.LogInformation("Item '{ItemId}' classified as CODE_CUSTOMIZATION — will be handled via code patching.", itemDetails.ItemId); - classifierAnalysis.AppendLine($"[{itemDetails.ItemId}] Classification: {itemDetails.Classification}, Reason: {itemDetails.Reason}"); - - // Don't try and fix the same feedback again - feedbackDictionary.Remove(itemDetails.ItemId); + logger.LogWarning("Some customizations failed to apply: {FailureReasons}", tspCustomizationResult.FailureReason); + feedbackItem.AppendContext(tspCustomizationResult.FailureReason ?? "Unknown failure", "TypeSpec customization failed"); + tspFixFailedReasons.Append(tspCustomizationResult.FailureReason); + tspFixFailedReasons.Append("; "); + tspFixFailed++; } - else if (itemDetails.Classification == ClassificationRequiresManualIntervention) - { - manualChanges++; - manualInterventions.Add($"'{itemDetails.Text}' (Reason: {itemDetails.Reason})"); + } + else if (itemDetails.Classification == ClassificationCodeCustomization) + { + codeCustomizations++; + logger.LogInformation("Item '{ItemId}' classified as CODE_CUSTOMIZATION — will be handled via code patching.", itemDetails.ItemId); + classifierAnalysis.AppendLine($"[{itemDetails.ItemId}] Classification: {itemDetails.Classification}, Reason: {itemDetails.Reason}"); + feedbackDictionary.Remove(itemDetails.ItemId); + } + else if (itemDetails.Classification == ClassificationRequiresManualIntervention) + { + manualChanges++; + manualInterventions.Add($"'{itemDetails.Text}' (Reason: {itemDetails.Reason})"); + feedbackDictionary.Remove(itemDetails.ItemId); + } + else if (itemDetails.Classification == ClassificationSuccess) + { + noChanges++; + feedbackDictionary.Remove(itemDetails.ItemId); + } + } - // Don't try and fix the same feedback again - feedbackDictionary.Remove(itemDetails.ItemId); - } - else if (itemDetails.Classification == ClassificationSuccess) - { - noChanges++; + // ── Early exit cases based on first classification ── - // Don't try and fix the same feedback again - feedbackDictionary.Remove(itemDetails.ItemId); - } - } + // Nothing was classified as tsp applicable and at least some feedback requires manual intervention + if (tspApplicable == 0 && codeCustomizations == 0 && manualChanges > 0) + { + return new CustomizedCodeUpdateResponse + { + Success = true, + Message = "The requested changes require manual intervention and cannot be applied via TypeSpec customizations.", + NextSteps = manualInterventions, + ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.ManualInterventionRequired + }; + } - // Exit cases for the first attempt - if (tries == 0) + // Everything was classified as success + if (tspApplicable == 0 && codeCustomizations == 0 && noChanges > 0) + { + return new CustomizedCodeUpdateResponse { - // Nothing was classified as tsp applicable and at least some feedback requires manual intervention - if (tspApplicable == 0 && codeCustomizations == 0 && manualChanges > 0) - { - return new CustomizedCodeUpdateResponse - { - Success = true, - Message = "The requested changes require manual intervention and cannot be applied via TypeSpec customizations.", - NextSteps = manualInterventions, - ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.ManualInterventionRequired - }; - } + Success = true, + Message = "No changes needed — the requested customizations are already in place." + }; + } - // Everything was classified as success - if (tspApplicable == 0 && codeCustomizations == 0 && noChanges > 0) + // ── Regen + Build if TSP fixes were applied ── + if (tspFixSucceeded > 0) + { + logger.LogDebug("Regenerating {packagePath}", packagePath); + var regenResult = await tspClientHelper.UpdateGenerationAsync(packagePath, localSpecRepoPath: tspProjectPath, isCli: false, ct: ct); + if (!regenResult.IsSuccessful) + { + logger.LogWarning("Regeneration failed: {Error}", regenResult.ResponseError); + // Enrich remaining items with regen failure context for the second classifier pass + foreach (var item in feedbackDictionary.Values) { - return new CustomizedCodeUpdateResponse - { - Success = true, - Message = "No changes needed — the requested customizations are already in place." - }; + item.AppendContext($"Regeneration failed: {regenResult.ResponseError}", "Regeneration Result"); } + } + else + { + logger.LogDebug("Building {packagePath}", packagePath); + var (success, error, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); + buildSucceeded = success; + buildError = error; - // All tsp fixes failed to be applied - signaling a deeper error - if (tspFixSucceeded == 0 && tspFixFailed > 0) + if (buildSucceeded && codeCustomizations == 0) { + logger.LogInformation("Build passed after TypeSpec customizations."); return new CustomizedCodeUpdateResponse { - Success = false, - ResponseError = $"Failed to apply any TypeSpec customizations: {tspFixFailedReasons}", - Message = $"Failed to apply any TypeSpec customizations: {tspFixFailedReasons}", - ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.TypeSpecCustomizationFailed + Success = true, + Message = "Build passed after attempting TypeSpec customizations.", + TypeSpecChangesSummary = changesMade, + NextSteps = manualInterventions, }; } - // All items are code customizations — no TSP changes were made, skip regen. - // Build will happen after the loop to get error context for the patch agent. - if (tspApplicable == 0 && codeCustomizations > 0) - { - logger.LogInformation("All items classified as CODE_CUSTOMIZATION — skipping regen."); - break; - } - } - - // No TSP progress on subsequent iterations — stop looping - if (tries > 0 && tspFixSucceeded == 0) - { - logger.LogInformation("No TSP fixes succeeded on iteration {Iteration}, stopping.", tries + 1); - break; - } - - // Don't waste time regenerating if no TSP fixes were successfully applied - if (tspFixSucceeded > 0) - { - logger.LogDebug("Regenerating {packagePath}", packagePath); - // Regenerate SDK using local spec repo - var regenResult = await tspClientHelper.UpdateGenerationAsync(packagePath, localSpecRepoPath: tspProjectPath, isCli: false, ct: ct); - if (!regenResult.IsSuccessful) + // Enrich remaining items with build error context for the second classifier pass + if (!buildSucceeded) { - logger.LogWarning("Regeneration failed: {Error}", regenResult.ResponseError); - - // Append regen failure context so the classifier can re-classify for manual intervention - var regenContext = $"Regeneration failed: {regenResult.ResponseError}"; foreach (var item in feedbackDictionary.Values) { - item.AppendContext(regenContext, "Regeneration Result"); + item.AppendContext(error ?? "Build failed with unknown error.", "Build Result"); } - - buildSucceeded = false; - // Reset buildError — no code was generated, nothing new to build - buildError = null; - tries++; - continue; } } + } - // Always build after regen to get fresh error context for the classifier - logger.LogDebug("Building {packagePath}", packagePath); - var (success, error, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); - - buildSucceeded = success; - buildError = error; + // ── Pass 2: Re-classify remaining items with regen/build context ── + // Items that had TSP fixes applied but regen/build failed get re-evaluated. + // The classifier can now reclassify them as CODE_CUSTOMIZATION or REQUIRES_MANUAL_INTERVENTION. + if (feedbackDictionary.Count > 0) + { + var secondResponse = await _classifierService.ClassifyItemsAsync([.. feedbackDictionary.Values], globalContext: string.Join(";", changesMade), tspProjectPath, ct: ct); - // Append build result context to all remaining feedback items for the next iteration - if (tries + 1 < maxTries) + if (secondResponse.Classifications != null) { - var buildContext = success ? "Build succeeded." : (error ?? "Build failed with unknown error."); - foreach (var item in feedbackDictionary.Values) + foreach (var itemDetails in secondResponse.Classifications) { - item.AppendContext(buildContext, "Build Result"); + if (itemDetails.Classification == ClassificationCodeCustomization) + { + codeCustomizations++; + logger.LogInformation("Item '{ItemId}' reclassified as CODE_CUSTOMIZATION on second pass.", itemDetails.ItemId); + classifierAnalysis.AppendLine($"[{itemDetails.ItemId}] Classification: {itemDetails.Classification}, Reason: {itemDetails.Reason}"); + feedbackDictionary.Remove(itemDetails.ItemId); + } + else if (itemDetails.Classification == ClassificationRequiresManualIntervention) + { + manualInterventions.Add($"'{itemDetails.Text}' (Reason: {itemDetails.Reason})"); + feedbackDictionary.Remove(itemDetails.ItemId); + } + else if (itemDetails.Classification == ClassificationSuccess) + { + feedbackDictionary.Remove(itemDetails.ItemId); + } } } + } - tries++; - } while (!buildSucceeded && tries < maxTries); - - // Build for error context if no build happened in the loop (pure CODE_CUSTOMIZATION path) + // Build for error context if no build happened yet (pure CODE_CUSTOMIZATION path or regen failed) if (!buildSucceeded && buildError == null) { logger.LogInformation("Building for error context..."); @@ -407,9 +391,9 @@ private async Task RunUpdateAsync(string packagePa buildError = e; } - if (buildSucceeded) + if (buildSucceeded && codeCustomizations == 0) { - logger.LogInformation("Build passed after Typespec customizations."); + logger.LogInformation("Build passed after TypeSpec customizations."); return new CustomizedCodeUpdateResponse { Success = true, @@ -419,7 +403,7 @@ private async Task RunUpdateAsync(string packagePa }; } - // Step 2: If the build failed, start customized code update process + // Step 2: If the build failed or CODE_CUSTOMIZATION items still need patching, start customized code update process if (!languageService.IsCustomizedCodeUpdateSupported) { @@ -497,34 +481,36 @@ private async Task RunUpdateAsync(string packagePa } // Step 6: Final build to validate patches - logger.LogInformation("Running final build to validate patches..."); + logger.LogInformation("Running final build to validate code customization patches..."); var (finalBuildSuccess, finalBuildError, _) = await languageService.BuildAsync(packagePath, CommandTimeoutInMinutes, ct); if (finalBuildSuccess) { - logger.LogInformation("Build passed after repairs."); + logger.LogInformation("Build passed after code customization patches."); return new CustomizedCodeUpdateResponse { Success = true, - Message = "Build passed after repairs.", + Message = "Build passed after code customization patches.", TypeSpecChangesSummary = changesMade, - AppliedPatches = patches + AppliedPatches = patches, + NextSteps = manualInterventions, }; } // Build still failing after patches - logger.LogInformation("Build still failing after patches."); + logger.LogInformation("Build still failing after code customization patches."); return new CustomizedCodeUpdateResponse { Success = false, ResponseError = string.IsNullOrWhiteSpace(finalBuildError) - ? "Patches applied but build still failing." - : $"Patches applied but build still failing.\n{finalBuildError}", - Message = "Patches applied but build still failing.", + ? "Code customization patches applied but build still failing." + : $"Code customization patches applied but build still failing.\n{finalBuildError}", + Message = "Code customization patches applied but build still failing.", ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.BuildAfterPatchesFailed, BuildResult = finalBuildError, TypeSpecChangesSummary = changesMade, - AppliedPatches = patches + AppliedPatches = patches, + NextSteps = manualInterventions, }; } From 3052138573237c32e44416d4024a7a521842a739 Mon Sep 17 00:00:00 2001 From: Sameeksha Vaity Date: Wed, 18 Mar 2026 13:44:54 -0700 Subject: [PATCH 4/4] update for manual intervention success reporting --- .../TypeSpec/CustomizedCodeUpdateToolTests.cs | 8 ++-- .../TypeSpec/CustomizedCodeUpdateTool.cs | 43 +++++++++++-------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs index 89a88e107e8..78e9ce726b8 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/TypeSpec/CustomizedCodeUpdateToolTests.cs @@ -155,7 +155,7 @@ public async Task TspFix_BuildFailsAfterRegen_FallsThroughToPatching() var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); Assert.That(result.Success, Is.True); - Assert.That(result.Message, Does.Contain("Build passed after repairs")); + Assert.That(result.Message, Does.Contain("Build passed after code customization patches.")); } // ======================================================================== @@ -284,7 +284,7 @@ public async Task Classification_OnlyNonTspItems_ReturnsSuccess() var tspDir = CreateTempDir(); var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); - Assert.That(result.Success, Is.True); + Assert.That(result.Success, Is.False); Assert.That(result.Message, Does.Contain("manual intervention")); } @@ -702,7 +702,7 @@ public async Task BuildFails_PatchesApplied_FinalBuildSucceeds_ReturnsSuccess() var result = await tool.UpdateAsync(packagePath: pkg, tspProjectPath: tspDir, customizationRequest: "test customization", ct: CancellationToken.None); Assert.That(result.Success, Is.True); - Assert.That(result.Message, Does.Contain("Build passed after repairs")); + Assert.That(result.Message, Does.Contain("Build passed after code customization patches.")); Assert.That(result.AppliedPatches, Is.Not.Null.And.Count.EqualTo(1)); } @@ -800,7 +800,7 @@ public async Task CodeCustomization_ClassifiedItem_SkipsTsp_PatchesApplied_Build // Verify the CODE_CUSTOMIZATION route succeeded end-to-end Assert.That(result.Success, Is.True); - Assert.That(result.Message, Does.Contain("Build passed after repairs")); + Assert.That(result.Message, Does.Contain("Build passed after code customization patches.")); Assert.That(result.AppliedPatches, Is.Not.Null.And.Count.EqualTo(1)); Assert.That(result.AppliedPatches![0].FilePath, Is.EqualTo("SpeechTranscriptionCustomization.java")); Assert.That(result.AppliedPatches[0].ReplacementCount, Is.EqualTo(2)); diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs index 361866a33c8..330ec1fce3b 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/TypeSpec/CustomizedCodeUpdateTool.cs @@ -6,13 +6,11 @@ using Azure.Sdk.Tools.Cli.Commands; using Azure.Sdk.Tools.Cli.Helpers; using Azure.Sdk.Tools.Cli.Models; -using Azure.Sdk.Tools.Cli.Models.Responses; using Azure.Sdk.Tools.Cli.Models.Responses.Package; using Azure.Sdk.Tools.Cli.Services; using Azure.Sdk.Tools.Cli.Services.Languages; using Azure.Sdk.Tools.Cli.Services.TypeSpec; using Azure.Sdk.Tools.Cli.Tools.Core; -using Microsoft.Graph.Models; using ModelContextProtocol.Server; namespace Azure.Sdk.Tools.Cli.Tools.TypeSpec; @@ -207,12 +205,12 @@ private async Task RunUpdateAsync(string packagePa List changesMade = new(); List manualInterventions = new(); - StringBuilder classifierAnalysis = new(); + StringBuilder codeCustomizationLog = new(); StringBuilder tspFixFailedReasons = new(); bool buildSucceeded = false; string? buildError = null; - // ── Pass 1: Classify feedback and apply TSP fixes ── + // Step 1: Classify feedback and apply TSP fixes // TODO - need to update this to avoid casting to/from list var response = await _classifierService.ClassifyItemsAsync([.. feedbackDictionary.Values], globalContext: string.Join(";", changesMade), tspProjectPath, ct: ct); @@ -273,7 +271,7 @@ private async Task RunUpdateAsync(string packagePa { codeCustomizations++; logger.LogInformation("Item '{ItemId}' classified as CODE_CUSTOMIZATION — will be handled via code patching.", itemDetails.ItemId); - classifierAnalysis.AppendLine($"[{itemDetails.ItemId}] Classification: {itemDetails.Classification}, Reason: {itemDetails.Reason}"); + codeCustomizationLog.AppendLine($"[{itemDetails.ItemId}] Classification: {itemDetails.Classification}, Reason: {itemDetails.Reason}"); feedbackDictionary.Remove(itemDetails.ItemId); } else if (itemDetails.Classification == ClassificationRequiresManualIntervention) @@ -296,7 +294,7 @@ private async Task RunUpdateAsync(string packagePa { return new CustomizedCodeUpdateResponse { - Success = true, + Success = false, Message = "The requested changes require manual intervention and cannot be applied via TypeSpec customizations.", NextSteps = manualInterventions, ErrorCode = CustomizedCodeUpdateResponse.KnownErrorCodes.ManualInterventionRequired @@ -339,10 +337,13 @@ private async Task RunUpdateAsync(string packagePa logger.LogInformation("Build passed after TypeSpec customizations."); return new CustomizedCodeUpdateResponse { - Success = true, - Message = "Build passed after attempting TypeSpec customizations.", + Success = manualInterventions.Count == 0, + Message = manualInterventions.Count == 0 + ? "Build passed after attempting TypeSpec customizations." + : "Build passed after attempting TypeSpec customizations, but some items require manual intervention.", TypeSpecChangesSummary = changesMade, NextSteps = manualInterventions, + ErrorCode = manualInterventions.Count > 0 ? CustomizedCodeUpdateResponse.KnownErrorCodes.ManualInterventionRequired : null, }; } @@ -372,7 +373,7 @@ private async Task RunUpdateAsync(string packagePa { codeCustomizations++; logger.LogInformation("Item '{ItemId}' reclassified as CODE_CUSTOMIZATION on second pass.", itemDetails.ItemId); - classifierAnalysis.AppendLine($"[{itemDetails.ItemId}] Classification: {itemDetails.Classification}, Reason: {itemDetails.Reason}"); + codeCustomizationLog.AppendLine($"[{itemDetails.ItemId}] Classification: {itemDetails.Classification}, Reason: {itemDetails.Reason}"); feedbackDictionary.Remove(itemDetails.ItemId); } else if (itemDetails.Classification == ClassificationRequiresManualIntervention) @@ -402,10 +403,13 @@ private async Task RunUpdateAsync(string packagePa logger.LogInformation("Build passed after TypeSpec customizations."); return new CustomizedCodeUpdateResponse { - Success = true, - Message = "Build passed after attempting TypeSpec customizations.", + Success = manualInterventions.Count == 0, + Message = manualInterventions.Count == 0 + ? "Build passed after attempting TypeSpec customizations." + : "Build passed after attempting TypeSpec customizations, but some items require manual intervention.", TypeSpecChangesSummary = changesMade, NextSteps = manualInterventions, + ErrorCode = manualInterventions.Count > 0 ? CustomizedCodeUpdateResponse.KnownErrorCodes.ManualInterventionRequired : null, }; } @@ -441,7 +445,7 @@ private async Task RunUpdateAsync(string packagePa } // Step 4: Apply patches based on build errors - var patchContext = BuildPatchContext(customizationRequest, classifierAnalysis, buildError); + var patchContext = BuildPatchContext(customizationRequest, codeCustomizationLog, buildError); logger.LogInformation("Applying patches to fix build errors..."); var patches = await languageService.ApplyPatchesAsync( @@ -500,11 +504,14 @@ private async Task RunUpdateAsync(string packagePa logger.LogInformation("Build passed after code customization patches."); return new CustomizedCodeUpdateResponse { - Success = true, - Message = "Build passed after code customization patches.", + Success = manualInterventions.Count == 0, + Message = manualInterventions.Count == 0 + ? "Build passed after code customization patches." + : "Build passed after code customization patches, but some items require manual intervention.", TypeSpecChangesSummary = changesMade, AppliedPatches = patches, NextSteps = manualInterventions, + ErrorCode = manualInterventions.Count > 0 ? CustomizedCodeUpdateResponse.KnownErrorCodes.ManualInterventionRequired : null, }; } @@ -558,10 +565,10 @@ private async Task ResolveLanguageServiceAsync(string packagePa /// classifier analysis, and build errors into labeled markdown sections. /// /// The original user customization request text. - /// Accumulated classifier analysis from all classification iterations. + /// Accumulated code customization classification log from all classification passes. /// The build error output, if any. /// A formatted markdown string combining all available context sections. - internal static string BuildPatchContext(string? customizationRequest, StringBuilder classifierAnalysis, string? buildError) + internal static string BuildPatchContext(string? customizationRequest, StringBuilder codeCustomizationLog, string? buildError) { var sb = new StringBuilder(); if (!string.IsNullOrWhiteSpace(customizationRequest)) @@ -570,10 +577,10 @@ internal static string BuildPatchContext(string? customizationRequest, StringBui sb.AppendLine(customizationRequest); sb.AppendLine(); } - if (classifierAnalysis.Length > 0) + if (codeCustomizationLog.Length > 0) { sb.AppendLine("## Classifier Analysis"); - sb.AppendLine(classifierAnalysis.ToString()); + sb.AppendLine(codeCustomizationLog.ToString()); } if (!string.IsNullOrWhiteSpace(buildError)) {