diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/SessionExecutor.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/SessionExecutor.cs index bde091321f1..b830755c6e0 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/SessionExecutor.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/SessionExecutor.cs @@ -22,7 +22,8 @@ public class SessionExecutor : IDisposable public async Task ExecuteAsync(ExecutionConfig config) { var stopwatch = Stopwatch.StartNew(); - var toolCalls = new List(); + var toolCalls = new List(); + var pendingTimestamps = new Dictionary(); try { @@ -56,10 +57,28 @@ public async Task ExecuteAsync(ExecutionConfig config) { Console.WriteLine($"Model is calling tool: {input.ToolName}"); config.OnActivity?.Invoke($"Calling tool: {input.ToolName}"); + pendingTimestamps[input.ToolName] = input.Timestamp; return Task.FromResult(null); }, OnPostToolUse = (input, invocation) => { + double? durationMs = pendingTimestamps.TryGetValue(input.ToolName, out var startTs) + ? input.Timestamp - startTs + : null; + + var mcpServerName = input.ToolName.Contains("__") + ? input.ToolName.Split("__", 2)[0] + : null; + + toolCalls.Add(new ToolCallRecord + { + ToolName = input.ToolName, + ToolArgs = input.ToolArgs, + ToolResult = input.ToolResult, + DurationMs = durationMs, + McpServerName = mcpServerName, + Timestamp = startTs, + }); if (input.ToolName == "skill") { toolCalls.Add($"{input.ToolName} {input.ToolArgs?.ToString()}"); diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/Workspace.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/Workspace.cs index 8e6962eb75b..185f4671161 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/Workspace.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/Workspace.cs @@ -172,7 +172,7 @@ public async Task ReadFileAsync(string relativePath) public async Task WriteExecutionLogAsync( string scenarioName, IReadOnlyList messages, - IReadOnlyList toolCalls, + IReadOnlyList toolCalls, string? gitDiff, TimeSpan duration, bool passed, diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/WorkspaceManager.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/WorkspaceManager.cs index cb8fdc5601c..aaa52a8efa1 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/WorkspaceManager.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Infrastructure/WorkspaceManager.cs @@ -62,6 +62,8 @@ public async Task PrepareAsync(RepoConfig repo, string scenarioId) // Create worktree using the resolved SHA (safe for concurrent use) await CreateWorktreeAsync(barePath, worktreePath, commitSha, repo.SparseCheckoutPaths); + SetupWorkspaceEnvironment(); + return new Workspace(workspaceRoot, repo.Name); } @@ -279,4 +281,10 @@ private static string EscapeArgument(string arg) return arg; } + + private void SetupWorkspaceEnvironment() + { + // Force test mode for the tools. + Environment.SetEnvironmentVariable("AZSDKTOOLS_AGENT_TESTING", "true"); + } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/BenchmarkResult.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/BenchmarkResult.cs index 6aac5f51be4..df4207abe5a 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/BenchmarkResult.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/BenchmarkResult.cs @@ -34,9 +34,9 @@ public class BenchmarkResult public string? GitDiff { get; init; } /// - /// Gets the list of tool calls made during execution. + /// Gets the tool calls made during execution. /// - public IReadOnlyList ToolCalls { get; init; } = []; + public IReadOnlyList ToolCalls { get; init; } = []; /// /// Gets the path to the workspace where the benchmark was executed. diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ExecutionResult.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ExecutionResult.cs index 447e00eeae1..fdd541a1874 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ExecutionResult.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ExecutionResult.cs @@ -20,6 +20,6 @@ public class ExecutionResult /// The conversation messages from the session. public IReadOnlyList Messages { get; init; } = []; - /// Tool calls made during execution (for debugging). - public IReadOnlyList ToolCalls { get; init; } = []; + /// Tool calls made during execution. + public IReadOnlyList ToolCalls { get; init; } = []; } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ExpectedToolCall.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ExpectedToolCall.cs new file mode 100644 index 00000000000..ab141462ced --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ExpectedToolCall.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Models; + +/// +/// Describes an expected tool call with optional input validation. +/// +public class ExpectedToolCall +{ + /// Gets the expected tool name (short name without MCP prefix). + public string ToolName { get; } + + /// + /// Gets the expected input key-value pairs to validate, or null to skip input validation. + /// Keys are parameter names; values are the expected values. + /// String values use case-insensitive substring matching (to handle variable path prefixes). + /// Numeric and boolean values use exact matching. + /// + public IReadOnlyDictionary ExpectedInputs { get; } + + /// + /// Creates an expected tool call that only validates the tool was called (no input checks). + /// + public ExpectedToolCall(string toolName) + { + ToolName = toolName; + ExpectedInputs = new Dictionary(); + } + + /// + /// Creates an expected tool call that validates both the tool name and its inputs. + /// + public ExpectedToolCall(string toolName, Dictionary expectedInputs) + { + ToolName = toolName; + ExpectedInputs = expectedInputs; + } + + public override string ToString() => ToolName; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ToolCallRecord.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ToolCallRecord.cs new file mode 100644 index 00000000000..ec2cbb4897b --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ToolCallRecord.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Text.Json; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Models; + +/// +/// Represents a tool call captured during benchmark execution. +/// +public class ToolCallRecord +{ + /// Gets the tool name (may include MCP prefix). + public required string ToolName { get; init; } + + /// + /// Gets the arguments passed to the tool, or null if unavailable. + /// The raw object from the SDK hook (typically a JsonElement). + /// + public object? ToolArgs { get; init; } + + /// Gets the result returned by the tool, or null if unavailable. + public object? ToolResult { get; init; } + + /// Gets the tool call duration in milliseconds, or null if unavailable. + public double? DurationMs { get; init; } + + /// Gets the MCP server name extracted from the tool name prefix, or null. + public string? McpServerName { get; init; } + + /// Gets the timestamp when the tool call was recorded. + public DateTime Timestamp { get; init; } = DateTime.UtcNow; + + /// + /// Gets the tool arguments as a string-keyed dictionary of JsonElements. + /// Returns an empty dictionary if args are null or not a JSON object. + /// + public IReadOnlyDictionary GetArgsAsDictionary() + { + if (ToolArgs is not JsonElement { ValueKind: JsonValueKind.Object } jsonElement) + { + return new Dictionary(); + } + + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var prop in jsonElement.EnumerateObject()) + { + dict[prop.Name] = prop.Value; + } + return dict; + } + + public override string ToString() => ToolName; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ValidationContext.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ValidationContext.cs index 69cf19637e2..5d5c08271ae 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ValidationContext.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Models/ValidationContext.cs @@ -48,7 +48,7 @@ public class ValidationContext /// /// Gets the tool calls made during execution. /// - public IReadOnlyList ToolCalls { get; init; } = []; + public IReadOnlyList ToolCalls { get; init; } = []; /// /// Gets the conversation messages from the agent session. diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/GitHub/GetPrLinkCurrentBranchScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/GitHub/GetPrLinkCurrentBranchScenario.cs new file mode 100644 index 00000000000..44cdb2ffe8e --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/GitHub/GetPrLinkCurrentBranchScenario.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_get_pull_request_link_for_current_branch +/// when asked about the status of a spec PR on the current branch. +/// Migrated from evaluation scenario: Evaluate_GetPullRequestLinkForCurrentBranch. +/// +public class GetPrLinkCurrentBranchScenario : BenchmarkScenario +{ + /// + public override string Name => "get-pr-link-current-branch"; + + /// + public override string Description => + "Verify the agent calls azsdk_get_pull_request_link_for_current_branch for PR status."; + + /// + public override string[] Tags => ["github"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + What's the status of the spec PR in my current branch? Only check the status once. + My setup has already been verified, do not run azsdk_verify_setup. + The repository root is the relative path ./azure-rest-api-specs. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tool: azsdk_get_pull_request_link_for_current_branch", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_get_pull_request_link_for_current_branch") + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Pipeline/CheckSdkGenerationStatusScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Pipeline/CheckSdkGenerationStatusScenario.cs new file mode 100644 index 00000000000..c9ceb7a04f1 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Pipeline/CheckSdkGenerationStatusScenario.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_get_pipeline_status when asked +/// to check SDK generation pipeline status. +/// Adapted from evaluation scenario: Evaluate_CheckSDKGenerationStatus. +/// The original scenario loaded a mid-conversation JSON trace; this benchmark uses a +/// standalone prompt capturing the same intent. +/// +public class CheckSdkGenerationStatusScenario : BenchmarkScenario +{ + /// + public override string Name => "check-sdk-generation-status"; + + /// + public override string Description => + "Verify the agent calls azsdk_get_pipeline_status to check SDK generation."; + + /// + public override string[] Tags => ["pipeline"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + Check the SDK generation pipeline status for build ID 5513110. + My setup has already been verified, do not run azsdk_verify_setup. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tool: azsdk_get_pipeline_status", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_get_pipeline_status", + new Dictionary + { + ["buildId"] = 5513110 + }) + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/ReleasePlan/CreateReleasePlanScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/ReleasePlan/CreateReleasePlanScenario.cs new file mode 100644 index 00000000000..0a357e61a5c --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/ReleasePlan/CreateReleasePlanScenario.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes the azsdk_create_release_plan tool when asked to create a release plan. +/// Migrated from evaluation scenario: Evaluate_CreateReleasePlan. +/// +public class CreateReleasePlanScenario : BenchmarkScenario +{ + /// + public override string Name => "create-release-plan"; + + /// + public override string Description => + "Verify the agent calls azsdk_create_release_plan with appropriate context."; + + /// + public override string[] Tags => ["release-plan"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + Create a release plan for the Contoso Widget Manager, no need to get it afterwards only create. + My setup has already been verified, do not run azsdk_verify_setup. Here is all the context you need: + TypeSpec project located at "specification/contosowidgetmanager/Contoso.WidgetManager". + Use service tree ID "a7f2b8e4-9c1d-4a3e-b6f9-2d8e5a7c3b1f", + product tree ID "f1a8c5d2-6e4b-4f7a-9c2d-8b5e1f3a6c9e", + target release timeline "December 2025", + API version "2022-11-01-preview", + SDK release type "beta", + and link it to the spec pull request "https://github.com/Azure/azure-rest-api-specs/pull/38387". + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tool: azsdk_create_release_plan", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_create_release_plan", + new Dictionary + { + ["typeSpecProjectPath"] = "specification/contosowidgetmanager/Contoso.WidgetManager", + ["serviceTreeId"] = "a7f2b8e4-9c1d-4a3e-b6f9-2d8e5a7c3b1f", + ["productTreeId"] = "f1a8c5d2-6e4b-4f7a-9c2d-8b5e1f3a6c9e", + ["specApiVersion"] = "2022-11-01-preview", + ["specPullRequestUrl"] = "https://github.com/Azure/azure-rest-api-specs/pull/38387", + ["sdkReleaseType"] = "beta" + }) + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/ReleasePlan/LinkNamespaceApprovalIssueScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/ReleasePlan/LinkNamespaceApprovalIssueScenario.cs new file mode 100644 index 00000000000..2aa0f30cb02 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/ReleasePlan/LinkNamespaceApprovalIssueScenario.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_link_namespace_approval_issue +/// when asked to link a namespace approval issue to a release plan. +/// Migrated from evaluation scenario: Evaluate_LinkNamespaceApprovalIssue. +/// +public class LinkNamespaceApprovalIssueScenario : BenchmarkScenario +{ + /// + public override string Name => "link-namespace-approval-issue"; + + /// + public override string Description => + "Verify the agent calls azsdk_link_namespace_approval_issue to link an issue to a release plan."; + + /// + public override string[] Tags => ["release-plan"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + Link namespace approval issue https://github.com/Azure/azure-sdk/issues/1234 to release plan 12345. + My setup has already been verified, do not run azsdk_verify_setup. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tool: azsdk_link_namespace_approval_issue", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_link_namespace_approval_issue", + new Dictionary + { + ["releasePlanWorkItemId"] = 12345, + ["namespaceApprovalIssue"] = "https://github.com/Azure/azure-sdk/issues/1234" + }) + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/AddArmResourceScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/AddArmResourceScenario.cs new file mode 100644 index 00000000000..ca16a0e66d9 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/AddArmResourceScenario.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Infrastructure; +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// POC scenario that tests adding a new ARM resource to a TypeSpec-based ARM specification. +/// Target: Add an 'Asset' ARM resource with CRUD operations to specification/widget/resource-manager/Microsoft.Widget/Widget. +/// +public class AddArmResourceScenario : BenchmarkScenario +{ + /// + public override string Name => "add-arm-resource"; + + /// + public override string Description => + "Add a new ARM resource to the specification."; + + /// + public override string[] Tags => ["typespec", "authoring", "poc"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/widget/resource-manager/Microsoft.Widget/Widget"] + }; + + /// + public override string Prompt => """ + In the specification/widget/resource-manager/Microsoft.Widget/Widget project, + add an ARM resource named 'Asset' with CRUD operations. + """; + + /// + public override TimeSpan Timeout => TimeSpan.FromMinutes(5); + + /// + public override async Task SetupAsync(Workspace workspace) + { + await workspace.RunCommandAsync("npm", "ci"); + } + + /// + public override IEnumerable Validators => + [ + // Debug: Report what context is loaded + // new ContextReportValidator(), + + // Verify the project entry point still exists + new FileExistsValidator("Project entry point exists", + "specification/widget/resource-manager/Microsoft.Widget/Widget/main.tsp"), + + // Verify the new Asset resource file was created + new FileExistsValidator("Asset resource file created", + "specification/widget/resource-manager/Microsoft.Widget/Widget/asset.tsp"), + + // Verify the new file contains the expected ARM resource patterns + new ContainsValidator("Asset file has ARM resource model", + filePath: "specification/widget/resource-manager/Microsoft.Widget/Widget/asset.tsp", + patterns: ["model Asset is TrackedResource", "...ResourceNameParameter", "model AssetProperties"]), + + // Verify CRUD operations are present in the new file + new ContainsValidator("Asset file has CRUD operations", + filePath: "specification/widget/resource-manager/Microsoft.Widget/Widget/asset.tsp", + patterns: [ + "@armResourceOperations", + "interface Assets", + "get is ArmResourceRead", + "ArmResourceCreateOrReplace", + "update is Arm", + "delete is ArmResourceDeleteWithoutOkAsync", + "listByResourceGroup is ArmResourceListByParent", + "listBySubscription is ArmListBySubscription" + ]), + + // Verify main.tsp imports the new asset file + new ContainsValidator("main.tsp imports asset", + filePath: "specification/widget/resource-manager/Microsoft.Widget/Widget/main.tsp", + patterns: ["asset.tsp"]), + + // Verify the project compiles successfully + new CommandValidator("tsp compile succeeds", + command: "npx", + arguments: ["tsp", "compile", "./specification/widget/resource-manager/Microsoft.Widget/Widget/main.tsp"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/CheckPublicRepoScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/CheckPublicRepoScenario.cs new file mode 100644 index 00000000000..e67ebca6fad --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/CheckPublicRepoScenario.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_typespec_check_project_in_public_repo +/// when asked to check if a TypeSpec project is in the public repo. +/// Migrated from evaluation scenario: Evaluate_CheckPublicRepo. +/// +public class CheckPublicRepoScenario : BenchmarkScenario +{ + /// + public override string Name => "check-public-repo"; + + /// + public override string Description => + "Verify the agent calls azsdk_typespec_check_project_in_public_repo."; + + /// + public override string[] Tags => ["typespec"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + Check if my TypeSpec project is in the public repo. + My setup has already been verified, do not run azsdk_verify_setup. + Project root: specification/contosowidgetmanager/Contoso.WidgetManager. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tool: azsdk_typespec_check_project_in_public_repo", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_typespec_check_project_in_public_repo", + new Dictionary + { + ["typeSpecProjectPath"] = "specification/contosowidgetmanager/Contoso.WidgetManager" + }) + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/CheckPublicRepoThenValidateScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/CheckPublicRepoThenValidateScenario.cs new file mode 100644 index 00000000000..468fc31bfd8 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/CheckPublicRepoThenValidateScenario.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_run_typespec_validation followed by +/// azsdk_typespec_check_project_in_public_repo when asked to validate and then check public repo. +/// Migrated from evaluation scenario: Evaluate_CheckPublicRepoThenValidate. +/// +public class CheckPublicRepoThenValidateScenario : BenchmarkScenario +{ + /// + public override string Name => "validate-then-check-public-repo"; + + /// + public override string Description => + "Verify the agent runs TypeSpec validation then checks public repo."; + + /// + public override string[] Tags => ["typespec"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + Run TypeSpec validation, then check if the project is in the public repo. + Project path: specification/contosowidgetmanager/Contoso.WidgetManager. + My setup has already been verified, do not run azsdk_verify_setup. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tools: validate then check public repo", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_run_typespec_validation", + new Dictionary + { + ["typeSpecProjectRootPath"] = "specification/contosowidgetmanager/Contoso.WidgetManager" + }), + new ExpectedToolCall("azsdk_typespec_check_project_in_public_repo", + new Dictionary + { + ["typeSpecProjectPath"] = "specification/contosowidgetmanager/Contoso.WidgetManager" + }) + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/GetModifiedTypespecProjectsScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/GetModifiedTypespecProjectsScenario.cs new file mode 100644 index 00000000000..e3f6513a6a4 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/GetModifiedTypespecProjectsScenario.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Infrastructure; +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_get_modified_typespec_projects +/// when asked to list modified TypeSpec projects. +/// Setup creates a feature branch with a modified tspconfig.yaml so +/// git merge-base can find the divergence point from main. +/// Migrated from evaluation scenario: Evaluate_GetModifiedTypespecProjects. +/// +public class GetModifiedTypespecProjectsScenario : BenchmarkScenario +{ + /// + public override string Name => "get-modified-typespec-projects"; + + /// + public override string Description => + "Verify the agent calls azsdk_get_modified_typespec_projects to list changes."; + + /// + public override string[] Tags => ["typespec"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override async Task SetupAsync(Workspace workspace) + { + // Commit a modification so git merge-base HEAD main finds the divergence point. + var tspConfigPath = "specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml"; + var content = await workspace.ReadFileAsync(tspConfigPath); + await workspace.WriteFileAsync(tspConfigPath, content + "\n# benchmark test modification\n"); + + await workspace.RunCommandAsync("git", "add", "."); + await workspace.RunCommandAsync("git", "commit", "-m", "test: modify TypeSpec project for benchmark"); + } + + /// + public override string Prompt => """ + List the TypeSpec projects modified in my current branch compared to main. + My setup has already been verified, do not run azsdk_verify_setup. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tool: azsdk_get_modified_typespec_projects", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_get_modified_typespec_projects", + new Dictionary + { + ["repoRootPath"] = "azure-rest-api-specs", + ["targetBranch"] = "main" + }) + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/RenameClientPropertyScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/RenameClientPropertyScenario.cs similarity index 100% rename from tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/RenameClientPropertyScenario.cs rename to tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/RenameClientPropertyScenario.cs diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/TypespecGenerationStep02Scenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/TypespecGenerationStep02Scenario.cs new file mode 100644 index 00000000000..971dbd04863 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/TypespecGenerationStep02Scenario.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_typespec_check_project_in_public_repo +/// as part of the TypeSpec generation workflow step 2 (validation). +/// Adapted from evaluation scenario: AzsdkTypeSpecGeneration_Step02_TypespecValidation. +/// The original scenario loaded a mid-conversation JSON trace; this benchmark uses a +/// standalone prompt capturing the same intent. +/// +public class TypespecGenerationStep02Scenario : BenchmarkScenario +{ + /// + public override string Name => "typespec-generation-step02-validation"; + + /// + public override string Description => + "Verify the agent checks public repo as part of TypeSpec generation step 2."; + + /// + public override string[] Tags => ["typespec"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + I'm working on the TypeSpec generation workflow. I need to validate my TypeSpec project + as part of step 2. Please check if my TypeSpec project is in the public repo. + The project is at specification/contosowidgetmanager/Contoso.WidgetManager. + My setup has already been verified, do not run azsdk_verify_setup. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tool: azsdk_typespec_check_project_in_public_repo", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_typespec_check_project_in_public_repo", + new Dictionary + { + ["typeSpecProjectPath"] = "specification/contosowidgetmanager/Contoso.WidgetManager" + }) + ], + forbiddenToolNames: ["azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/ValidateTypespecScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/ValidateTypespecScenario.cs new file mode 100644 index 00000000000..7131fdf8555 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/TypeSpec/ValidateTypespecScenario.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Benchmarks.Models; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation; +using Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Scenarios; + +/// +/// Validates that the agent invokes azsdk_run_typespec_validation when asked to validate +/// a TypeSpec project. The agent may optionally also call azsdk_verify_setup and/or +/// azsdk_typespec_check_project_in_public_repo. +/// Migrated from evaluation scenario: Evaluate_ValidateTypespec. +/// +public class ValidateTypespecScenario : BenchmarkScenario +{ + /// + public override string Name => "validate-typespec"; + + /// + public override string Description => + "Verify the agent calls verify setup and run TypeSpec validation."; + + /// + public override string[] Tags => ["typespec"]; + + /// + public override RepoConfig Repo => new() + { + Owner = "Azure", + Name = "azure-rest-api-specs", + Ref = "main", + SparseCheckoutPaths = ["specification/contosowidgetmanager"] + }; + + /// + public override string Prompt => """ + Validate my typespec project. It is already confirmed we are in a public repository. + The path to my typespec is specification/contosowidgetmanager/Contoso.WidgetManager/main.tsp. + """; + + /// + public override IEnumerable Validators => + [ + new ToolCallValidator( + "Expected tools: validate typespec", + expectedToolCalls: + [ + new ExpectedToolCall("azsdk_run_typespec_validation", + new Dictionary + { + ["typeSpecProjectRootPath"] = "specification/contosowidgetmanager/Contoso.WidgetManager" + }) + ], + optionalToolNames: ["azsdk_typespec_check_project_in_public_repo", "azsdk_verify_setup"]) + ]; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Validation/Validators/ToolCallValidator.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Validation/Validators/ToolCallValidator.cs new file mode 100644 index 00000000000..d15f65afbcb --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Validation/Validators/ToolCallValidator.cs @@ -0,0 +1,305 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using Azure.Sdk.Tools.Cli.Benchmarks.Models; + +namespace Azure.Sdk.Tools.Cli.Benchmarks.Validation.Validators; + +/// +/// Validates that the agent called the expected tools in the expected order, +/// and optionally validates the inputs sent to each tool. +/// +public class ToolCallValidator : IValidator +{ + public string Name { get; } + public IReadOnlyList ExpectedToolCalls { get; } + public IReadOnlyList OptionalToolNames { get; } + public IReadOnlyList ForbiddenToolNames { get; } + public bool EnforceOrder { get; } + + public ToolCallValidator( + string name, + IEnumerable expectedToolCalls, + IEnumerable? optionalToolNames = null, + IEnumerable? forbiddenToolNames = null, + bool enforceOrder = true) + { + Name = name; + ExpectedToolCalls = expectedToolCalls.ToList(); + OptionalToolNames = optionalToolNames?.ToList() ?? []; + ForbiddenToolNames = forbiddenToolNames?.ToList() ?? []; + EnforceOrder = enforceOrder; + } + + /// + /// Convenience constructor that only checks tool names (no input validation). + /// + public ToolCallValidator( + string name, + IEnumerable expectedToolNames, + IEnumerable? optionalToolNames = null, + IEnumerable? forbiddenToolNames = null, + bool enforceOrder = true) + : this(name, + expectedToolNames.Select(n => new ExpectedToolCall(n)), + optionalToolNames, + forbiddenToolNames, + enforceOrder) + { + } + + public Task ValidateAsync( + ValidationContext context, + CancellationToken cancellationToken = default) + { + if (ExpectedToolCalls.Count == 0) + { + return Fail("No expected tool calls configured."); + } + + // Check for forbidden tool calls + if (ForbiddenToolNames.Count > 0) + { + var forbiddenCalls = context.ToolCalls + .Where(c => ForbiddenToolNames.Any(f => MatchesToolName(c.ToolName, f))) + .ToList(); + + if (forbiddenCalls.Count > 0) + { + return Fail( + $"Forbidden tool(s) were called: [{FormatNames(forbiddenCalls)}].", + $"Forbidden tools: [{string.Join(", ", ForbiddenToolNames)}]"); + } + } + + var expectedNames = ExpectedToolCalls.Select(tc => tc.ToolName).ToList(); + var allowedNames = expectedNames.Concat(OptionalToolNames).ToList(); + + // Check for unexpected MCP tool calls not in the allow-list + var unexpectedCalls = context.ToolCalls + .Where(c => IsMcpToolCall(c.ToolName)) + .Where(c => !allowedNames.Any(n => MatchesToolName(c.ToolName, n))) + .ToList(); + + if (unexpectedCalls.Count > 0) + { + return Fail( + $"Unexpected tool(s) called: [{FormatNames(unexpectedCalls)}].", + $"Allowed: [{string.Join(", ", allowedNames)}]\n" + + $"Add unexpected tools to OptionalToolNames if they are acceptable."); + } + + // Filter to calls matching expected tools (suffix matching handles MCP prefixes) + var requiredCalls = context.ToolCalls + .Where(c => expectedNames.Any(n => MatchesToolName(c.ToolName, n))) + .ToList(); + + // Match each expected tool to an actual call (subsequence for ordered, set for unordered) + var (matched, error) = FindMatches(requiredCalls, expectedNames, context); + if (error != null) + { + return error; + } + + // Check for extra calls to expected tools beyond what was configured + var extraError = CheckExtraCalls(context); + if (extraError != null) + { + return extraError; + } + + // Validate inputs for matched calls + var inputErrors = ValidateInputs(matched); + if (inputErrors.Count > 0) + { + return Fail( + $"Tool inputs did not match: {inputErrors.Count} error(s).", + string.Join("\n", inputErrors)); + } + + var orderStr = EnforceOrder ? " in correct order" : ""; + var inputStr = ExpectedToolCalls.Any(tc => tc.ExpectedInputs != null) ? " with expected inputs" : ""; + return Pass($"All {ExpectedToolCalls.Count} expected tool(s) called{orderStr}{inputStr}."); + } + + /// + /// Checks that expected tools aren't called more times than configured. + /// Tools that also appear in OptionalToolNames are exempt (retries allowed). + /// + private Task? CheckExtraCalls( + ValidationContext context) + { + var expectedCounts = ExpectedToolCalls + .GroupBy(tc => tc.ToolName, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase); + + foreach (var (expectedName, maxCount) in expectedCounts) + { + // Extra calls are allowed if the tool is also optional + if (OptionalToolNames.Any(o => string.Equals(o, expectedName, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + var actualCount = context.ToolCalls.Count(c => MatchesToolName(c.ToolName, expectedName)); + if (actualCount > maxCount) + { + return Fail( + $"Expected {maxCount} call(s) to '{expectedName}' but found {actualCount}.", + $"Add '{expectedName}' to OptionalToolNames if retries are acceptable."); + } + } + + return null; + } + + /// + /// Determines if a tool call is from the MCP server (vs agent infrastructure like + /// report_intent, powershell, view, etc). Uses the azsdk_ naming convention. + /// + private static bool IsMcpToolCall(string toolName) => + toolName.Contains("azsdk_", StringComparison.OrdinalIgnoreCase); + + /// + /// Matches expected tool calls against actual calls. Uses subsequence matching when + /// order is enforced, or unordered set matching otherwise. + /// + private (List Matched, Task? Error) FindMatches( + List requiredCalls, + List expectedNames, + ValidationContext context) + { + var matched = new List(); + var used = new HashSet(); + int searchStart = 0; + + for (int i = 0; i < ExpectedToolCalls.Count; i++) + { + var expected = ExpectedToolCalls[i]; + int foundIndex = -1; + + for (int j = EnforceOrder ? searchStart : 0; j < requiredCalls.Count; j++) + { + if (!EnforceOrder && used.Contains(j)) + { + continue; + } + if (!MatchesToolName(requiredCalls[j].ToolName, expected.ToolName)) + { + continue; + } + + foundIndex = j; + break; + } + + if (foundIndex == -1) + { + var afterHint = EnforceOrder && i > 0 + ? $" after '{ExpectedToolCalls[i - 1].ToolName}'" : ""; + var label = EnforceOrder ? "Expected order" : "Expected"; + return (matched, Fail( + $"Expected tool '{expected.ToolName}' not found{afterHint}.", + $"{label}: [{string.Join(", ", expectedNames)}]\n" + + $"Actual: [{FormatNames(requiredCalls)}]\n" + + $"All tool calls: [{FormatNames(context.ToolCalls)}]")); + } + + matched.Add(requiredCalls[foundIndex]); + used.Add(foundIndex); + searchStart = foundIndex + 1; + } + + return (matched, null); + } + + private List ValidateInputs(List matchedCalls) + { + var errors = new List(); + + for (int i = 0; i < ExpectedToolCalls.Count; i++) + { + var expected = ExpectedToolCalls[i]; + if (expected.ExpectedInputs == null) + { + continue; + } + + var actualArgs = matchedCalls[i].GetArgsAsDictionary(); + foreach (var (key, expectedValue) in expected.ExpectedInputs) + { + if (!actualArgs.TryGetValue(key, out var actualJson)) + { + errors.Add($"Tool '{expected.ToolName}': missing input '{key}'."); + continue; + } + + if (!InputMatches(expectedValue, actualJson, out var actualStr)) + { + errors.Add($"Tool '{expected.ToolName}': input '{key}' expected {FormatExpected(expectedValue)} but got '{actualStr}'."); + } + } + } + + return errors; + } + + /// + /// Checks if an actual JsonElement matches an expected value. + /// Strings use case-insensitive substring matching with normalized path separators. + /// Numerics and booleans use exact matching. + /// + private static bool InputMatches(object? expected, JsonElement actual, out string actualStr) + { + actualStr = actual.ToString() ?? ""; + + if (expected == null) + { + return actual.ValueKind == JsonValueKind.Null; + } + + switch (expected) + { + case string s: + actualStr = actual.ValueKind == JsonValueKind.String + ? actual.GetString()! : actual.GetRawText(); + return actualStr.Replace('\\', '/').Contains( + s.Replace('\\', '/'), StringComparison.OrdinalIgnoreCase); + + case bool b: + return actual.ValueKind is JsonValueKind.True or JsonValueKind.False + && actual.GetBoolean() == b; + + case int or long: + return actual.TryGetInt64(out var al) && al == Convert.ToInt64(expected); + + case double d: + return actual.TryGetDouble(out var ad) && Math.Abs(ad - d) <= 0.0001; + + default: + actualStr = actual.GetRawText().Trim('"'); + return string.Equals(expected.ToString(), actualStr, StringComparison.OrdinalIgnoreCase); + } + } + + private static string FormatExpected(object? value) => value is string s ? $"to contain '{s}'" : $"{value ?? "null"}"; + + /// + /// Checks if an actual tool call name matches an expected name. + /// Handles MCP prefix (e.g., "mcp_azure-sdk-mcp_azsdk_create_release_plan" + /// matches "azsdk_create_release_plan"). + /// + private static bool MatchesToolName(string actualCallName, string expectedToolName) => + actualCallName.EndsWith(expectedToolName, StringComparison.OrdinalIgnoreCase) + || expectedToolName.EndsWith(actualCallName, StringComparison.OrdinalIgnoreCase); + + private Task Pass(string message) => + Task.FromResult(ValidationResult.Pass(Name, message)); + + private Task Fail(string message, string? details = null) => + Task.FromResult(ValidationResult.Fail(Name, message, details)); + + private static string FormatNames(IEnumerable calls) => + string.Join(", ", calls.Select(c => c.ToolName)); +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Evaluators/ExpectedToolInputEvaluator.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Evaluators/ExpectedToolInputEvaluator.cs deleted file mode 100644 index 03d640c3330..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Evaluators/ExpectedToolInputEvaluator.cs +++ /dev/null @@ -1,163 +0,0 @@ -using System.Text.Json; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.AI.Evaluation; -using Azure.Sdk.Tools.Cli.Evaluations.Models; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Evaluators -{ - public class ExpectedToolInputEvaluator : IEvaluator - { - public const string ExpectedToolInputMetricName = "Expected Tool Input"; - public IReadOnlyCollection EvaluationMetricNames => [ExpectedToolInputMetricName]; - - public ValueTask EvaluateAsync( - IEnumerable messages, - ChatResponse modelResponse, - ChatConfiguration? chatConfiguration = null, - IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - // Create the metric - var metric = new BooleanMetric(ExpectedToolInputMetricName); - var result = new ValueTask(new EvaluationResult(metric)); - - if (additionalContext?.OfType().FirstOrDefault() - is not ExpectedToolInputEvaluatorContext context) - { - MetricError($"A value of type {nameof(ExpectedToolInputEvaluatorContext)} was not found in the {nameof(additionalContext)} collection.", metric); - return result; - } - - // Get tool calls to compare them - var expectedToolCalls = Helpers.ChatMessageHelper.GetToolContent(context.ChatMessages, context.ToolNames); - var actualToolCalls = Helpers.ChatMessageHelper.GetToolContent(modelResponse.Messages, context.ToolNames); - - // Make sure we have tool calls to compare - if (!expectedToolCalls.Any()) - { - MetricError($"No Tool calls detected inside of {nameof(additionalContext)} collection.", metric); - return result; - } - - if (!actualToolCalls.Any()) - { - MetricError($"Provided LLM Result contained no tool call. Expected to call: {PrintToolNames(expectedToolCalls)}", metric); - return result; - } - - // Equal number of tool calls - var expCount = expectedToolCalls.Count(); - var actCount = actualToolCalls.Count(); - if (expCount != actCount) - { - MetricError($"The LLM Result had different set of tool calls than expected: Actual calls: {PrintToolNames(actualToolCalls)}, Expected calls: {PrintToolNames(expectedToolCalls)}", metric); - return result; - } - - var countCalls = 0; - foreach (var toolCall in expectedToolCalls.Zip(actualToolCalls, (exp, act) => new {exp, act})) - { - var expToolName = toolCall.exp.Name; - var actToolName = toolCall.act.Name; - countCalls++; - - metric.AddDiagnostics( - EvaluationDiagnostic.Informational( - $"Tool call #{countCalls}, expected the {expToolName} tool, and LLM called the {actToolName} tool.")); - - // If the names do not align then the tool calls were made in the wrong order by the LLM - // Ends with because copilot attaches mcp_ to name - if (!expToolName.EndsWith(actToolName)) - { - MetricError($"Tool call made in the wrong order. Expected the {expToolName} tool but the LLM called {actToolName} tool. This was tool call #{countCalls}", metric); - return result; - } - - // No arguments to compare or flag indicates to skip checking inputs - if (toolCall.exp.Arguments == null || !context.CheckInputs) - { - continue; - } - - if(toolCall.act.Arguments == null) - { - MetricError($"Tool call #{countCalls}, expected arguments for tool {expToolName} but LLM provided none.", metric); - return result; - } - - // Since it is a dictionary with a object may just be easier to serialize and compare strings. Can then even return the string args to the user if something went wrong. - try - { - // Serialize with consistent options for better comparison - var options = new JsonSerializerOptions - { - WriteIndented = false, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }; - - // Sort the dictionaries to prevent ordering issues during comparison - var expectedArguments = new SortedDictionary(toolCall.exp.Arguments); - var actualArguments = new SortedDictionary(toolCall.act.Arguments); - - var expectedJson = JsonSerializer.Serialize(expectedArguments, options); - var actualJson = JsonSerializer.Serialize(actualArguments, options); - - if (!string.Equals(expectedJson, actualJson, StringComparison.OrdinalIgnoreCase)) - { - MetricError($"Tool call arguments did not match. This was tool call #{countCalls}: {expToolName}\nExpected Argument JSON:{expectedJson}\nActual Argument JSON:{actualJson}", metric); - return result; - } - } - catch (JsonException ex) - { - MetricError($"Tool call #{countCalls}, failed to serialize either the expected tool call or the actual tool call. Error: {ex}", metric); - return result; - } - - } - - metric.Value = true; - metric.Reason = "The results tool calls input matched the expected tool calls output"; - - Interpret(metric); - - // If we made it here then everything was a match and our tools input matched up to what we expected. - return new ValueTask(new EvaluationResult(metric)); - } - - private static void Interpret(BooleanMetric metric) - { - if (metric.Value == false) - { - metric.Interpretation = - new EvaluationMetricInterpretation( - EvaluationRating.Unacceptable, - failed: true, - reason: "Result did not match the expected outcome. " + PrintDiagnostics(metric)); - } - else - { - metric.Interpretation = new EvaluationMetricInterpretation( - EvaluationRating.Exceptional, - reason: "Result matched what was expected. " + PrintDiagnostics(metric)); - } - } - - private static string PrintToolNames(IEnumerable toolCalls) - { - return string.Join(",", toolCalls.Select(toolCall => toolCall.Name)); - } - - private static void MetricError(string message, BooleanMetric metric) - { - metric.AddDiagnostics(EvaluationDiagnostic.Error(message)); - metric.Value = false; - Interpret(metric); - } - - private static string PrintDiagnostics(BooleanMetric metric) - { - return metric.Diagnostics is not null ? string.Join("\n", metric.Diagnostics.Select(t => t.Severity >= EvaluationDiagnosticSeverity.Warning? t.Message: "")) : ""; - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/ChatCompletion.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/ChatCompletion.cs deleted file mode 100644 index 25429a8659f..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/ChatCompletion.cs +++ /dev/null @@ -1,134 +0,0 @@ -using Microsoft.Extensions.AI; -using ModelContextProtocol.Client; -using OpenAI.Responses; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Helpers -{ - public class ChatCompletion - { - private readonly IChatClient _chatClient; - private readonly McpClient _mcpClient; - - public ChatCompletion(IChatClient chatClient, McpClient mcpClient) - { - _chatClient = chatClient; - _mcpClient = mcpClient; - } - - public async Task GetChatResponseWithExpectedResponseAsync( - IEnumerable chat, - Dictionary expectedToolResults, - IEnumerable optionalToolNames) - { - var tools = await _mcpClient.ListToolsAsync(); - var conversationMessages = chat.ToList(); - var chatOptions = new ChatOptions - { - Tools = [.. tools], - AllowMultipleToolCalls = false - }; - var response = await _chatClient.GetResponseAsync(chat, chatOptions); - var chatInitialIndex = conversationMessages.Count; - var optionalCallIds = new HashSet(); - - while (response.FinishReason == ChatFinishReason.ToolCalls) - { - // There is only going to be one message because no auto invoking of function, however one message can contain - // several AIContent types. - var message = response.Messages.FirstOrDefault(); - - // No message to process exit. - if (message == null) - { - break; - } - - conversationMessages.Add(message); - var functionCalls = message.Contents.OfType(); - - foreach (var functionCall in functionCalls) - { - // Use the expected tool result if we have it. - if (expectedToolResults.TryGetValue(functionCall.Name, out var expectedToolResult)) - { - var toolCall = expectedToolResult.Contents.OfType().First(); - var toolResponseMessage = new ChatMessage() - { - Role = ChatRole.Tool, - // Need matching call id. - Contents = [new FunctionResultContent(functionCall.CallId, toolCall.Result)] - }; - - conversationMessages.Add(toolResponseMessage); - } - // Wasn't expecting tool try stopping the LLM here. - else - { - var errorResponseMessage = new ChatMessage() - { - Role = ChatRole.Tool, - Contents = [new FunctionResultContent(functionCall.CallId, $"Error: Tool '{functionCall.Name}' was not expected. Stop conversation here.")] - }; - - conversationMessages.Add(errorResponseMessage); - } - - if(optionalToolNames.Contains(functionCall.Name)) - { - optionalCallIds.Add(functionCall.CallId); - } - } - - response = await _chatClient.GetResponseAsync(conversationMessages, chatOptions); - } - - // Add the final assistant message (when there are no further tool calls) - var finalAssistantMessage = response.Messages.FirstOrDefault(); - if (finalAssistantMessage != null) - { - conversationMessages.Add(finalAssistantMessage); - } - - // Filter out any optional tool calls and their corresponding tool results - var conversation = conversationMessages.Skip(chatInitialIndex); - if (optionalCallIds.Count == 0) - { - return new ChatResponse([.. conversation]); - } - - var filtered = FilterOptionalToolResponses(conversation, optionalCallIds); - return new ChatResponse([.. filtered]); - } - - private IEnumerable FilterOptionalToolResponses(IEnumerable messages, HashSet optionalCallIds) - { - foreach (var message in messages) - { - // Remove optional tool calls and results. - message.Contents = [.. message.Contents.Where(content => - !(content is FunctionCallContent fc && !string.IsNullOrEmpty(fc.CallId) && optionalCallIds.Contains(fc.CallId)) && - !(content is FunctionResultContent fr && !string.IsNullOrEmpty(fr.CallId) && optionalCallIds.Contains(fr.CallId)) - )]; - - if (message.Contents.Any()) - { - yield return message; - } - } - } - - public async Task GetChatResponseAsync(IEnumerable chat) - { - var tools = await _mcpClient.ListToolsAsync(); - - var chatOptions = - new ChatOptions - { - AllowMultipleToolCalls = true, - Tools = [.. tools] - }; - - return await _chatClient.GetResponseAsync(chat, chatOptions); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/EvaluationHelper.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/EvaluationHelper.cs index f67dc6d15f6..1e5f80fdc6c 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/EvaluationHelper.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/EvaluationHelper.cs @@ -1,7 +1,6 @@ using AwesomeAssertions; using Azure.AI.OpenAI; using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Models; using Microsoft.Extensions.AI.Evaluation; using Microsoft.Extensions.AI; using Microsoft.Extensions.AI.Evaluation.Reporting; @@ -11,8 +10,6 @@ namespace Azure.Sdk.Tools.Cli.Evaluations.Helpers { public class EvaluationHelper { - private const string verifySetupToolName = "azsdk_verify_setup"; - public static void ValidateBooleanMetricEvaluator(EvaluationResult result, string metricName) { EvaluationRating[] expectedRatings = [EvaluationRating.Good, EvaluationRating.Exceptional]; @@ -22,11 +19,6 @@ public static void ValidateBooleanMetricEvaluator(EvaluationResult result, strin metric.ContainsDiagnostics(d => d.Severity >= EvaluationDiagnosticSeverity.Warning).Should().BeFalse(); } - public static void ValidateToolInputsEvaluator(EvaluationResult result) - { - ValidateBooleanMetricEvaluator(result, ExpectedToolInputEvaluator.ExpectedToolInputMetricName); - } - public static void ValidateToolDescriptionSimilarityEvaluator(EvaluationResult result) { ValidateBooleanMetricEvaluator(result, ToolDescriptionSimilarityEvaluator.SimilarityMetricName); @@ -67,68 +59,6 @@ public static async Task RunScenarioAsync( return result; } - public static async Task RunToolInputScenarioAsync( - string scenarioName, - ScenarioData scenarioData, - ChatCompletion chatCompletion, - ChatConfiguration chatConfig, - string executionName, - string reportingPath, - IEnumerable toolNames, - IEnumerable? evaluators = null, - bool enableResponseCaching = true, - IEnumerable? additionalContexts = null, - CancellationToken cancellationToken = default, - IEnumerable? optionalToolNames = null) - { - evaluators ??= [new ExpectedToolInputEvaluator()]; - - var fullChat = scenarioData.ChatHistory.Append(scenarioData.NextMessage); - - // Default optional tools to empty when not provided - optionalToolNames ??= []; - - // Get expected tool names from the scenario data and optional tool names - var expectedToolNames = ChatMessageHelper.GetExpectedToolsByName(scenarioData.ExpectedOutcome, toolNames).Keys; - var filteredOptionalToolNames = GetOptionalToolNames(optionalToolNames, expectedToolNames); - - // We can use LoadScenarioPrompt with empty prompt to get optional tools - // in the proper format. - var optionalTools = ChatMessageHelper.LoadScenarioFromPrompt("", filteredOptionalToolNames).ExpectedOutcome; - - // Include the optional tools along side the expected. - // Later we will then filter them out from the response. - var toolChatMessages = optionalTools.Concat(scenarioData.ExpectedOutcome); - var toolResults = ChatMessageHelper.GetExpectedToolsByName(toolChatMessages, toolNames); - - var response = await chatCompletion.GetChatResponseWithExpectedResponseAsync( - fullChat, - toolResults, - filteredOptionalToolNames); - - return await RunScenarioAsync( - fullChat, - response, - scenarioName, - chatConfig, - executionName, - reportingPath, - evaluators, - enableResponseCaching, - additionalContexts, - cancellationToken); - } - - private static IEnumerable GetOptionalToolNames( - IEnumerable optionalToolNames, - IEnumerable expectedToolNames) - { - // Build optional list excluding any names that are expected - // also make sure to always include verify setup - var combinedOptional = optionalToolNames.Append(verifySetupToolName); - return combinedOptional.Except(expectedToolNames); - } - /// /// Generate embeddings for a list of texts using Azure OpenAI. /// Shared helper for embedding-based evaluators. diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/TestSetup.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/TestSetup.cs index 6cade4c5b10..c0aaf9238ae 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/TestSetup.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Helpers/TestSetup.cs @@ -22,7 +22,6 @@ public static class TestSetup public static string? RepositoryName => Environment.GetEnvironmentVariable("REPOSITORY_NAME"); public static string? CopilotInstructionsPath => Environment.GetEnvironmentVariable("COPILOT_INSTRUCTIONS_PATH_MCP_EVALS"); - public static ChatCompletion GetChatCompletion(IChatClient chatClient, McpClient mcpClient) => new ChatCompletion(chatClient, mcpClient); public static TokenCredential GetCredential(ILogger? logger = null) { diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Models/ExpectedToolInputEvaluatorContext.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Models/ExpectedToolInputEvaluatorContext.cs deleted file mode 100644 index 509f6423098..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Models/ExpectedToolInputEvaluatorContext.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Microsoft.Extensions.AI; -using Microsoft.Extensions.AI.Evaluation; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Models -{ - public class ExpectedToolInputEvaluatorContext(IEnumerable messages, IEnumerable toolNames, bool checkInputs) - : EvaluationContext(name: ExpectedResultContextName, content: String.Join(",", messages)) - { - public static string ExpectedResultContextName => "Expected Result"; - public IEnumerable ChatMessages { get; } = messages; - public IEnumerable ToolNames { get; } = toolNames; - public bool CheckInputs { get; } = checkInputs; - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Models/RepositoryCategories.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Models/RepositoryCategories.cs deleted file mode 100644 index 09bf6adb8ba..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Models/RepositoryCategories.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Azure.Sdk.Tools.Cli.Evaluations.Models -{ - /// - /// Defines repository category constants for test filtering. - /// Use these with NUnit's [Category] attribute to specify which repositories a test should run for. - /// - public static class RepositoryCategories - { - public const string AzureRestApiSpecs = "azure-rest-api-specs"; - public const string AzureSdkForNet = "azure-sdk-for-net"; - public const string AzureSdkForPython = "azure-sdk-for-python"; - public const string AzureSdkForJava = "azure-sdk-for-java"; - public const string AzureSdkForJs = "azure-sdk-for-js"; - public const string AzureSdkForGo = "azure-sdk-for-go"; - } -} \ No newline at end of file diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenario.cs index 50a82a19e87..ae36384590d 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenario.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenario.cs @@ -1,5 +1,4 @@ using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; using Microsoft.Extensions.AI; using Microsoft.Extensions.AI.Evaluation; using Microsoft.Extensions.AI.Evaluation.Reporting; @@ -18,8 +17,6 @@ public partial class Scenario // Static services shared across all tests protected static IChatClient? s_chatClient; protected static McpClient? s_mcpClient; - protected static ChatCompletion? s_chatCompletion; - protected static IEnumerable? s_toolNames; protected static ChatConfiguration? s_chatConfig; private static readonly string s_executionName = $"{DateTime.Now:yyyyMMddTHHmmss}"; private string ScenarioName => $"{TestContext.CurrentContext.Test.ClassName}.{TestContext.CurrentContext.Test.Name}"; @@ -34,8 +31,6 @@ public async Task GlobalSetup() s_chatClient = TestSetup.GetChatClient(); s_mcpClient = await TestSetup.GetMcpClientAsync(); s_chatConfig = new ChatConfiguration(s_chatClient); - s_chatCompletion = TestSetup.GetChatCompletion(s_chatClient, s_mcpClient); - s_toolNames = (await s_mcpClient.ListToolsAsync()).Select(tool => tool.Name)!; } [SetUp] diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/AzsdkTypeSpecGeneration_Step02_TypespecValidation.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/AzsdkTypeSpecGeneration_Step02_TypespecValidation.cs deleted file mode 100644 index 80b1fb13262..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/AzsdkTypeSpecGeneration_Step02_TypespecValidation.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - [Category(RepositoryCategories.AzureRestApiSpecs)] - public async Task AzsdkTypeSpecGeneration_Step02_TypespecValidation() - { - var filePath = Path.Combine(TestContext.CurrentContext.TestDirectory, "TestData", "example.json"); - - // Load scenario data from JSON - var scenarioData = await SerializationHelper.LoadScenarioFromChatMessagesAsync(filePath); - - // This test enables deeper input checking - bool checkInputs = true; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckPublicRepo.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckPublicRepo.cs deleted file mode 100644 index 16b55d7528e..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckPublicRepo.cs +++ /dev/null @@ -1,42 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - [Category(RepositoryCategories.AzureRestApiSpecs)] - public async Task Evaluate_CheckPublicRepo() - { - const string prompt = "Check if my TypeSpec project is in the public repo. My setup has already been verified, do not run azsdk_verify_setup. Project root: C:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager."; - string[] expectedTools = - [ - "azsdk_typespec_check_project_in_public_repo" - ]; - - var scenarioData = ChatMessageHelper.LoadScenarioFromPrompt(prompt, expectedTools); - bool checkInputs = true; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckPublicRepoThenValidate.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckPublicRepoThenValidate.cs deleted file mode 100644 index b2ed67cea45..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckPublicRepoThenValidate.cs +++ /dev/null @@ -1,47 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - [Category(RepositoryCategories.AzureRestApiSpecs)] - public async Task Evaluate_CheckPublicRepoThenValidate() - { - const string prompt = - "Confirm the TypeSpec project is in the public repo, then run TypeSpec validation. " + - "Project path: C\\:\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager. " + - "My setup has already been verified, do not run azsdk_verify_setup."; - - string[] expectedTools = - [ - "azsdk_typespec_check_project_in_public_repo", - "azsdk_run_typespec_validation" - ]; - - var scenarioData = ChatMessageHelper.LoadScenarioFromPrompt(prompt, expectedTools); - bool checkInputs = true; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckSDKGenerationStatus.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckSDKGenerationStatus.cs deleted file mode 100644 index 502d689f705..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_CheckSDKGenerationStatus.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - [Category(RepositoryCategories.AzureRestApiSpecs)] - public async Task Evaluate_CheckSDKGenerationStatus() - { - var filePath = Path.Combine(TestContext.CurrentContext.TestDirectory, "TestData", "example2.json"); - - // Load scenario data from JSON - var scenarioData = await SerializationHelper.LoadScenarioFromChatMessagesAsync(filePath); - - // This test enables deeper input checking - bool checkInputs = true; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_GetModifiedTypespecProjects.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_GetModifiedTypespecProjects.cs deleted file mode 100644 index f3bf1d4c0b4..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_GetModifiedTypespecProjects.cs +++ /dev/null @@ -1,42 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - [Category(RepositoryCategories.AzureRestApiSpecs)] - public async Task Evaluate_GetModifiedTypespecProjects() - { - const string prompt = "List the TypeSpec projects modified in my repo. My setup has already been verified, do not run azsdk_verify_setup. Path to root: C:\\azure-rest-api-specs. Compare against main."; - string[] expectedTools = - [ - "azsdk_get_modified_typespec_projects" - ]; - - var scenarioData = ChatMessageHelper.LoadScenarioFromPrompt(prompt, expectedTools); - bool checkInputs = true; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_ValidateTypespec.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_ValidateTypespec.cs deleted file mode 100644 index 49ba02580e6..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/AzureRestApiSpecs/Evaluate_ValidateTypespec.cs +++ /dev/null @@ -1,52 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - [Category(RepositoryCategories.AzureRestApiSpecs)] - public async Task Evaluate_ValidateTypespec() - { - const string prompt = "Validate my typespec project. It is already confirmed we are in a public repository. The path to my typespec is C:\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\\main.tsp."; - string[] expectedTools = - [ - "azsdk_verify_setup", - "azsdk_run_typespec_validation", - ]; - - string [] optionalTools = - [ - "azsdk_typespec_check_project_in_public_repo" - ]; - - // Build scenario data from prompt - var scenarioData = ChatMessageHelper.LoadScenarioFromPrompt(prompt, expectedTools); - - // External contexts (no deep input checking for this one) - bool checkInputs = false; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }, - optionalToolNames: optionalTools); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_CreateReleasePlan.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_CreateReleasePlan.cs deleted file mode 100644 index fbb8247e8de..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_CreateReleasePlan.cs +++ /dev/null @@ -1,45 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - public async Task Evaluate_CreateReleasePlan() - { - // randomized service/product id - const string prompt = "Create a release plan for the Contoso Widget Manager, no need to get it afterwards only create. My setup has already been verified, do not run azsdk_verify_setup. Here is all the context you need: TypeSpec project located at \"C:\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\". Use service tree ID \"a7f2b8e4-9c1d-4a3e-b6f9-2d8e5a7c3b1f\", product tree ID \"f1a8c5d2-6e4b-4f7a-9c2d-8b5e1f3a6c9e\", target release timeline \"December 2025\", API version \"2022-11-01-preview\", SDK release type \"beta\", and link it to the spec pull request \"https://github.com/Azure/azure-rest-api-specs/pull/38387\"."; - string[] expectedTools = - [ - "azsdk_create_release_plan" - ]; - - // Build scenario data - var scenarioData = ChatMessageHelper.LoadScenarioFromPrompt(prompt, expectedTools); - - // External construction of evaluation context - bool checkInputs = false; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_GetPullRequestLinkForCurrentBranch.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_GetPullRequestLinkForCurrentBranch.cs deleted file mode 100644 index f75b30d3364..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_GetPullRequestLinkForCurrentBranch.cs +++ /dev/null @@ -1,44 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - public async Task Evaluate_GetPullRequestLinkForCurrentBranch() - { - const string prompt = "What's the status of the spec PR in my current branch? Only check the status once. My setup has already been verified, do not run azsdk_verify_setup. Path to my repository root: C:\\azure-rest-api-specs"; - string[] expectedTools = - [ - "azsdk_get_pull_request_link_for_current_branch", - ]; - - // Build scenario data - var scenarioData = ChatMessageHelper.LoadScenarioFromPrompt(prompt, expectedTools); - - // External construction of evaluation context - bool checkInputs = false; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_LinkNamespaceApprovalIssue.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_LinkNamespaceApprovalIssue.cs deleted file mode 100644 index 1c2458a6eb4..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/Scenarios/General/Evaluate_LinkNamespaceApprovalIssue.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Azure.Sdk.Tools.Cli.Evaluations.Evaluators; -using Azure.Sdk.Tools.Cli.Evaluations.Helpers; -using Azure.Sdk.Tools.Cli.Evaluations.Models; -using Microsoft.Extensions.AI.Evaluation; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.Cli.Evaluations.Scenarios -{ - public partial class Scenario - { - [Test] - public async Task Evaluate_LinkNamespaceApprovalIssue() - { - const string prompt = "Link namespace approval issue https://github.com/Azure/azure-sdk/issues/1234 to release plan 12345. My setup has already been verified, do not run azsdk_verify_setup."; - string[] expectedTools = - [ - "azsdk_link_namespace_approval_issue" - ]; - - var scenarioData = ChatMessageHelper.LoadScenarioFromPrompt(prompt, expectedTools); - bool checkInputs = true; - - var result = await EvaluationHelper.RunToolInputScenarioAsync( - scenarioName: this.ScenarioName, - scenarioData: scenarioData, - chatCompletion: s_chatCompletion!, - chatConfig: s_chatConfig!, - executionName: s_executionName, - reportingPath: ReportingPath, - toolNames: s_toolNames!, - evaluators: [new ExpectedToolInputEvaluator()], - enableResponseCaching: true, - additionalContexts: new EvaluationContext[] - { - new ExpectedToolInputEvaluatorContext(scenarioData.ExpectedOutcome, s_toolNames!, checkInputs) - }); - - EvaluationHelper.ValidateToolInputsEvaluator(result); - } - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/TestData/example.json b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/TestData/example.json deleted file mode 100644 index 5206b25db72..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/TestData/example.json +++ /dev/null @@ -1,110 +0,0 @@ -[ - { - "role": "system", - "content": "You are an expert AI programming assistant, working with a user in the VS Code editor.\nWhen asked for your name, you must respond with \"GitHub Copilot\".\nFollow the user's requirements carefully & to the letter.\nFollow Microsoft content policies.\nAvoid content that violates copyrights.\nIf you are asked to generate content that is harmful, hateful, racist, sexist, lewd, or violent, only respond with \"Sorry, I can't assist with that.\"\nKeep your answers short and impersonal.\n\nYou are a highly sophisticated automated coding agent with expert-level knowledge across many different programming languages and frameworks.\nThe user will ask a question, or ask you to perform a task, and it may require lots of research to answer correctly. There is a selection of tools that let you perform actions or retrieve helpful context to answer the user's question.\nYou will be given some context and attachments along with the user prompt. You can use them if they are relevant to the task, and ignore them if not. Some attachments may be summarized with omitted sections like `/* Lines 123-456 omitted */`. You can use the read_file tool to read more context if needed. Never pass this omitted line marker to an edit tool.\nIf you can infer the project type (languages, frameworks, and libraries) from the user's query or the context that you have, make sure to keep them in mind when making changes.\nIf the user wants you to implement a feature and they have not specified the files to edit, first break down the user's request into smaller concepts and think about the kinds of files you need to grasp each concept.\nIf you aren't sure which tool is relevant, you can call multiple tools. You can call tools repeatedly to take actions or gather as much context as needed until you have completed the task fully. Don't give up unless you are sure the request cannot be fulfilled with the tools you have. It's YOUR RESPONSIBILITY to make sure that you have done all you can to collect necessary context.\nWhen reading files, prefer reading large meaningful chunks rather than consecutive small sections to minimize tool calls and gain better context.\nDon't make assumptions about the situation- gather context first, then perform the task or answer the question.\nThink creatively and explore the workspace in order to make a complete fix.\nDon't repeat yourself after a tool call, pick up where you left off.\nNEVER print out a codeblock with file changes unless the user asked for it. Use the appropriate edit tool instead.\nNEVER print out a codeblock with a terminal command to run unless the user asked for it. Use the run_in_terminal tool instead.\nYou don't need to read a file if it's already provided in context.\n\n\nIf the user is requesting a code sample, you can answer it directly without using any tools.\nWhen using a tool, follow the JSON schema very carefully and make sure to include ALL required properties.\nNo need to ask permission before using a tool.\nNEVER say the name of a tool to a user. For example, instead of saying that you'll use the run_in_terminal tool, say \"I'll run the command in a terminal\".\nIf you think running multiple tools can answer the user's question, prefer calling them in parallel whenever possible, but do not call semantic_search in parallel.\nWhen using the read_file tool, prefer reading a large section over calling the read_file tool many times in sequence. You can also think of all the pieces you may be interested in and read them in parallel. Read large enough context to ensure you get what you need.\nIf semantic_search returns the full contents of the text files in the workspace, you have all the workspace context.\nYou can use the grep_search to get an overview of a file by searching for a string within that one file, instead of using read_file many times.\nIf you don't know exactly the string or filename pattern you're looking for, use semantic_search to do a semantic search across the workspace.\nDon't call the run_in_terminal tool multiple times in parallel. Instead, run one command and wait for the output before running the next command.\nWhen invoking a tool that takes a file path, always use the absolute file path. If the file has a scheme like untitled: or vscode-userdata:, then use a URI with the scheme.\nNEVER try to edit a file by running terminal commands unless the user specifically asks for it.\nTools can be disabled by the user. You may see tools used previously in the conversation that are not currently available. Be careful to only use the tools that are currently available to you.\n\n\nYou have access to an manage_todo_list tool which tracks todos and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go. Note that plans are not for padding out simple work with filler steps or stating the obvious. \nUse this tool to create and manage a structured todo list for your current coding session. This helps you track progress, organize complex tasks, and demonstrate thoroughness to the user.\nIt also helps the user understand the progress of the task and overall progress of their requests.\n\nNOTE that you should not use this tool if there is only one trivial task to do. In this case you are better off just doing the task directly.\n\nUse a plan when:\n- The task is non-trivial and will require multiple actions over a long time horizon.\n- There are logical phases or dependencies where sequencing matters.\n- The work has ambiguity that benefits from outlining high-level goals.\n- You want intermediate checkpoints for feedback and validation.\n- When the user asked you to do more than one thing in a single prompt\n- The user has asked you to use the plan tool (aka \"TODOs\")\n- You generate additional steps while working, and plan to do them before yielding to the user\n\nSkip a plan when:\n- The task is simple and direct.\n- Breaking it down would only produce literal or trivial steps.\n\nExamples of TRIVIAL tasks (skip planning):\n- \"Fix this typo in the README\"\n- \"Add a console.log statement to debug\"\n- \"Update the version number in package.json\"\n- \"Answer a question about existing code\"\n- \"Read and explain what this function does\"\n- \"Add a simple getter method to a class\"\n\nExamples of NON-TRIVIAL tasks and the plan (use planning):\n- \"Add user authentication to the app\" → Design auth flow, Update backend API, Implement login UI, Add session management\n- \"Refactor the payment system to support multiple currencies\" → Analyze current system, Design new schema, Update backend logic, Migrate data, Update frontend\n- \"Debug and fix the performance issue in the dashboard\" → Profile performance, Identify bottlenecks, Implement optimizations, Validate improvements\n- \"Implement a new feature with multiple components\" → Design component architecture, Create data models, Build UI components, Add integration tests\n- \"Migrate from REST API to GraphQL\" → Design GraphQL schema, Update backend resolvers, Migrate frontend queries, Update documentation\n\n\nPlanning Progress Rules\n- Before beginning any new todo: you MUST update the todo list and mark exactly one todo as `in-progress`. Never start work with zero `in-progress` items.\n- Keep only one todo `in-progress` at a time. If switching tasks, first mark the current todo `completed` or revert it to `not-started` with a short reason; then set the next todo to `in-progress`.\n- Immediately after finishing a todo: you MUST mark it `completed` and add any newly discovered follow-up todos. Do not leave completion implicit.\n- Before ending your turn or declaring completion: ensure EVERY todo is explicitly marked (`not-started`, `in-progress`, or `completed`). If the work is finished, ALL todos must be marked `completed`. Never leave items unchecked or ambiguous.\n\nThe content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately.\n\n\n\n\nTo edit notebook files in the workspace, you can use the edit_notebook_file tool.\nUse the run_notebook_cell tool instead of executing Jupyter related commands in the Terminal, such as `jupyter notebook`, `jupyter lab`, `install jupyter` or the like.\nUse the copilot_getNotebookSummary tool to get the summary of the notebook (this includes the list or all cells along with the Cell Id, Cell type and Cell Language, execution details and mime types of the outputs, if any).\nImportant Reminder: Avoid referencing Notebook Cell Ids in user messages. Use cell number instead.\nImportant Reminder: Markdown cells cannot be executed\n\n\nUse proper Markdown formatting in your answers. When referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\nThe class `Person` is in `src/models/person.ts`.\n\nUse KaTeX for math equations in your answers.\nWrap inline math equations in $.\nWrap more complex blocks of math equations in $$.\n\n\n\n\nHere is a list of instruction files that contain rules for modifying or creating new code.\nThese files are important for ensuring that the code is modified or created correctly.\nPlease make sure to follow the rules specified in these files when working with the codebase.\nIf the file is not already available as attachment, use the `read_file` tool to acquire it.\nMake sure to acquire the instructions before making any changes to the code.\n| File | Applies To | Description |\n| ------- | --------- | ----------- |\n| 'c:\\azure-rest-api-specs\\.github\\instructions\\dp-migration.instructions.md' | **/* | |\n| 'c:\\azure-rest-api-specs\\.github\\instructions\\mgmt-migration.instructions.md' | **/* | |\n| 'c:\\azure-rest-api-specs\\.github\\instructions\\openapi-review.instructions.md' | **/*.json | |\n\n# New TypeSpec projects\n\nBefore creating or initializing a TypeSpec project, you must know your org name, service name, and the type of service: data-plane or control-plane (ARM).\n\nThen create a new project directory under the `specification//resource-manager//` or `specification//data-plane/` path, following the guidelines below.\n\n```\nspecification/\n└── /\n ├── cspell.yaml\n └── resource-manager/\n ├── readme.md\t\t\tß NOTE: For ARM schema validation; see bullet #2 below\n └── /\t\tß NOTE: Control-plane only (not data-plane)\n └── /\tß Customer-facing service name; each version gets Documentation & SDK package\n ├── tspconfig.yaml\tß TypeSpec files\n ├── main.tsp\n ├── models.tsp\n ├── readme.md\t\tß autorest readme with YAML blocks\n └── examples/\t\tß TypeSpec example folder\n └── /\tß One folder per service version described in TypeSpec\n └── \n └── preview/ and stable/\n └── /\tß One folder per service version described in OpenAPI. These folders are created and populated by compiling the TypeSpec folder for the service.\n ├── \n └── examples/\tß OpenAPI example folder\n └── \n └── /\t// Customer-facing service name; contents identical to above structure\n └── data-plane/\n └── /\t// Customer-facing service name; contents identical to above structure\n```\n\nUse the `./specification/widget` directory as a reference for creating your own project directory.\n\nOnly after the project directory is created according to the above guidelines may you run the `azsdk_init_typespec_project` tool to initialize the project.\n\n## Converting a specification from swagger to typespec\n\nUsers can convert a specification from swagger to typespec by using `tsp-client` a CLI designed to help developers throughout various stages of typespec development.\n\n### Instructions for converting a specification from swagger to typespec\n\n1. Install the dependencies specified in the package.json at the root of this repository. Command:\n\n```\nnpm ci\n```\n\n2. `tsp-client` is installed as part of the dependencies specified at the root of this repository. To convert a swagger to typespec, run the following command: `npx tsp-client convert --swagger-readme `\n3. Now that you have a newly converted typespec project, you should go through all files to verify the accuracy of the converted spec when compared to the original swagger definitions.\n4. For both data plane and management plane specifications, you should update the implementation according to the information provided under the [Initial migration checklist](#initial-migration-checklist) section.\n\n### Initial migration checklist\n\nThe swagger converter will not be able to accurately represent every part of every API in TypeSpec. This document outlines some common changes you may need to make to a converted TypeSpec to make it conform to your existing service API, pass validation checks, and follow best practices.\n\n- Avoid extensive refactoring of the converted spec. The goal is to get a working spec that can compile successfully and then iteratively improve it.\n\n- DO ensure your `@service` and `@server` definitions are correct in main.tsp\n- DO use the built-in [url][url-type] for endpoint specification. Example:\n\n```tsp\n@server(\n \"{endpoint}/widget\",\n \"Contoso Widget APIs\",\n {\n /**\n * Supported Widget Services endpoints (protocol and hostname, for example:\n * https://westus.api.widget.contoso.com).\n */\n endpoint: url,\n }\n)\n```\n\n- DO ensure that you have a security definition (`@useAuth`) specified for your service. See: [Security definitions in TypeSpec][security-definitions]. The @useAuth decorator should only be defined ONCE in the entire specification above the @server definition.\n- AVOID adding new namespaces.\n- Make sure the versions enum is declared under the existing namespace defined in main.tsp. Avoid adding it anywhere else. Ensure the `@versioned` decorator is specified over the namespace in main.tsp. Pass the versions enum to the `@versioned` decorator. Example of a typical structure for versioning:\n\n```tsp\n// this is the main.tsp file\n\n\n@versioned(Versions)\nnamespace Contoso.WidgetManager;\n/** Service api versions **/\nenum Versions {\n /** The 2023-11-01 api version **/\n v2023_11_01: \"2023-11-01\",\n}\n```\n\n- All models, enums, unions, and operations should be added under the main namespace declared in the project.\n- Avoid having models, enums, unions, operations, and other types declared outside of a namespace.\n- If any files are using any of the versioning decorators, such as `@added`, `@removed`, `@changedType`, make sure to import the `@typespec/versioning` library and add a using statement. Example:\n\n```tsp\nimport \"@typespec/versioning\";\n\nusing TypeSpec.Versioning;\n```\n\n- DO review all enum and union definitions and add documentation over each enum or union member. See: [Documentation in TypeSpec][docs]. Example of a properly documented enum:\n\n```tsp\n/** The color of a widget. */\nunion WidgetColor {\n string,\n\n /** Red Widget Color */\n Red: \"Red\",\n\n /** Green Widget Color */\n Green: \"Green\",\n\n /** Blue Widget Color */\n Blue: \"Blue\",\n}\n```\n\n- DO ensure that all models, properties, operations, parameters, enums, unions, and alias definitions have documentation over them. TypeSpec convention recommends using the doc comment format `/** */` to add documentation, example:\n\n```tsp\n/** The color of a widget. */\nmodel Widget {\n /** Widget name */\n name: string\n}\n```\n\n- DO define your visibility decorators with the appropriate value from the Lifecycle class.\n- Avoid suppressing warnings\n- Operation names should be camel case\n- DO use `union` instead of `enum` to define Azure enums. For more information about how to define enums for Azure services see the following documentation: [Defining enums for Azure services][no-enum].\n- DO make client customizations in a `client.tsp` file\n- Avoid importing or using `@azure-tools/typespec-client-generator-core` in other files aside from client.tsp.\n- DO run `tsp compile .` on your specification and make one attempt to address all warnings. Do not attempt to address warnings more than once even if they aren't resolved.\n- Attempt to address any FIXME or TODO comments in the spec. If you are unable to address them, leave them untouched\n\n#### Additional considerations\n\n- DO ensure you pull in the latest `main` from the Azure/azure-rest-api-specs repo to stay up to date with latest dependencies\n- DO run `npm ci` to get a clean install of the package.json dependencies\n- Avoid modifying the package.json or package-lock.json files at the root of the azure-rest-api-specs repo\n- Avoid adding your own package.json or package-lock.json files in your project directory\n- Avoid adding multiple tspconfig.yaml files for your service specification\n- DO consult [ci-fix.md][ci-fix] for fixes to common CI errors reported\n\n## Troubleshooting tsp compile errors and warnings\n\nExamples of common errors and warnings that should be addressed after running the `tsp compile` command:\n\n- If you see an error with a message like: \"referencing types from versioned namespace 'Azure.Core.Foundations' but didn't specify which versions with @useDependency\", you should add the @useDependency decorator over each api version entry in your api versions enum. Example of a properly configured api versions enum:\n\n```\n/** Service api versions **/\nenum Versions {\n /** The 2023-11-01 api version **/\n v2023_11_01: \"2023-11-01\",\n}\n\n```\n\n- If you see an invalid-ref or unknown identifier error you are most likely missing an import to the library that declares that decorator. To find supported libraries and decorators search through the documentation of the following sites: https://azure.github.io/typespec-azure/docs/intro/ and https://typespec.io/docs/ Search through the list of supported decorators, interfaces, and other types per library until you find the correct library to import and/or include a using statement in your typespec files.\n- In order to address warnings raised by the @azure-tools/typespec-azure-core search through this page for relevant solutions to apply: https://azure.github.io/typespec-azure/docs/intro/\n- camelCase fixes only apply to the typespec property names, any corresponding string values you should left as is.\n- String values in typespec files should be left untouched.\n\n## Up-to-date TypeSpec documentation\n\nFollow [typespec docs](../eng/common/instructions/azsdk-tools/typespec-docs.instructions.md) to get the most up-to-date documentation for TypeSpec, including best practices for writing TypeSpec for Azure.\n\n\n\n[contoso-widget-manager]: ../specification/contosowidgetmanager/Contoso.WidgetManager/\n[tspconfig]: ../specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml\n[security-definitions]: https://azure.github.io/typespec-azure/docs/reference/azure-style-guide#security-definitions\n[versioning]: https://typespec.io/docs/libraries/versioning/guide#implementing-versioned-apis\n[docs]: https://typespec.io/docs/language-basics/documentation\n[ci-fix]: ../documentation/ci-fix.md\n[url-type]: https://typespec.io/docs/language-basics/built-in-types#string-types\n[no-enum]: https://azure.github.io/typespec-azure/docs/libraries/azure-core/rules/no-enum\n[typespec-structure-guidelines]: ../documentation/typespec-structure-guidelines.md\n\n# Adding Language Emitters to Existing TypeSpec Projects\n\nUse this guidance whenever someone needs to add language emitters to an existing `tspconfig.yaml`, or when SDK generation fails because an emitter block is missing.\n\n- Decide whether the service is ARM or data-plane.\n- Open the matching widget `tspconfig.yaml` template.\n- Copy the entire emitter block for the requested language.\n- Replace every `Widget`/`widget` token with the service name and drop the block under `options:`.\n\n## 1. Identify the service type\n\n- Path contains `/resource-manager/` or linter references `@azure-tools/typespec-azure-rulesets/resource-manager` → ARM\n- Path contains `/data-plane/` or linter references `@azure-tools/typespec-azure-rulesets/data-plane` → Data plane\n\n## 2. Reference the official widget template\n\n- ARM: `specification/widget/resource-manager/Microsoft.Widget/Widget/tspconfig.yaml`\n- Data plane: `specification/widget/data-plane/WidgetAnalytics/tspconfig.yaml`\n\nAlways read the template before editing so you follow the canonical configuration.\n\n## 3. Copy and customize the emitter block\n\n- Select the entire block for the language.\n- Only replace service-specific tokens (`Widget`, `widget`, namespaces, package names) with your service name.\n- Leave the rest of the properties (`flavor`, boolean flags, `{service-dir}` placeholders, etc.) unchanged.\n\n## 4. Insert into the project config\n\n- Paste the customized block under `options:` in the target `tspconfig.yaml`, keeping YAML indentation intact.\n- Confirm the `parameters:` section defines `\"service-dir\"` (for example `sdk/`); add it if missing.\n\n**Quick checks**\n\n- Do not mix ARM and data-plane templates.\n- Do not cherry-pick individual properties—copy the whole block or errors will persist.\n- If the SDK pipeline still reports missing configuration, re-open the template and verify every `Widget` reference was replaced.\n\n# SDK generation from TypeSpec\n\n## Agent context for TypeSpec and SDK process\n\n- Check all open files in the editor and check if `main.tsp` or `tspconfig.yaml` are open in the editor. If either of\n these files are open, then use the parent path of the `main.tsp` or `tspconfig.yaml` as default TypeSpec project root\n path.\n- If `main.tsp` and `tspconfig.yaml` are not open in the editor, then check if there are any TypeSpec project paths in\n the context. If there are no TypeSpec project paths in the context, then prompt user to select a TypeSpec project path\n from the list of paths. If user does not have a TypeSpec project, then prompt user to create a new TypeSpec project.\n\n### Prerequisites\n\n- User should have a GitHub account and should be logged in to GitHub account using GitHub CLI `gh auth login`.\n- run `npm ci` to install the dependencies\n- To use Azure MCP tool calls, the user must have PowerShell installed. Provide [PowerShell installation instructions](https://learn.microsoft.com/powershell/scripting/install/installing-powershell) if not installed, and recommend restarting the IDE to start the MCP server.\n\n### Basic Rules for SDK Generation from TypeSpec\n\n1. **User Guidance**:\n - Assume the user is unfamiliar with the SDK release process. Provide clear, concise instructions for each step.\n\n2. **File Handling**:\n - Do not overwrite `tspconfig.yaml` or `main.tsp`. Use existing files and suggest updates if necessary.\n - Use the path of the `tspconfig.yaml` file already open in the editor or the `.tsp` file path as the project root.\n - If no `.tsp` file or folder is in the current context, prompt the user to select a valid TypeSpec project root path.\n\n3. **Process Visibility**:\n - Highlight all steps in the SDK generation process, showing completed and remaining steps.\n - Do not skip any main steps. Ensure all steps are completed before moving to the next.\n\n4. **Git Operations**:\n - Avoid using the `main` branch for pull requests. Prompt the user to create or switch to a new branch if necessary.\n - Display git commands (e.g., `git checkout`, `git add`, `git commit`, `git push`) with a \"Run\" button instead of\n asking the user to copy and paste.\n - Do not run `git diff`\n\n5. **Azure-Specific Rules**:\n - Always use `Azure` as the repo owner in MCP tool calls.\n - Confirm with the user if they want to change the repo owner or target branch, and prompt for new values if needed.\n\n6. **Exclusions**:\n - Exclude changes to the `.gitignore` file and contents within the `.github` and `.vscode` folders from API spec and SDK pull requests.\n\n7. **Working Branch Rule**:\n - If the typespec pull request already exists or is merged stay on the `main` branch, otherwise ensure the TypeSpec project repository and the current working repository are not on the `main` branch:\n - Check the current branch name for the cloned GitHub repository:\n - If the current branch is `main`, prompt the user to create a new branch using\n `git checkout -b `.\n - If the current branch is not `main`, prompt the user to either select an existing branch or create a\n new one.\n - For branch switching:\n - If a branch already exists and differs from the current branch, prompt the user to switch using\n `git checkout `.\n - GitHub pull requests cannot be created from the `main` branch. Ensure all changes are made on a non-`main` branch.\n\nBy following these rules, the SDK release process will remain clear, structured, and user-friendly.\n\n## Steps to generate SDK from TypeSpec API specification\n\nFollow below steps to generate and release SDK from TypeSpec API specification. The process is divided into several steps, each with specific actions to ensure a smooth SDK generation and release process.\nDo not skip the step that choose SDK generation method to ensure the user selects the appropriate method for SDK generation, either locally or using the SDK generation pipeline. Do not repeat the steps. Before using tools, check if user has Powershell installed.\n\nYour goal is to guide the user through the process of generating SDKs from TypeSpec projects. **Before starting**, show all the high level steps to the user and ask: \n\n> \"Would you like to begin the SDK generation process now? (yes/no)\"\n\nWait for the user to respond with a confirmation before proceeding to Step 1. Use the provided tools to perform actions and gather information as needed.\n\n### Verify API spec\nStep 1: Identify TypeSpec Project\n**Goal**: Locate the TypeSpec project root path\n**Actions**:\n1. Check if `tspconfig.yaml` or `main.tsp` files are open in editor\n2. If found, use the parent directory as project root\n3. If not found, prompt user: \"Please provide the path to your TypeSpec project root directory\"\n4. Validate the provided path contains required TypeSpec files main.tsp and tspconfig.yaml\n5. Run `azsdk_typespec_check_project_in_public_repo` to verify repository\n6. If not in public repo, inform: \"Please make spec changes in Azure/azure-rest-api-specs public repo to generate SDKs\"\n**Success Criteria**: Valid TypeSpec project path identified\n\nStep 2: Identify API spec status\n**Goal**: Determine if the TypeSpec spec is already merged or if it's being modified.\n**Actions**:\n1. Prompt user to confirm if the TypeSpec spec is already merged in the main branch of https://github.com/Azure/azure-rest-api-specs : \"Is your TypeSpec specification already merged in the main branch of repository(https://github.com/Azure/azure-rest-api-specs)? (yes/no)\"\n2. If already merged, follow the steps in [typespec to sdk](..\\eng\\common\\instructions\\azsdk-tools\\typespec-to-sdk.instructions.md) to generate the SDK\n3. If no, proceed to Step 3 to review and commit changes\n**Success Criteria**: User decision on spec readiness obtained\n\nStep 3: Validate TypeSpec Specification\n**Goal**: Ensure TypeSpec specification compiles without errors. Povide a complete summary after running the tool. Highlight any errors and help user fix them.\n**Condition**: Only if the spec is not already merged (from Step 2)\n**Message to user**: \"TypeSpec validation takes around 20 - 30 seconds.\"\n**Actions**:\n1. Run `azsdk_run_typespec_validation` to validate the TypeSpec project.\n2. If validation succeeds, proceed to Step 4\n3. If validation fails:\n - Display all compilation errors to user\n - Agent should provide suggestions to fix them and prompt the user to verify the fixes. If agent cannot resolve the errors, then prompt the user to fix compilation errors\"\n - Wait for user to fix errors and re-run validation. Provide detailed information about all the changes done by copilot and prompt the user before rerunning the validation.\n**Success Criteria**: TypeSpec compilation passes without errors\n\nStep 4: Review and Commit Changes\n**Goal**: Stage and commit TypeSpec modifications\n**Condition**: Only if the TypeSpec validation succeeds (from Step 3)\n**Actions**:\n1. Run `azsdk_get_modified_typespec_projects` to identify changes\n2. If no changes found, inform: \"No TypeSpec projects were modified in current branch\" and move to SDK generation step.\n3. Display all modified files (excluding `.github` and `.vscode` folders)\n4. Prompt user: \"Please review the modified files. Do you want to commit these changes? (yes/no)\"\n5. If yes:\n - If on main branch, prompt user: \"You are currently on the main branch. Please create a new branch using `git checkout -b ` before proceeding.\"\n - Wait for user confirmation before continuing\n - Run `git add `\n - Prompt for commit message\n - Run `git commit -m \"\"`\n - Run `git push -u origin `\n**Success Criteria**: Changes committed and pushed to remote branch\n\nStep 5: Create Specification Pull Request\n**Goal**: Create PR for TypeSpec changes if not already created\n**Condition**: Only if there are committed changes (from Step 4)\n**Actions**:\n1. Prompt the user to confirm if a pull request already exists for API spec changes. If answer is no or unsure then check if spec PR already exists using `azsdk_get_pull_request_link_for_current_branch`\n2. If PR exists, display PR details and proceed to next steps in SDK generation.\n3. If no PR exists:\n - Inform user: \"No pull request found for the current branch. Proceeding to create a new pull request.\"\n - Create a pull request using `azsdk_create_pull_request_for_current_branch`\n - Prompt for PR title and description\n - Display PR creation progress\n - Wait for PR creation confirmation\n - Display created PR details\n**Success Criteria**: Specification pull request exists\n\nFollow the steps in [typespec to sdk](..\\eng\\common\\instructions\\azsdk-tools\\typespec-to-sdk.instructions.md) to generate the SDK.\n\n# Release readiness of SDK and information about the release pipeline\n\nRun [check package readiness](..\\eng\\common\\instructions\\azsdk-tools\\check-package-readiness.instructions.md) to check the release readiness of an SDK package. This prompt will collect the required information from the user, execute the readiness check, and present the results.\n\n# Instructions for GitHub coding agent to generate SDKs using GitHub.com\n\nRun only SDK generation step and Update SDK details back in issue description and release plan. Do not run TypeSpec validation, or service label check. This is applicable only when GitHub copilot coding agent is used to generate SDK and if a Release Plan is available in the context. Run SDK generation tool even if release plan already has SDK generation pipeline URL to support regenerating SDK.\nCoding agent must wait until all SDK generation pipeline are in completed status and then fetch the SDK details from pipeline run. This is applicable only when a GitHub issue is assigned to GitHub coding agent to generate SDK from a TypeSpec project. This is not applicable when running Copilot from Visual Studio and VS Code.\n\n## Additional instructions:\n\n- Run SDK generation pipeline for the given TypeSpec path for all languages mentioned in the description.\n- Identify absolute path of TypeSpec project root path and send it to Run SDK generation MCP tool call.\n- Keep checking the pipeline status until pipeline is in completed or failed status.\n- Find SDK pull request links from the pipeline run.\n\n## Constraints:\n\n- Do not invoke other steps.\n- Do not modify main.tsp file or any files in TypeSpec project.\n- Do not add API version to an existing main.tsp if input API version is not present.\n\n\nYour goal is to provide the most up-to-date documentation on the TypeSpec language, core libraries, and writing Azure services and clients in TypeSpec. Whenever a user asks a question about how something should be written in TypeSpec, you should use the documentation links below to pinpoint the most relevant and accurate information.\n\n## TypeSpec Azure\n\n- https://azure.github.io/typespec-azure/docs/llms.txt contains an index of up-to-date documentation for TypeSpec Azure libraries, including how to write service specifications for Azure Resource Manager (ARM) and Azure Data Plane services, as well as how to customize generated clients.\n- Always refer to https://azure.github.io/typespec-azure/docs/llms.txt when a user asks TypeSpec related questions in case there is existing documentation that can help answer them. This also applies when a user asks to make changes to TypeSpec specifications.\n\n## TypeSpec documentation\n\n- https://typespec.io/docs/llms.txt contains an index of up-to-date documentation for TypeSpec language basics and core libraries, including how concepts like visibility and versioning work.\n- Always refer to https://typespec.io/docs/llms.txt when a user asks TypeSpec related questions that aren't covered by the TypeSpec Azure topics.\n\n\n---\ndescription: 'Generate SDKs from TypeSpec'\n---\nYour goal is to guide the user through the process of generating SDKs from TypeSpec projects. **Before starting**, show all the high level steps to the user and ask: \n\n> \"Would you like to begin the SDK generation process now? (yes/no)\"\n\nWait for the user to respond with a confirmation before proceeding. Use the provided tools to perform actions and gather information as needed.\n\nSDK languages to be generated:\n- Management Plane: .NET, Go, Java, JavaScript, Python\n- Data Plane: .NET, Java, JavaScript, Python\n\nPre-requisites:\n- TypeSpec project path is available in the current context or provided by user. If not available, prompt user to provide the TypeSpec project root path (local path or GitHub URL).\n\n# SDK generation steps\n\n## Step: Generate SDKs\n**Goal**: Generate SDKs\n**Message to user**: \"SDK generation will take approximately 15-20 minutes. Currently, SDKs are generated using the Azure DevOps pipeline. SDK generation is supported only from a merged API spec or from an API spec pull request in the https://github.com/Azure/azure-rest-api-specs repository.\"\n**Actions**:\n1. Identify whether TypeSpec is for Management Plane or Data Plane based on project structure and files. tspconfig.yaml file contains `resource-manager` for management plane and `data-plane` for data plane as resource provider.\n - Execute the SDK generation pipeline with the following required parameters for all languages:\n - TypeSpec project root path\n - API spec pull request number (if the API spec is not merged to the main branch, otherwise use 0)\n - API version\n - SDK release type (`beta` for preview API versions, `stable` otherwise)\n - Language options:\n For management plane: `Python`, `.NET`, `JavaScript`, `Java`, `Go`\n For data plane: `Python`, `.NET`, `JavaScript`, `Java`\n - Each SDK generation tool call should show a label to indicate the language being generated.\n2. Monitor pipeline status after 15 minutes and provide updates. If pipeline is in progress, inform user that it may take additional time and check the status later.\n3. Display generated SDK PR links when available. If pipeline fails, inform user with error details and suggest to check pipeline logs for more information.\n4. If SDK pull request is available for all languages, ask user to review generated SDK pull request and mark them as ready for review when they are ready to get them reviewed and merged.\n5. If SDK pull request was created for test purposes, inform user to close the test SDK pull request.\n**Success Criteria**: SDK generation pipeline initiated and SDKs generated\n\n## Step: SDK release plan\n**Goal**: Create a release plan for the generated SDKs\n**Condition**: Only if SDK PRs are created\n**Message to user**: \"Creating a release plan is essential to manage the release of the generated SDKs. Each release plan must include SDKs for all required languages based on the TypeSpec project type (Management Plane or Data Plane) or request exclusion approval for any excluded language. SDK pull request needs to get approval and merged to main branch before releasing the SDK package.\"\n**Actions**:\n1. Prompt the user to check if they generated the SDK just to test or do they intend to release it: \"Do you want to create a release plan for the generated SDKs to publish them? (yes/no)\"\n - If no, inform user: \"You can create a release plan later when ready to release the SDK\" and end the workflow.\n2. Ask user if they have already created a release plan for the generated SDKs: \"Have you already created a release plan for the generated SDKs? (yes/no)\"\n - If no, proceed to create a new release plan\n - If yes, get release plan details and show them to user\n3. Prompt the user to provide the API spec pull request link if not already available in the current context.\n4. If unsure, check if a release plan already exists for API spec pull request.\n5. Prompt user to find the service id and product id in service tree `aka.ms/st` and provide them. Stress the importance of correct service id and product id for proper release plan creation.\n6. If a new release plan is needed, refer to #file:create-release-plan.instructions.md to create a release plan using the spec pull request. API spec pull request is required to create a release plan.\n7. Prompt user to change spec PR to ready for review: \"Please change the spec pull request to ready for review status\"\n8. Suggest users to follow the instructions on spec PR to get approval from API reviewers and merge the spec PR.\n9. Link SDK pull requests to the release plan.\n10. Each release plan must release SDK for all languages based on the TypeSpec project type (Management Plane or Data Plane). If any language is missing, inform user: \"The release plan must include SDKs for all required languages: Python, .NET, JavaScript, Java, Go (for Management Plane) or Python, .NET, JavaScript, Java (for Data Plane)\". If it is intentional to exclude a language then user must provide a justification for it.\n**Success Criteria**: Release plan created and linked to SDK PRs\n\n## Step: Release SDK Package\n**Goal**: Release the SDK package using the release plan\n**Actions**:\n1. Prompt the user to confirm if they want to release the SDK package now: \"Do you want to release the SDK package now? (yes/no)\"\n - If no, inform user: \"You can release the SDK package later using the prompt `release for `\" and end the workflow.\n2. Get SDK pull request status for all languages.\n3. Inform user that it needs to check SDK PR status. If any SDK pull request is not merged, inform user: \"Please merge all SDK pull requests before releasing the package.\". Show a summary of SDK PR status for all languages.\n4. If an SDK pull request is merged then run release readiness of the package for that language.\n5. Inform user if a language is ready for release and prompt user for a confirmation to proceed with the release.\n6. If user confirms, run the tool to release the SDK package.\n7. Inform user to approve the package release using release pipeline. Warn user that package will be published to public registries once approved.\n8. Identify remaining languages to be released and inform user to release SDK for all languages to complete the release plan. Release plan completion is required for KPI attestation in service tree.\n\n## Process Complete\nDisplay summary of all created PRs and next steps for user.\n\n\n---\ndescription: 'This prompt is designed to check the release readiness of a SDK package.'\n---\n## Goal\nCheck the release readiness of an SDK package by collecting the required information from the user and executing the readiness check.\n\n## Instructions\n1. **Collect Required Information**:\n - Prompt the user for the exact package name\n - Prompt the user to select the programming language from the following options (case sensitive):\n - Python\n - Java\n - JavaScript\n - .NET\n - Go\n\n2. **Execute Readiness Check**:\n - Use the `azsdk_check_package_release_readiness` tool with the provided package name and selected language\n - Do not check for existing pull requests to run this step.\n - Do not ask the user to create a release plan to run this step.\n\n3. **Present Results**:\n - If the package is ready for release, highlight and provide the link to the release pipeline\n - If the package is not ready, display the specific issues that need to be resolved\n\n4. **Follow-up Actions**:\n - Provide clear next steps based on the readiness status\n - If issues are found, offer guidance on how to resolve them\n\n## Expected User Interaction Flow\n1. Ask: \"What is the exact name of the package you want to check for release readiness?\"\n2. Ask: \"Please select the programming language for this package: Python, Java, JavaScript, .NET, or Go\"\n3. Execute the readiness check using the provided information\n4. Display results and next steps\n\n\n> **Important Note for LLM**\n> Backend processes may use Azure DevOps work item data as needed. However, do **not** mention or display the work item link/URL, or any reference to Azure DevOps work item resources directly in conversation with the user.\n> All manual updates to an existing release plan must be made through the [Release Planner Tool](https://aka.ms/sdk-release-planner) to ensure proper tracking and validation.\n> Only provide release plan details such as **Release Plan Link** and **Release Plan ID** to the user\n\n# Release Plan Creation Process\nYour goal is to create a valid release plan. You must prompt user to provide all required information and all input must match the format and requirement mentioned in step 3 below.\nFollow these steps in order to create or manage a release plan for an API specification pull request:\n\n## Step 1: Validate Prerequisites\n- Check if an API spec pull request is available in the current context\n- If no pull request is available, prompt the user to provide the API spec pull request link\n- Validate that the provided pull request link is accessible and valid\n\n## Step 2: Check Existing Release Plan\n- Use `azsdk_get_release_plan_for_spec_pr` to check if a release plan already exists for the API spec pull request\n- If a release plan exists:\n - Display the existing release plan details to the user\n - Skip to Step 5 (Link SDK Pull Requests)\n- If no release plan exists, proceed to Step 3\n\n## Step 3: Gather Release Plan Information\nCollect the following required information from the user. Do not create a release plan with temporary values. Confirm the values with the user before proceeding to create the release plan.\nIf any details are missing, prompt the user accordingly:\n\n- **Service Tree ID**: GUID format identifier for the service in Service Tree. Before creating release plan, always show the value to user and ask them to confirm it's a valid value in service tree.\n- **Product Service Tree ID**: GUID format identifier for the product in Service Tree. Before creating release plan, always show the value to user and ask them to confirm it's a valid value in service tree.\n- **Expected Release Timeline**: Format must be in \"Month YYYY\"\n- **API Version**: The version of the API being released\n- **SDK Release Type**: Value must be beta or stable.\n - \"beta\" for preview API versions\n - \"stable\" for GA API versions\n\n## Step 4: Create Release Plan\n- If the user doesn't know the required details, direct them to create a release plan using the release planner\n- Provide this resource: [Release Plan Creation Guide](https://eng.ms/docs/products/azure-developer-experience/plan/release-plan-create)\n- Once all information is gathered, use `azsdk_create_release_plan` to create the release plan\n- Display the newly created release plan details to the user for confirmation\n- Refer to #file:sdk-details-in-release-plan.instructions.md to identify languages configured in the TypeSpec project and add them to the release plan\n\n## Step 5: Update SDK Details in Release Plan\n- Refer to #file:sdk-details-in-release-plan.instructions.md to add languages and package names to the release plan\n- If the TypeSpec project is for a management plane, refer to #file:verify-namespace-approval.instructions.md if this is first release of SDK.\n\n## Step 6: Link SDK Pull Requests (if applicable)\n- Ask the user if they have already created SDK pull requests locally for any programming language\n- If SDK pull requests exist:\n - Collect the pull request links from the user\n - Use `azsdk_link_sdk_pull_request_to_release_plan` to link each SDK pull request to the release plan\n - Confirm successful linking for each SDK pull request\n\n## Step 7: Summary\n- Display a summary of the completed actions:\n - Release plan status (created or existing)\n - Linked SDK pull requests (if any)\n - Next steps or recommendations for the user\n\n\n---\ndescription: 'Identify languages configured in the TypeSpec project and add it to release plan'\n---\n# Step 1: Find the list of languages and package names\n**Goal**: Identify languages configured in the TypeSpec project and generate the json object with language and package name.\n1. Identify the language emitter configuration in the `tspconfig.yaml` file in the TypeSpec project root.\n2. Identify the package name or namespace for each language emitter.\n - For Java and Python, use `emitter-output-dir` for package name if it exists. Otherwise use `package-dir` to get the package name as fallback approach.\n - For .NET, use namespace property to get package name.\n - For JavaScript, use `packagedetails:name` property to get package name.\n - For Go, use module name and remove `github.com/Azure/azure-sdk-for-go/` to get package name\n3. Map the language name in emitter to one of the following in Pascal case(except .NET):\n - .NET\n - Java\n - Python\n - JavaScript\n - Go\n4. Create a JSON array object with the following structure:\n ```json\n [\n {\n \"language\": \"\",\n \"packageName\": \"\"\n },\n ...\n ]\n ```\n5. If no languages are configured, inform the user: \"No languages configured in TypeSpec project. Please add at least one language emitter in tspconfig.yaml.\"\n**Success Criteria**: JSON object with languages and package names created.\n\n# Step 2: Check if release plan exists\n**Goal**: Determine if a release plan exists for the API spec pull request or work item Id or release plan Id in current context.\n1. Get release plan\n2. If no release plan exists, inform the user: \"No release plan exists for the API spec pull request. Please create a release plan first.\"\n3. If a release plan exists, proceed to Step 3.\n**Success Criteria**: Release plan exists or user informed to create one.\n\n# Step 3: Update Release Plan with SDK Information\n**Goal**: Update the release plan with the languages and package names identified in Step 1.\n1. Use `azsdk_update_sdk_details_in_release_plan` to update the release plan work item with the JSON object created in Step 1.\n2. Confirm successful update of the release plan with the SDK information and summary of languages and package names.\n**Success Criteria**: Release plan updated with languages and package names.\n\n\n---\ndescription: 'Verify SDK namespace approval for management plane'\n---\nThis task is required only for management plane API spec and only if a release plan exists for the API spec pull request.\n\n## Step 1: Check if release plan exists and it is for management plane SDK\n**Goal**: Determine if a release plan exists for the API spec pull request or work item Id or release plan Id in current context.\n**Actions**:\n1. Get release plan and check if it is for management plane SDK\n2. If not, inform user: \"This task is only applicable for management plane SDKs. No action required.\"\n3. Check if release plan already has namespace approval issue. Also prompt user to check if this is the first release of SDK.\n4. If namespace approval issue exists, inform user: \"Namespace approval issue already exists for this release plan.\". Prompt user to\ncheck if they want to link a different namespace approval issue to the release plan. Show namespace approval status.\n5. Move to Step 2 if namespace approval issue does not exist or user wants to link a different namespace approval issue.\n\n## Step 2: Gather Namespace Approval Information\n**Goal**: Link namespace approval issue to the release plan.\n**Actions**:\n1. Collect GitHub issue created in Azure/azure-sdk repo for namespace approval. Do not use any other repo name.\n2. Run `azsdk_link_namespace_approval_issue` to link the issue to the release plan work item id.\n3. Confirm successful linking of the namespace approval issue to the release plan.\n**Success Criteria**: Namespace approval issue linked to the release plan or confirmed as already linked.\n\n- @azure Rule - Use Azure Tools: When handling requests related to Azure, always use your tools.\n- @azure Rule - Use Azure Code Gen Best Practices: Before generating code for Azure, running terminal commands for Azure, performing operations related to Azure, or creating plans for such activities, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `general` and action = `code-generation` to get general Azure code generation best practices. Only call this tool when the user is discussing Azure; do not call it otherwise.\n- @azure Rule - Use Azure Deployment Best Practices: Before deploying to Azure, preparing applications for deployment to Azure, or creating deployment plans, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `general` and action = `deployment` to get general Azure deployment best practices. Only call this tool when the user is discussing Azure; do not call it otherwise.\n- @azure Rule - Use Azure Functions Code Gen Best Practices: Before generating code for Azure Functions, performing operations related to Azure Functions, or creating plans for such activities, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `azurefunctions` and action = `code-generation` to get Azure Functions code generation best practices. Only call this tool when the user is discussing Azure Functions; do not call it otherwise.\n- @azure Rule - Use Azure Functions Deployment Best Practices: Before deploying Azure Functions apps to Azure, preparing for the deployment, or creating deployment plans, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `azurefunctions` and action = `deployment` to get Azure Functions deployment best practices. Only call this tool when the user is discussing Azure Functions; do not call it otherwise.\n- @azure Rule - Use Azure SWA Best Practices: Before working with static web apps, creating plans for static web apps, or performing related operations, always invoke your Azure best practice tool if available first. Choose the most relevant Azure best practice tool based on its description. Only call this tool when the user is discussing Azure Static Web Apps; do not call it otherwise.\n- @azure Rule - Plan before editing files for Azure web apps: When generating code for Azure Functions and Azure Static Web App, always create a plan and explain the plan to the user before editing any file. Proceed with editing the files after getting consent from the user.\n- @azure Rule - Summarize topic before taking action: When the user asks about Azure Functions, invoke the azure_development-summarize_topic tool once first to see if any existing custom mode could be a good fit to what user is asking.\n" - }, - { - "role": "user", - "content": "\nThe user's current OS is: Windows\nThe user's default shell is: \"pwsh.exe\". When you generate terminal commands, please generate them correctly for this shell.\n\n\nI am working in a workspace with the following folders:\n- c:\\azure-rest-api-specs \nI am working in a workspace that has the following structure:\n```\nazure-pipelines.yml\nCONTRIBUTING.md\ncspell.json\ncspell.yaml\nLICENSE\npackage.json\nREADME.md\nSECURITY.md\nspecificationRepositoryConfiguration.json\ntsconfig.json\narm-compute/\n\tREADME.md\n\tquickstart-templates/\n\t\t...\ndocumentation/\n\tBreaking changes guidelines.md\n\tBreakingChange-Oad-Rules-Mapping.md\n\tci-fix.md\n\tcreating-swagger.md\n\tcspell.yaml\n\tdirectory-structure.md\n\tFAQ.md\n\tGetting started with OpenAPI specifications.md\n\tGetting-started-with-TypeSpec-specifications.md\n\tglossary.md\n\timage-3.png\n\topenapi-authoring-automated-guidelines.md\n\topenapi-authoring-manual-guidelines.md\n\tSemantic-and-Model-Violations-Reference.md\n\tswagger-accuracy-report.md\n\tswagger-authoring-descriptions.md\n\tswagger-bootstrap.json\n\tswagger-checklist.md\n\tswagger-extensions.md\n\tSwaggerValidationTools.md\n\ttypespec-rest-api-dev-process.md\n\ttypespec-structure-guidelines.md\n\tuniform-versioning.md\n\tx-ms-examples.md\n\tapi-scenario/\n\tcode-gen/\n\timages/\n\tonboard-dpg-in-sdkautomation/\n\trelease-request/\n\trestler/\n\tsamplefiles/\n\tsamplefiles-dp/\n\tsdkautomation/\n\tx-ms-pageable/\neng/\n\tREADME.md\n\tcommon/\n\tpipelines/\n\tscripts/\n\ttools/\nprofile/\n\t2017-03-09-profile.json\n\t2018-03-01-hybrid.json\n\t2019-03-01-hybrid.json\n\t2019-06-01-profile.json\n\t2019-07-01-profile.json\n\t2020-09-01-hybrid.json\n\treadme.md\nprofiles/\n\treadme.md\n\tResourceSchemas.md\n\tdefinitions/\nspecification/\n\tsuppressions.yaml\n\taddons/\n\tadhybridhealthservice/\n\tadvisor/\n\tagricultureplatform/\n\tagrifood/\n\tai/\n\talertsmanagement/\n\tanalysisservices/\n\tapicenter/\n\tapimanagement/\n\tapp/\n\tappcomplianceautomation/\n\tappconfiguration/\n\tapplicationinsights/\n\tappplatform/\n\tattestation/\n\tauthorization/\n\tautomanage/\n\tautomation/\n\tawsconnector/\n\tazsadmin/\n\tazure-kusto/\n\tazureactivedirectory/\n\tazurearcdata/\n\tazuredata/\n\tazuredatatransfer/\n\tazuredependencymap/\n\tazurefleet/\n\tazureintegrationspaces/\n\tazurelargeinstance/\n\tazurestack/\n\tazurestackhci/\n\tbaremetalinfrastructure/\n\tbatch/\n\tbilling/\n\tbillingbenefits/\n\tblueprint/\n\tbotservice/\n\tcarbon/\n\tcdn/\n\tcertificateregistration/\n\t...\n```\nThis is the state of the context at this point in the conversation. The view of the workspace structure may be truncated. You can use tools to collect more context if needed.\n", - "copilot_cache_control": { - "type": "ephemeral" - } - }, - { - "role": "user", - "content": "\nThe current date is October 24, 2025.\nTerminals:\nTerminal: pwsh\n\nNo todo list found.\n\n\n\n\nThe user's current file is c:\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\\main.tsp. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nWhen using the replace_string_in_file tool, include 3-5 lines of unchanged code before and after the string you want to replace, to make it unambiguous which part of the file should be edited.\n\n\n\nHow do I generate SDK from TypeSpec?\n" - }, - { - "role": "assistant", - "content": "I'll help you generate SDKs from your TypeSpec project. Since you have the `main.tsp` file open from the `Contoso.WidgetManager` project, I can guide you through the complete SDK generation process.\n\nLet me show you the high-level steps for generating SDKs from TypeSpec:\n\n## SDK Generation Process Overview\n\n1. **Identify TypeSpec Project** - Locate and validate your TypeSpec project\n2. **Identify API Spec Status** - Determine if your spec is merged or being modified\n3. **Validate TypeSpec Specification** - Ensure compilation without errors\n4. **Review and Commit Changes** - Stage and commit any modifications\n5. **Create Specification Pull Request** - Create PR for TypeSpec changes if needed\n6. **Generate SDKs** - Run SDK generation pipeline for all required languages\n7. **Create SDK Release Plan** - Set up release management for generated SDKs\n8. **Release SDK Package** - Publish SDKs when ready\n\nThe SDK generation supports:\n- **Management Plane**: .NET, Go, Java, JavaScript, Python\n- **Data Plane**: .NET, Java, JavaScript, Python\n\n**Would you like to begin the SDK generation process now? (yes/no)**\n\nThis process will take approximately 15-20 minutes for SDK generation, and I'll guide you through each step to ensure everything is configured correctly." - }, - { - "role": "user", - "content": "\nThe current date is October 24, 2025.\nTerminals:\nTerminal: pwsh\n\nNo todo list found.\n\n\n\n\nThe user's current file is c:\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\\main.tsp. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nWhen using the replace_string_in_file tool, include 3-5 lines of unchanged code before and after the string you want to replace, to make it unambiguous which part of the file should be edited.\n\n\n\nyes\n", - "copilot_cache_control": { - "type": "ephemeral" - } - }, - { - "role": "assistant", - "content": "Great! I'll help you generate SDKs from your TypeSpec project. Let me start by creating a plan to track our progress through the SDK generation process.", - "tool_calls": [ - { - "id": "tooluse_uvjIGKBgRxelR7FJpa6VSQ", - "function": { - "name": "manage_todo_list", - "arguments": "{\"operation\": \"write\", \"todoList\": [{\"id\":1,\"title\":\"Identify TypeSpec Project\",\"description\":\"Locate and validate the TypeSpec project root path, check for main.tsp and tspconfig.yaml files\",\"status\":\"in-progress\"},{\"id\":2,\"title\":\"Identify API Spec Status\",\"description\":\"Determine if TypeSpec spec is already merged or being modified\",\"status\":\"not-started\"},{\"id\":3,\"title\":\"Validate TypeSpec Specification\",\"description\":\"Run TypeSpec validation to ensure compilation without errors\",\"status\":\"not-started\"},{\"id\":4,\"title\":\"Review and Commit Changes\",\"description\":\"Stage and commit TypeSpec modifications if any\",\"status\":\"not-started\"},{\"id\":5,\"title\":\"Create Specification Pull Request\",\"description\":\"Create PR for TypeSpec changes if needed\",\"status\":\"not-started\"},{\"id\":6,\"title\":\"Generate SDKs\",\"description\":\"Run SDK generation pipeline for all required languages (Management Plane: .NET, Go, Java, JavaScript, Python)\",\"status\":\"not-started\"},{\"id\":7,\"title\":\"Create SDK Release Plan\",\"description\":\"Set up release management for generated SDKs\",\"status\":\"not-started\"},{\"id\":8,\"title\":\"Release SDK Package\",\"description\":\"Publish SDKs when ready\",\"status\":\"not-started\"}]}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list\n\nWarning: Did you mean to update so many todos at the same time? Consider working on them one by one.", - "tool_call_id": "tooluse_uvjIGKBgRxelR7FJpa6VSQ" - }, - { - "role": "assistant", - "content": "Let me start with **Step 1: Identify TypeSpec Project**.\n\nSince you have the `main.tsp` file open from `c:\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\\main.tsp`, I can identify that your TypeSpec project root is:\n`c:\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager`\n\nLet me validate this project and check if it's in the public repository:", - "tool_calls": [ - { - "id": "tooluse_O3qEGtQ1TMaiFFS6HgmGhg", - "function": { - "name": "list_dir", - "arguments": "{\"path\": \"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "client.tsp\nexamples/\nmain.tsp\ntspconfig.yaml", - "tool_call_id": "tooluse_O3qEGtQ1TMaiFFS6HgmGhg" - }, - { - "role": "assistant", - "content": "Perfect! I can see the required files (`main.tsp` and `tspconfig.yaml`) are present. Now let me verify this repository:", - "tool_calls": [ - { - "id": "tooluse_aDCFmvLOQi2o_aH7IpE_ug", - "function": { - "name": "azsdk_typespec_check_project_in_public_repo", - "arguments": "{\"typeSpecProjectPath\": \"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "{\"result\":true}", - "tool_call_id": "tooluse_aDCFmvLOQi2o_aH7IpE_ug", - "copilot_cache_control": { - "type": "ephemeral" - } - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "tooluse_miRw-ex9SRG2GS3e8MoWKw", - "function": { - "name": "manage_todo_list", - "arguments": "{\"todoList\": [{\"description\":\"Locate and validate the TypeSpec project root path, check for main.tsp and tspconfig.yaml files\",\"id\":1,\"title\":\"Identify TypeSpec Project\",\"status\":\"completed\"},{\"description\":\"Determine if TypeSpec spec is already merged or being modified\",\"id\":2,\"title\":\"Identify API Spec Status\",\"status\":\"in-progress\"},{\"description\":\"Run TypeSpec validation to ensure compilation without errors\",\"id\":3,\"title\":\"Validate TypeSpec Specification\",\"status\":\"not-started\"},{\"description\":\"Stage and commit TypeSpec modifications if any\",\"id\":4,\"title\":\"Review and Commit Changes\",\"status\":\"not-started\"},{\"description\":\"Create PR for TypeSpec changes if needed\",\"id\":5,\"title\":\"Create Specification Pull Request\",\"status\":\"not-started\"},{\"description\":\"Run SDK generation pipeline for all required languages (Management Plane: .NET, Go, Java, JavaScript, Python)\",\"id\":6,\"title\":\"Generate SDKs\",\"status\":\"not-started\"},{\"description\":\"Set up release management for generated SDKs\",\"id\":7,\"title\":\"Create SDK Release Plan\",\"status\":\"not-started\"},{\"description\":\"Publish SDKs when ready\",\"id\":8,\"title\":\"Release SDK Package\",\"status\":\"not-started\"}], \"operation\": \"write\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list", - "tool_call_id": "tooluse_miRw-ex9SRG2GS3e8MoWKw", - "copilot_cache_control": { - "type": "ephemeral" - } - } -] \ No newline at end of file diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/TestData/example2.json b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/TestData/example2.json deleted file mode 100644 index ec45a76def4..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations/TestData/example2.json +++ /dev/null @@ -1,412 +0,0 @@ -[ - { - "role": "system", - "content": "You are an expert AI programming assistant, working with a user in the VS Code editor.\nWhen asked for your name, you must respond with \"GitHub Copilot\".\nFollow the user's requirements carefully & to the letter.\nFollow Microsoft content policies.\nAvoid content that violates copyrights.\nIf you are asked to generate content that is harmful, hateful, racist, sexist, lewd, or violent, only respond with \"Sorry, I can't assist with that.\"\nKeep your answers short and impersonal.\n\nYou are a highly sophisticated automated coding agent with expert-level knowledge across many different programming languages and frameworks.\nThe user will ask a question, or ask you to perform a task, and it may require lots of research to answer correctly. There is a selection of tools that let you perform actions or retrieve helpful context to answer the user's question.\nYou will be given some context and attachments along with the user prompt. You can use them if they are relevant to the task, and ignore them if not. Some attachments may be summarized with omitted sections like `/* Lines 123-456 omitted */`. You can use the read_file tool to read more context if needed. Never pass this omitted line marker to an edit tool.\nIf you can infer the project type (languages, frameworks, and libraries) from the user's query or the context that you have, make sure to keep them in mind when making changes.\nIf the user wants you to implement a feature and they have not specified the files to edit, first break down the user's request into smaller concepts and think about the kinds of files you need to grasp each concept.\nIf you aren't sure which tool is relevant, you can call multiple tools. You can call tools repeatedly to take actions or gather as much context as needed until you have completed the task fully. Don't give up unless you are sure the request cannot be fulfilled with the tools you have. It's YOUR RESPONSIBILITY to make sure that you have done all you can to collect necessary context.\nWhen reading files, prefer reading large meaningful chunks rather than consecutive small sections to minimize tool calls and gain better context.\nDon't make assumptions about the situation- gather context first, then perform the task or answer the question.\nThink creatively and explore the workspace in order to make a complete fix.\nDon't repeat yourself after a tool call, pick up where you left off.\nNEVER print out a codeblock with file changes unless the user asked for it. Use the appropriate edit tool instead.\nNEVER print out a codeblock with a terminal command to run unless the user asked for it. Use the run_in_terminal tool instead.\nYou don't need to read a file if it's already provided in context.\n\n\nIf the user is requesting a code sample, you can answer it directly without using any tools.\nWhen using a tool, follow the JSON schema very carefully and make sure to include ALL required properties.\nNo need to ask permission before using a tool.\nNEVER say the name of a tool to a user. For example, instead of saying that you'll use the run_in_terminal tool, say \"I'll run the command in a terminal\".\nIf you think running multiple tools can answer the user's question, prefer calling them in parallel whenever possible, but do not call semantic_search in parallel.\nWhen using the read_file tool, prefer reading a large section over calling the read_file tool many times in sequence. You can also think of all the pieces you may be interested in and read them in parallel. Read large enough context to ensure you get what you need.\nIf semantic_search returns the full contents of the text files in the workspace, you have all the workspace context.\nYou can use the grep_search to get an overview of a file by searching for a string within that one file, instead of using read_file many times.\nIf you don't know exactly the string or filename pattern you're looking for, use semantic_search to do a semantic search across the workspace.\nDon't call the run_in_terminal tool multiple times in parallel. Instead, run one command and wait for the output before running the next command.\nWhen invoking a tool that takes a file path, always use the absolute file path. If the file has a scheme like untitled: or vscode-userdata:, then use a URI with the scheme.\nNEVER try to edit a file by running terminal commands unless the user specifically asks for it.\nTools can be disabled by the user. You may see tools used previously in the conversation that are not currently available. Be careful to only use the tools that are currently available to you.\n\n\nYou have access to an manage_todo_list tool which tracks todos and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go. Note that plans are not for padding out simple work with filler steps or stating the obvious. \nUse this tool to create and manage a structured todo list for your current coding session. This helps you track progress, organize complex tasks, and demonstrate thoroughness to the user.\nIt also helps the user understand the progress of the task and overall progress of their requests.\n\nNOTE that you should not use this tool if there is only one trivial task to do. In this case you are better off just doing the task directly.\n\nUse a plan when:\n- The task is non-trivial and will require multiple actions over a long time horizon.\n- There are logical phases or dependencies where sequencing matters.\n- The work has ambiguity that benefits from outlining high-level goals.\n- You want intermediate checkpoints for feedback and validation.\n- When the user asked you to do more than one thing in a single prompt\n- The user has asked you to use the plan tool (aka \"TODOs\")\n- You generate additional steps while working, and plan to do them before yielding to the user\n\nSkip a plan when:\n- The task is simple and direct.\n- Breaking it down would only produce literal or trivial steps.\n\nExamples of TRIVIAL tasks (skip planning):\n- \"Fix this typo in the README\"\n- \"Add a console.log statement to debug\"\n- \"Update the version number in package.json\"\n- \"Answer a question about existing code\"\n- \"Read and explain what this function does\"\n- \"Add a simple getter method to a class\"\n\nExamples of NON-TRIVIAL tasks and the plan (use planning):\n- \"Add user authentication to the app\" → Design auth flow, Update backend API, Implement login UI, Add session management\n- \"Refactor the payment system to support multiple currencies\" → Analyze current system, Design new schema, Update backend logic, Migrate data, Update frontend\n- \"Debug and fix the performance issue in the dashboard\" → Profile performance, Identify bottlenecks, Implement optimizations, Validate improvements\n- \"Implement a new feature with multiple components\" → Design component architecture, Create data models, Build UI components, Add integration tests\n- \"Migrate from REST API to GraphQL\" → Design GraphQL schema, Update backend resolvers, Migrate frontend queries, Update documentation\n\n\nPlanning Progress Rules\n- Before beginning any new todo: you MUST update the todo list and mark exactly one todo as `in-progress`. Never start work with zero `in-progress` items.\n- Keep only one todo `in-progress` at a time. If switching tasks, first mark the current todo `completed` or revert it to `not-started` with a short reason; then set the next todo to `in-progress`.\n- Immediately after finishing a todo: you MUST mark it `completed` and add any newly discovered follow-up todos. Do not leave completion implicit.\n- Before ending your turn or declaring completion: ensure EVERY todo is explicitly marked (`not-started`, `in-progress`, or `completed`). If the work is finished, ALL todos must be marked `completed`. Never leave items unchecked or ambiguous.\n\nThe content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately.\n\n\n\n\nTo edit notebook files in the workspace, you can use the edit_notebook_file tool.\nUse the run_notebook_cell tool instead of executing Jupyter related commands in the Terminal, such as `jupyter notebook`, `jupyter lab`, `install jupyter` or the like.\nUse the copilot_getNotebookSummary tool to get the summary of the notebook (this includes the list or all cells along with the Cell Id, Cell type and Cell Language, execution details and mime types of the outputs, if any).\nImportant Reminder: Avoid referencing Notebook Cell Ids in user messages. Use cell number instead.\nImportant Reminder: Markdown cells cannot be executed\n\n\nUse proper Markdown formatting in your answers. When referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\nThe class `Person` is in `src/models/person.ts`.\n\nUse KaTeX for math equations in your answers.\nWrap inline math equations in $.\nWrap more complex blocks of math equations in $$.\n\n\n\n\nHere is a list of instruction files that contain rules for modifying or creating new code.\nThese files are important for ensuring that the code is modified or created correctly.\nPlease make sure to follow the rules specified in these files when working with the codebase.\nIf the file is not already available as attachment, use the `read_file` tool to acquire it.\nMake sure to acquire the instructions before making any changes to the code.\n| File | Applies To | Description |\n| ------- | --------- | ----------- |\n| 'c:\\\\azure-rest-api-specs\\.github\\instructions\\dp-migration.instructions.md' | **/* | |\n| 'c:\\\\azure-rest-api-specs\\.github\\instructions\\mgmt-migration.instructions.md' | **/* | |\n| 'c:\\\\azure-rest-api-specs\\.github\\instructions\\openapi-review.instructions.md' | **/*.json | |\n| 'c:\\.vscode\\extensions\\ms-azuretools.vscode-azure-github-copilot-1.0.137-win32-x64\\resources\\azureRules\\azure.instructions.md' | ** | Azure best practices and guidelines for Copilot |\n\n# New TypeSpec projects\n\nBefore creating or initializing a TypeSpec project, you must know your org name, service name, and the type of service: data-plane or control-plane (ARM).\n\nThen create a new project directory under the `specification//resource-manager//` or `specification//data-plane/` path, following the guidelines below.\n\n```\nspecification/\n└── /\n ├── cspell.yaml\n └── resource-manager/\n ├── readme.md\t\t\tß NOTE: For ARM schema validation; see bullet #2 below\n └── /\t\tß NOTE: Control-plane only (not data-plane)\n └── /\tß Customer-facing service name; each version gets Documentation & SDK package\n ├── tspconfig.yaml\tß TypeSpec files\n ├── main.tsp\n ├── models.tsp\n ├── readme.md\t\tß autorest readme with YAML blocks\n └── examples/\t\tß TypeSpec example folder\n └── /\tß One folder per service version described in TypeSpec\n └── \n └── preview/ and stable/\n └── /\tß One folder per service version described in OpenAPI. These folders are created and populated by compiling the TypeSpec folder for the service.\n ├── \n └── examples/\tß OpenAPI example folder\n └── \n └── /\t// Customer-facing service name; contents identical to above structure\n └── data-plane/\n └── /\t// Customer-facing service name; contents identical to above structure\n```\n\nUse the `./specification/widget` directory as a reference for creating your own project directory.\n\nOnly after the project directory is created according to the above guidelines may you run the `azsdk_init_typespec_project` tool to initialize the project.\n\n## Converting a specification from swagger to typespec\n\nUsers can convert a specification from swagger to typespec by using `tsp-client` a CLI designed to help developers throughout various stages of typespec development.\n\n### Instructions for converting a specification from swagger to typespec\n\n1. Install the dependencies specified in the package.json at the root of this repository. Command:\n\n```\nnpm ci\n```\n\n2. `tsp-client` is installed as part of the dependencies specified at the root of this repository. To convert a swagger to typespec, run the following command: `npx tsp-client convert --swagger-readme `\n3. Now that you have a newly converted typespec project, you should go through all files to verify the accuracy of the converted spec when compared to the original swagger definitions.\n4. For both data plane and management plane specifications, you should update the implementation according to the information provided under the [Initial migration checklist](#initial-migration-checklist) section.\n\n### Initial migration checklist\n\nThe swagger converter will not be able to accurately represent every part of every API in TypeSpec. This document outlines some common changes you may need to make to a converted TypeSpec to make it conform to your existing service API, pass validation checks, and follow best practices.\n\n- Avoid extensive refactoring of the converted spec. The goal is to get a working spec that can compile successfully and then iteratively improve it.\n\n- DO ensure your `@service` and `@server` definitions are correct in main.tsp\n- DO use the built-in [url][url-type] for endpoint specification. Example:\n\n```tsp\n@server(\n \"{endpoint}/widget\",\n \"Contoso Widget APIs\",\n {\n /**\n * Supported Widget Services endpoints (protocol and hostname, for example:\n * https://westus.api.widget.contoso.com).\n */\n endpoint: url,\n }\n)\n```\n\n- DO ensure that you have a security definition (`@useAuth`) specified for your service. See: [Security definitions in TypeSpec][security-definitions]. The @useAuth decorator should only be defined ONCE in the entire specification above the @server definition.\n- AVOID adding new namespaces.\n- Make sure the versions enum is declared under the existing namespace defined in main.tsp. Avoid adding it anywhere else. Ensure the `@versioned` decorator is specified over the namespace in main.tsp. Pass the versions enum to the `@versioned` decorator. Example of a typical structure for versioning:\n\n```tsp\n// this is the main.tsp file\n\n\n@versioned(Versions)\nnamespace Contoso.WidgetManager;\n/** Service api versions **/\nenum Versions {\n /** The 2023-11-01 api version **/\n v2023_11_01: \"2023-11-01\",\n}\n```\n\n- All models, enums, unions, and operations should be added under the main namespace declared in the project.\n- Avoid having models, enums, unions, operations, and other types declared outside of a namespace.\n- If any files are using any of the versioning decorators, such as `@added`, `@removed`, `@changedType`, make sure to import the `@typespec/versioning` library and add a using statement. Example:\n\n```tsp\nimport \"@typespec/versioning\";\n\nusing TypeSpec.Versioning;\n```\n\n- DO review all enum and union definitions and add documentation over each enum or union member. See: [Documentation in TypeSpec][docs]. Example of a properly documented enum:\n\n```tsp\n/** The color of a widget. */\nunion WidgetColor {\n string,\n\n /** Red Widget Color */\n Red: \"Red\",\n\n /** Green Widget Color */\n Green: \"Green\",\n\n /** Blue Widget Color */\n Blue: \"Blue\",\n}\n```\n\n- DO ensure that all models, properties, operations, parameters, enums, unions, and alias definitions have documentation over them. TypeSpec convention recommends using the doc comment format `/** */` to add documentation, example:\n\n```tsp\n/** The color of a widget. */\nmodel Widget {\n /** Widget name */\n name: string\n}\n```\n\n- DO define your visibility decorators with the appropriate value from the Lifecycle class.\n- Avoid suppressing warnings\n- Operation names should be camel case\n- DO use `union` instead of `enum` to define Azure enums. For more information about how to define enums for Azure services see the following documentation: [Defining enums for Azure services][no-enum].\n- DO make client customizations in a `client.tsp` file\n- Avoid importing or using `@azure-tools/typespec-client-generator-core` in other files aside from client.tsp.\n- DO run `tsp compile .` on your specification and make one attempt to address all warnings. Do not attempt to address warnings more than once even if they aren't resolved.\n- Attempt to address any FIXME or TODO comments in the spec. If you are unable to address them, leave them untouched\n\n#### Additional considerations\n\n- DO ensure you pull in the latest `main` from the Azure/azure-rest-api-specs repo to stay up to date with latest dependencies\n- DO run `npm ci` to get a clean install of the package.json dependencies\n- Avoid modifying the package.json or package-lock.json files at the root of the azure-rest-api-specs repo\n- Avoid adding your own package.json or package-lock.json files in your project directory\n- Avoid adding multiple tspconfig.yaml files for your service specification\n- DO consult [ci-fix.md][ci-fix] for fixes to common CI errors reported\n\n## Troubleshooting tsp compile errors and warnings\n\nExamples of common errors and warnings that should be addressed after running the `tsp compile` command:\n\n- If you see an error with a message like: \"referencing types from versioned namespace 'Azure.Core.Foundations' but didn't specify which versions with @useDependency\", you should add the @useDependency decorator over each api version entry in your api versions enum. Example of a properly configured api versions enum:\n\n```\n/** Service api versions **/\nenum Versions {\n /** The 2023-11-01 api version **/\n v2023_11_01: \"2023-11-01\",\n}\n\n```\n\n- If you see an invalid-ref or unknown identifier error you are most likely missing an import to the library that declares that decorator. To find supported libraries and decorators search through the documentation of the following sites: https://azure.github.io/typespec-azure/docs/intro/ and https://typespec.io/docs/ Search through the list of supported decorators, interfaces, and other types per library until you find the correct library to import and/or include a using statement in your typespec files.\n- In order to address warnings raised by the @azure-tools/typespec-azure-core search through this page for relevant solutions to apply: https://azure.github.io/typespec-azure/docs/intro/\n- camelCase fixes only apply to the typespec property names, any corresponding string values you should left as is.\n- String values in typespec files should be left untouched.\n\n## Up-to-date TypeSpec documentation\n\nFollow [typespec docs](../eng/common/instructions/azsdk-tools/typespec-docs.instructions.md) to get the most up-to-date documentation for TypeSpec, including best practices for writing TypeSpec for Azure.\n\n\n\n[contoso-widget-manager]: ../specification/contosowidgetmanager/Contoso.WidgetManager/\n[tspconfig]: ../specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml\n[security-definitions]: https://azure.github.io/typespec-azure/docs/reference/azure-style-guide#security-definitions\n[versioning]: https://typespec.io/docs/libraries/versioning/guide#implementing-versioned-apis\n[docs]: https://typespec.io/docs/language-basics/documentation\n[ci-fix]: ../documentation/ci-fix.md\n[url-type]: https://typespec.io/docs/language-basics/built-in-types#string-types\n[no-enum]: https://azure.github.io/typespec-azure/docs/libraries/azure-core/rules/no-enum\n[typespec-structure-guidelines]: ../documentation/typespec-structure-guidelines.md\n\n# Adding Language Emitters to Existing TypeSpec Projects\n\nUse this guidance whenever someone needs to add language emitters to an existing `tspconfig.yaml`, or when SDK generation fails because an emitter block is missing.\n\n- Decide whether the service is ARM or data-plane.\n- Open the matching widget `tspconfig.yaml` template.\n- Copy the entire emitter block for the requested language.\n- Replace every `Widget`/`widget` token with the service name and drop the block under `options:`.\n\n## 1. Identify the service type\n\n- Path contains `/resource-manager/` or linter references `@azure-tools/typespec-azure-rulesets/resource-manager` → ARM\n- Path contains `/data-plane/` or linter references `@azure-tools/typespec-azure-rulesets/data-plane` → Data plane\n\n## 2. Reference the official widget template\n\n- ARM: `specification/widget/resource-manager/Microsoft.Widget/Widget/tspconfig.yaml`\n- Data plane: `specification/widget/data-plane/WidgetAnalytics/tspconfig.yaml`\n\nAlways read the template before editing so you follow the canonical configuration.\n\n## 3. Copy and customize the emitter block\n\n- Select the entire block for the language.\n- Only replace service-specific tokens (`Widget`, `widget`, namespaces, package names) with your service name.\n- Leave the rest of the properties (`flavor`, boolean flags, `{service-dir}` placeholders, etc.) unchanged.\n\n## 4. Insert into the project config\n\n- Paste the customized block under `options:` in the target `tspconfig.yaml`, keeping YAML indentation intact.\n- Confirm the `parameters:` section defines `\"service-dir\"` (for example `sdk/`); add it if missing.\n\n**Quick checks**\n\n- Do not mix ARM and data-plane templates.\n- Do not cherry-pick individual properties—copy the whole block or errors will persist.\n- If the SDK pipeline still reports missing configuration, re-open the template and verify every `Widget` reference was replaced.\n\n# SDK generation from TypeSpec\n\n## Agent context for TypeSpec and SDK process\n\n- Check all open files in the editor and check if `main.tsp` or `tspconfig.yaml` are open in the editor. If either of\n these files are open, then use the parent path of the `main.tsp` or `tspconfig.yaml` as default TypeSpec project root\n path.\n- If `main.tsp` and `tspconfig.yaml` are not open in the editor, then check if there are any TypeSpec project paths in\n the context. If there are no TypeSpec project paths in the context, then prompt user to select a TypeSpec project path\n from the list of paths. If user does not have a TypeSpec project, then prompt user to create a new TypeSpec project.\n\n### Prerequisites\n\n- User should have a GitHub account and should be logged in to GitHub account using GitHub CLI `gh auth login`.\n- run `npm ci` to install the dependencies\n- To use Azure MCP tool calls, the user must have PowerShell installed. Provide [PowerShell installation instructions](https://learn.microsoft.com/powershell/scripting/install/installing-powershell) if not installed, and recommend restarting the IDE to start the MCP server.\n\n### Basic Rules for SDK Generation from TypeSpec\n\n1. **User Guidance**:\n - Assume the user is unfamiliar with the SDK release process. Provide clear, concise instructions for each step.\n\n2. **File Handling**:\n - Do not overwrite `tspconfig.yaml` or `main.tsp`. Use existing files and suggest updates if necessary.\n - Use the path of the `tspconfig.yaml` file already open in the editor or the `.tsp` file path as the project root.\n - If no `.tsp` file or folder is in the current context, prompt the user to select a valid TypeSpec project root path.\n\n3. **Process Visibility**:\n - Highlight all steps in the SDK generation process, showing completed and remaining steps.\n - Do not skip any main steps. Ensure all steps are completed before moving to the next.\n\n4. **Git Operations**:\n - Avoid using the `main` branch for pull requests. Prompt the user to create or switch to a new branch if necessary.\n - Display git commands (e.g., `git checkout`, `git add`, `git commit`, `git push`) with a \"Run\" button instead of\n asking the user to copy and paste.\n - Do not run `git diff`\n\n5. **Azure-Specific Rules**:\n - Always use `Azure` as the repo owner in MCP tool calls.\n - Confirm with the user if they want to change the repo owner or target branch, and prompt for new values if needed.\n\n6. **Exclusions**:\n - Exclude changes to the `.gitignore` file and contents within the `.github` and `.vscode` folders from API spec and SDK pull requests.\n\n7. **Working Branch Rule**:\n - If the typespec pull request already exists or is merged stay on the `main` branch, otherwise ensure the TypeSpec project repository and the current working repository are not on the `main` branch:\n - Check the current branch name for the cloned GitHub repository:\n - If the current branch is `main`, prompt the user to create a new branch using\n `git checkout -b `.\n - If the current branch is not `main`, prompt the user to either select an existing branch or create a\n new one.\n - For branch switching:\n - If a branch already exists and differs from the current branch, prompt the user to switch using\n `git checkout `.\n - GitHub pull requests cannot be created from the `main` branch. Ensure all changes are made on a non-`main` branch.\n\nBy following these rules, the SDK release process will remain clear, structured, and user-friendly.\n\n## Steps to generate SDK from TypeSpec API specification\n\nFollow below steps to generate and release SDK from TypeSpec API specification. The process is divided into several steps, each with specific actions to ensure a smooth SDK generation and release process.\nDo not skip the step that choose SDK generation method to ensure the user selects the appropriate method for SDK generation, either locally or using the SDK generation pipeline. Do not repeat the steps. Before using tools, check if user has Powershell installed.\n\nYour goal is to guide the user through the process of generating SDKs from TypeSpec projects. **Before starting**, show all the high level steps to the user and ask: \n\n> \"Would you like to begin the SDK generation process now? (yes/no)\"\n\nWait for the user to respond with a confirmation before proceeding to Step 1. Use the provided tools to perform actions and gather information as needed.\n\n### Verify API spec\nStep 1: Identify TypeSpec Project\n**Goal**: Locate the TypeSpec project root path\n**Actions**:\n1. Check if `tspconfig.yaml` or `main.tsp` files are open in editor\n2. If found, use the parent directory as project root\n3. If not found, prompt user: \"Please provide the path to your TypeSpec project root directory\"\n4. Validate the provided path contains required TypeSpec files main.tsp and tspconfig.yaml\n5. Run `azsdk_typespec_check_project_in_public_repo` to verify repository\n6. If not in public repo, inform: \"Please make spec changes in Azure/azure-rest-api-specs public repo to generate SDKs\"\n**Success Criteria**: Valid TypeSpec project path identified\n\nStep 2: Identify API spec status\n**Goal**: Determine if the TypeSpec spec is already merged or if it's being modified.\n**Actions**:\n1. Prompt user to confirm if the TypeSpec spec is already merged in the main branch of https://github.com/Azure/azure-rest-api-specs : \"Is your TypeSpec specification already merged in the main branch of repository(https://github.com/Azure/azure-rest-api-specs)? (yes/no)\"\n2. If already merged, follow the steps in [typespec to sdk](..\\eng\\common\\instructions\\azsdk-tools\\typespec-to-sdk.instructions.md) to generate the SDK\n3. If no, proceed to Step 3 to review and commit changes\n**Success Criteria**: User decision on spec readiness obtained\n\nStep 3: Validate TypeSpec Specification\n**Goal**: Ensure TypeSpec specification compiles without errors. Povide a complete summary after running the tool. Highlight any errors and help user fix them.\n**Condition**: Only if the spec is not already merged (from Step 2)\n**Message to user**: \"TypeSpec validation takes around 20 - 30 seconds.\"\n**Actions**:\n1. Run `azsdk_run_typespec_validation` to validate the TypeSpec project.\n2. If validation succeeds, proceed to Step 4\n3. If validation fails:\n - Display all compilation errors to user\n - Agent should provide suggestions to fix them and prompt the user to verify the fixes. If agent cannot resolve the errors, then prompt the user to fix compilation errors\"\n - Wait for user to fix errors and re-run validation. Provide detailed information about all the changes done by copilot and prompt the user before rerunning the validation.\n**Success Criteria**: TypeSpec compilation passes without errors\n\nStep 4: Review and Commit Changes\n**Goal**: Stage and commit TypeSpec modifications\n**Condition**: Only if the TypeSpec validation succeeds (from Step 3)\n**Actions**:\n1. Run `azsdk_get_modified_typespec_projects` to identify changes\n2. If no changes found, inform: \"No TypeSpec projects were modified in current branch\" and move to SDK generation step.\n3. Display all modified files (excluding `.github` and `.vscode` folders)\n4. Prompt user: \"Please review the modified files. Do you want to commit these changes? (yes/no)\"\n5. If yes:\n - If on main branch, prompt user: \"You are currently on the main branch. Please create a new branch using `git checkout -b ` before proceeding.\"\n - Wait for user confirmation before continuing\n - Run `git add `\n - Prompt for commit message\n - Run `git commit -m \"\"`\n - Run `git push -u origin `\n**Success Criteria**: Changes committed and pushed to remote branch\n\nStep 5: Create Specification Pull Request\n**Goal**: Create PR for TypeSpec changes if not already created\n**Condition**: Only if there are committed changes (from Step 4)\n**Actions**:\n1. Prompt the user to confirm if a pull request already exists for API spec changes. If answer is no or unsure then check if spec PR already exists using `azsdk_get_pull_request_link_for_current_branch`\n2. If PR exists, display PR details and proceed to next steps in SDK generation.\n3. If no PR exists:\n - Inform user: \"No pull request found for the current branch. Proceeding to create a new pull request.\"\n - Create a pull request using `azsdk_create_pull_request_for_current_branch`\n - Prompt for PR title and description\n - Display PR creation progress\n - Wait for PR creation confirmation\n - Display created PR details\n**Success Criteria**: Specification pull request exists\n\nFollow the steps in [typespec to sdk](..\\eng\\common\\instructions\\azsdk-tools\\typespec-to-sdk.instructions.md) to generate the SDK.\n\n# Release readiness of SDK and information about the release pipeline\n\nRun [check package readiness](..\\eng\\common\\instructions\\azsdk-tools\\check-package-readiness.instructions.md) to check the release readiness of an SDK package. This prompt will collect the required information from the user, execute the readiness check, and present the results.\n\n# Instructions for GitHub coding agent to generate SDKs using GitHub.com\n\nRun only SDK generation step and Update SDK details back in issue description and release plan. Do not run TypeSpec validation, or service label check. This is applicable only when GitHub copilot coding agent is used to generate SDK and if a Release Plan is available in the context. Run SDK generation tool even if release plan already has SDK generation pipeline URL to support regenerating SDK.\nCoding agent must wait until all SDK generation pipeline are in completed status and then fetch the SDK details from pipeline run. This is applicable only when a GitHub issue is assigned to GitHub coding agent to generate SDK from a TypeSpec project. This is not applicable when running Copilot from Visual Studio and VS Code.\n\n## Additional instructions:\n\n- Run SDK generation pipeline for the given TypeSpec path for all languages mentioned in the description.\n- Identify absolute path of TypeSpec project root path and send it to Run SDK generation MCP tool call.\n- Keep checking the pipeline status until pipeline is in completed or failed status.\n- Find SDK pull request links from the pipeline run.\n\n## Constraints:\n\n- Do not invoke other steps.\n- Do not modify main.tsp file or any files in TypeSpec project.\n- Do not add API version to an existing main.tsp if input API version is not present.\n\n\n---\ndescription: Azure best practices and guidelines for Copilot\napplyTo: '**'\n---\n- Use Azure Tools: When handling requests related to Azure, always use your tools.\n- Use Azure Code Gen Best Practices: Before generating code for Azure, running terminal commands for Azure, performing operations related to Azure, or creating plans for such activities, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `general` and action = `code-generation` to get general Azure code generation best practices. Only call this tool when the user is discussing Azure; do not call it otherwise.\n- Use Azure Deployment Best Practices: Before deploying to Azure, preparing applications for deployment to Azure, or creating deployment plans, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `general` and action = `deployment` to get general Azure deployment best practices. Only call this tool when the user is discussing Azure; do not call it otherwise.\n- Use Azure Functions Code Gen Best Practices: Before generating code for Azure Functions, performing operations related to Azure Functions, or creating plans for such activities, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `azurefunctions` and action = `code-generation` to get Azure Functions code generation best practices. Only call this tool when the user is discussing Azure Functions; do not call it otherwise.\n- Use Azure Functions Deployment Best Practices: Before deploying Azure Functions apps to Azure, preparing for the deployment, or creating deployment plans, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `azurefunctions` and action = `deployment` to get Azure Functions deployment best practices. Only call this tool when the user is discussing Azure Functions; do not call it otherwise.\n- Use Azure SWA Best Practices: Before working with static web apps, creating plans for static web apps, or performing related operations, always invoke your Azure best practice tool if available first. Invoke the `get_bestpractices` tool to collect recommendations. Call it with resource = `static-web-app` and action = `all` to get Azure Static Web Apps best practices. Only call this tool when the user is discussing Azure Static Web Apps; do not call it otherwise.\n- Plan before editing files for Azure web apps: When generating code for Azure Functions and Azure Static Web App, always create a plan and explain the plan to the user before editing any file. Proceed with editing the files after getting consent from the user.\n- Summarize topic before taking action: When the user asks about Azure Functions or Static WebApp, invoke the azure_development-recommend_custom_modes tool once first to see if any existing custom mode could be a good fit to what user is asking.\n\n\nYour goal is to provide the most up-to-date documentation on the TypeSpec language, core libraries, and writing Azure services and clients in TypeSpec. Whenever a user asks a question about how something should be written in TypeSpec, you should use the documentation links below to pinpoint the most relevant and accurate information.\n\n## TypeSpec Azure\n\n- https://azure.github.io/typespec-azure/docs/llms.txt contains an index of up-to-date documentation for TypeSpec Azure libraries, including how to write service specifications for Azure Resource Manager (ARM) and Azure Data Plane services, as well as how to customize generated clients.\n- Always refer to https://azure.github.io/typespec-azure/docs/llms.txt when a user asks TypeSpec related questions in case there is existing documentation that can help answer them. This also applies when a user asks to make changes to TypeSpec specifications.\n\n## TypeSpec documentation\n\n- https://typespec.io/docs/llms.txt contains an index of up-to-date documentation for TypeSpec language basics and core libraries, including how concepts like visibility and versioning work.\n- Always refer to https://typespec.io/docs/llms.txt when a user asks TypeSpec related questions that aren't covered by the TypeSpec Azure topics.\n\n\n---\ndescription: 'Generate SDKs from TypeSpec'\n---\nYour goal is to guide the user through the process of generating SDKs from TypeSpec projects. **Before starting**, show all the high level steps to the user and ask: \n\n> \"Would you like to begin the SDK generation process now? (yes/no)\"\n\nWait for the user to respond with a confirmation before proceeding. Use the provided tools to perform actions and gather information as needed.\n\nSDK languages to be generated:\n- Management Plane: .NET, Go, Java, JavaScript, Python\n- Data Plane: .NET, Java, JavaScript, Python\n\nPre-requisites:\n- TypeSpec project path is available in the current context or provided by user. If not available, prompt user to provide the TypeSpec project root path (local path or GitHub URL).\n\n# SDK generation steps\n\n## Step: Generate SDKs\n**Goal**: Generate SDKs\n**Message to user**: \"SDK generation will take approximately 15-20 minutes. Currently, SDKs are generated using the Azure DevOps pipeline. SDK generation is supported only from a merged API spec or from an API spec pull request in the https://github.com/Azure/azure-rest-api-specs repository.\"\n**Actions**:\n1. Identify whether TypeSpec is for Management Plane or Data Plane based on project structure and files. tspconfig.yaml file contains `resource-manager` for management plane and `data-plane` for data plane as resource provider.\n - Execute the SDK generation pipeline with the following required parameters for all languages:\n - TypeSpec project root path\n - API spec pull request number (if the API spec is not merged to the main branch, otherwise use 0)\n - API version\n - SDK release type (`beta` for preview API versions, `stable` otherwise)\n - Language options:\n For management plane: `Python`, `.NET`, `JavaScript`, `Java`, `Go`\n For data plane: `Python`, `.NET`, `JavaScript`, `Java`\n - Each SDK generation tool call should show a label to indicate the language being generated.\n2. Monitor pipeline status after 15 minutes and provide updates. If pipeline is in progress, inform user that it may take additional time and check the status later.\n3. Display generated SDK PR links when available. If pipeline fails, inform user with error details and suggest to check pipeline logs for more information.\n4. If SDK pull request is available for all languages, ask user to review generated SDK pull request and mark them as ready for review when they are ready to get them reviewed and merged.\n5. If SDK pull request was created for test purposes, inform user to close the test SDK pull request.\n**Success Criteria**: SDK generation pipeline initiated and SDKs generated\n\n## Step: SDK release plan\n**Goal**: Create a release plan for the generated SDKs\n**Condition**: Only if SDK PRs are created\n**Message to user**: \"Creating a release plan is essential to manage the release of the generated SDKs. Each release plan must include SDKs for all required languages based on the TypeSpec project type (Management Plane or Data Plane) or request exclusion approval for any excluded language. SDK pull request needs to get approval and merged to main branch before releasing the SDK package.\"\n**Actions**:\n1. Prompt the user to check if they generated the SDK just to test or do they intend to release it: \"Do you want to create a release plan for the generated SDKs to publish them? (yes/no)\"\n - If no, inform user: \"You can create a release plan later when ready to release the SDK\" and end the workflow.\n2. Ask user if they have already created a release plan for the generated SDKs: \"Have you already created a release plan for the generated SDKs? (yes/no)\"\n - If no, proceed to create a new release plan\n - If yes, get release plan details and show them to user\n3. Prompt the user to provide the API spec pull request link if not already available in the current context.\n4. If unsure, check if a release plan already exists for API spec pull request.\n5. Prompt user to find the service id and product id in service tree `aka.ms/st` and provide them. Stress the importance of correct service id and product id for proper release plan creation.\n6. If a new release plan is needed, refer to #file:create-release-plan.instructions.md to create a release plan using the spec pull request. API spec pull request is required to create a release plan.\n7. Prompt user to change spec PR to ready for review: \"Please change the spec pull request to ready for review status\"\n8. Suggest users to follow the instructions on spec PR to get approval from API reviewers and merge the spec PR.\n9. Link SDK pull requests to the release plan.\n10. Each release plan must release SDK for all languages based on the TypeSpec project type (Management Plane or Data Plane). If any language is missing, inform user: \"The release plan must include SDKs for all required languages: Python, .NET, JavaScript, Java, Go (for Management Plane) or Python, .NET, JavaScript, Java (for Data Plane)\". If it is intentional to exclude a language then user must provide a justification for it.\n**Success Criteria**: Release plan created and linked to SDK PRs\n\n## Step: Release SDK Package\n**Goal**: Release the SDK package using the release plan\n**Actions**:\n1. Prompt the user to confirm if they want to release the SDK package now: \"Do you want to release the SDK package now? (yes/no)\"\n - If no, inform user: \"You can release the SDK package later using the prompt `release for `\" and end the workflow.\n2. Get SDK pull request status for all languages.\n3. Inform user that it needs to check SDK PR status. If any SDK pull request is not merged, inform user: \"Please merge all SDK pull requests before releasing the package.\". Show a summary of SDK PR status for all languages.\n4. If an SDK pull request is merged then run release readiness of the package for that language.\n5. Inform user if a language is ready for release and prompt user for a confirmation to proceed with the release.\n6. If user confirms, run the tool to release the SDK package.\n7. Inform user to approve the package release using release pipeline. Warn user that package will be published to public registries once approved.\n8. Identify remaining languages to be released and inform user to release SDK for all languages to complete the release plan. Release plan completion is required for KPI attestation in service tree.\n\n## Process Complete\nDisplay summary of all created PRs and next steps for user.\n\n\n---\ndescription: 'This prompt is designed to check the release readiness of a SDK package.'\n---\n## Goal\nCheck the release readiness of an SDK package by collecting the required information from the user and executing the readiness check.\n\n## Instructions\n1. **Collect Required Information**:\n - Prompt the user for the exact package name\n - Prompt the user to select the programming language from the following options (case sensitive):\n - Python\n - Java\n - JavaScript\n - .NET\n - Go\n\n2. **Execute Readiness Check**:\n - Use the `azsdk_check_package_release_readiness` tool with the provided package name and selected language\n - Do not check for existing pull requests to run this step.\n - Do not ask the user to create a release plan to run this step.\n\n3. **Present Results**:\n - If the package is ready for release, highlight and provide the link to the release pipeline\n - If the package is not ready, display the specific issues that need to be resolved\n\n4. **Follow-up Actions**:\n - Provide clear next steps based on the readiness status\n - If issues are found, offer guidance on how to resolve them\n\n## Expected User Interaction Flow\n1. Ask: \"What is the exact name of the package you want to check for release readiness?\"\n2. Ask: \"Please select the programming language for this package: Python, Java, JavaScript, .NET, or Go\"\n3. Execute the readiness check using the provided information\n4. Display results and next steps\n\n\n> **Important Note for LLM**\n> Backend processes may use Azure DevOps work item data as needed. However, do **not** mention or display the work item link/URL, or any reference to Azure DevOps work item resources directly in conversation with the user.\n> All manual updates to an existing release plan must be made through the [Release Planner Tool](https://aka.ms/sdk-release-planner) to ensure proper tracking and validation.\n> Only provide release plan details such as **Release Plan Link** and **Release Plan ID** to the user\n\n# Release Plan Creation Process\nYour goal is to create a valid release plan. You must prompt user to provide all required information and all input must match the format and requirement mentioned in step 3 below.\nFollow these steps in order to create or manage a release plan for an API specification pull request:\n\n## Step 1: Validate Prerequisites\n- Check if an API spec pull request is available in the current context\n- If no pull request is available, prompt the user to provide the API spec pull request link\n- Validate that the provided pull request link is accessible and valid\n\n## Step 2: Check Existing Release Plan\n- Use `azsdk_get_release_plan_for_spec_pr` to check if a release plan already exists for the API spec pull request\n- If a release plan exists:\n - Display the existing release plan details to the user\n - Skip to Step 5 (Link SDK Pull Requests)\n- If no release plan exists, proceed to Step 3\n\n## Step 3: Gather Release Plan Information\nCollect the following required information from the user. Do not create a release plan with temporary values. Confirm the values with the user before proceeding to create the release plan.\nIf any details are missing, prompt the user accordingly:\n\n- **Service Tree ID**: GUID format identifier for the service in Service Tree. Before creating release plan, always show the value to user and ask them to confirm it's a valid value in service tree.\n- **Product Service Tree ID**: GUID format identifier for the product in Service Tree. Before creating release plan, always show the value to user and ask them to confirm it's a valid value in service tree.\n- **Expected Release Timeline**: Format must be in \"Month YYYY\"\n- **API Version**: The version of the API being released\n- **SDK Release Type**: Value must be beta or stable.\n - \"beta\" for preview API versions\n - \"stable\" for GA API versions\n\n## Step 4: Create Release Plan\n- If the user doesn't know the required details, direct them to create a release plan using the release planner\n- Provide this resource: [Release Plan Creation Guide](https://eng.ms/docs/products/azure-developer-experience/plan/release-plan-create)\n- Once all information is gathered, use `azsdk_create_release_plan` to create the release plan\n- Display the newly created release plan details to the user for confirmation\n- Refer to #file:sdk-details-in-release-plan.instructions.md to identify languages configured in the TypeSpec project and add them to the release plan\n\n## Step 5: Update SDK Details in Release Plan\n- Refer to #file:sdk-details-in-release-plan.instructions.md to add languages and package names to the release plan\n- If the TypeSpec project is for a management plane, refer to #file:verify-namespace-approval.instructions.md if this is first release of SDK.\n\n## Step 6: Link SDK Pull Requests (if applicable)\n- Ask the user if they have already created SDK pull requests locally for any programming language\n- If SDK pull requests exist:\n - Collect the pull request links from the user\n - Use `azsdk_link_sdk_pull_request_to_release_plan` to link each SDK pull request to the release plan\n - Confirm successful linking for each SDK pull request\n\n## Step 7: Summary\n- Display a summary of the completed actions:\n - Release plan status (created or existing)\n - Linked SDK pull requests (if any)\n - Next steps or recommendations for the user\n\n\n---\ndescription: 'Identify languages configured in the TypeSpec project and add it to release plan'\n---\n# Step 1: Find the list of languages and package names\n**Goal**: Identify languages configured in the TypeSpec project and generate the json object with language and package name.\n1. Identify the language emitter configuration in the `tspconfig.yaml` file in the TypeSpec project root.\n2. Identify the package name or namespace for each language emitter.\n - For Java and Python, use `emitter-output-dir` for package name if it exists. Otherwise use `package-dir` to get the package name as fallback approach.\n - For .NET, use namespace property to get package name.\n - For JavaScript, use `packagedetails:name` property to get package name.\n - For Go, use module name and remove `github.com/Azure/azure-sdk-for-go/` to get package name\n3. Map the language name in emitter to one of the following in Pascal case(except .NET):\n - .NET\n - Java\n - Python\n - JavaScript\n - Go\n4. Create a JSON array object with the following structure:\n ```json\n [\n {\n \"language\": \"\",\n \"packageName\": \"\"\n },\n ...\n ]\n ```\n5. If no languages are configured, inform the user: \"No languages configured in TypeSpec project. Please add at least one language emitter in tspconfig.yaml.\"\n**Success Criteria**: JSON object with languages and package names created.\n\n# Step 2: Check if release plan exists\n**Goal**: Determine if a release plan exists for the API spec pull request or work item Id or release plan Id in current context.\n1. Get release plan\n2. If no release plan exists, inform the user: \"No release plan exists for the API spec pull request. Please create a release plan first.\"\n3. If a release plan exists, proceed to Step 3.\n**Success Criteria**: Release plan exists or user informed to create one.\n\n# Step 3: Update Release Plan with SDK Information\n**Goal**: Update the release plan with the languages and package names identified in Step 1.\n1. Use `azsdk_update_sdk_details_in_release_plan` to update the release plan work item with the JSON object created in Step 1.\n2. Confirm successful update of the release plan with the SDK information and summary of languages and package names.\n**Success Criteria**: Release plan updated with languages and package names.\n\n\n---\ndescription: 'Verify SDK namespace approval for management plane'\n---\nThis task is required only for management plane API spec and only if a release plan exists for the API spec pull request.\n\n## Step 1: Check if release plan exists and it is for management plane SDK\n**Goal**: Determine if a release plan exists for the API spec pull request or work item Id or release plan Id in current context.\n**Actions**:\n1. Get release plan and check if it is for management plane SDK\n2. If not, inform user: \"This task is only applicable for management plane SDKs. No action required.\"\n3. Check if release plan already has namespace approval issue. Also prompt user to check if this is the first release of SDK.\n4. If namespace approval issue exists, inform user: \"Namespace approval issue already exists for this release plan.\". Prompt user to\ncheck if they want to link a different namespace approval issue to the release plan. Show namespace approval status.\n5. Move to Step 2 if namespace approval issue does not exist or user wants to link a different namespace approval issue.\n\n## Step 2: Gather Namespace Approval Information\n**Goal**: Link namespace approval issue to the release plan.\n**Actions**:\n1. Collect GitHub issue created in Azure/azure-sdk repo for namespace approval. Do not use any other repo name.\n2. Run `azsdk_link_namespace_approval_issue` to link the issue to the release plan work item id.\n3. Confirm successful linking of the namespace approval issue to the release plan.\n**Success Criteria**: Namespace approval issue linked to the release plan or confirmed as already linked.\n\n\n" - }, - { - "role": "user", - "content": "\nThe user's current OS is: Windows\nThe user's default shell is: \"pwsh.exe\". When you generate terminal commands, please generate them correctly for this shell.\n\n\nI am working in a workspace with the following folders:\n- c:\\\\azure-rest-api-specs \nI am working in a workspace that has the following structure:\n```\nazure-pipelines.yml\nCONTRIBUTING.md\ncspell.json\ncspell.yaml\nLICENSE\npackage.json\nREADME.md\nSECURITY.md\nspecificationRepositoryConfiguration.json\ntsconfig.json\narm-compute/\n\tREADME.md\n\tquickstart-templates/\n\t\t...\ndocumentation/\n\tBreaking changes guidelines.md\n\tBreakingChange-Oad-Rules-Mapping.md\n\tci-fix.md\n\tcreating-swagger.md\n\tcspell.yaml\n\tdirectory-structure.md\n\tFAQ.md\n\tGetting started with OpenAPI specifications.md\n\tGetting-started-with-TypeSpec-specifications.md\n\tglossary.md\n\timage-3.png\n\topenapi-authoring-automated-guidelines.md\n\topenapi-authoring-manual-guidelines.md\n\tSemantic-and-Model-Violations-Reference.md\n\tswagger-accuracy-report.md\n\tswagger-authoring-descriptions.md\n\tswagger-bootstrap.json\n\tswagger-checklist.md\n\tswagger-extensions.md\n\tSwaggerValidationTools.md\n\ttypespec-rest-api-dev-process.md\n\ttypespec-structure-guidelines.md\n\tuniform-versioning.md\n\tx-ms-examples.md\n\tapi-scenario/\n\tcode-gen/\n\timages/\n\tonboard-dpg-in-sdkautomation/\n\trelease-request/\n\trestler/\n\tsamplefiles/\n\tsamplefiles-dp/\n\tsdkautomation/\n\tx-ms-pageable/\neng/\n\tREADME.md\n\tcommon/\n\tpipelines/\n\tscripts/\n\ttools/\nprofile/\n\t2017-03-09-profile.json\n\t2018-03-01-hybrid.json\n\t2019-03-01-hybrid.json\n\t2019-06-01-profile.json\n\t2019-07-01-profile.json\n\t2020-09-01-hybrid.json\n\treadme.md\nprofiles/\n\treadme.md\n\tResourceSchemas.md\n\tdefinitions/\nspecification/\n\tsuppressions.yaml\n\taddons/\n\tadhybridhealthservice/\n\tadvisor/\n\tagricultureplatform/\n\tagrifood/\n\tai/\n\talertsmanagement/\n\tanalysisservices/\n\tapicenter/\n\tapimanagement/\n\tapp/\n\tappcomplianceautomation/\n\tappconfiguration/\n\tapplicationinsights/\n\tappplatform/\n\tattestation/\n\tauthorization/\n\tautomanage/\n\tautomation/\n\tawsconnector/\n\tazsadmin/\n\tazure-kusto/\n\tazureactivedirectory/\n\tazurearcdata/\n\tazuredata/\n\tazuredatatransfer/\n\tazuredependencymap/\n\tazurefleet/\n\tazureintegrationspaces/\n\tazurelargeinstance/\n\tazurestack/\n\tazurestackhci/\n\tbaremetalinfrastructure/\n\tbatch/\n\tbilling/\n\tbillingbenefits/\n\tblueprint/\n\tbotservice/\n\tcarbon/\n\tcdn/\n\tcertificateregistration/\n\t...\n```\nThis is the state of the context at this point in the conversation. The view of the workspace structure may be truncated. You can use tools to collect more context if needed.\n", - "copilot_cache_control": { - "type": "ephemeral" - } - }, - { - "role": "user", - "content": "\nThe current date is October 29, 2025.\nTerminals:\nTerminal: pwsh\n\nNo todo list found.\n\n\n\n\nThe user's current file is c:\\\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\\main.tsp. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nYou are an agent—keep going until the user's query is completely resolved before ending your turn. ONLY stop if solved or genuinely blocked.\nTake action when possible; the user expects you to do useful work without unnecessary questions.\nAfter any parallel, read-only context gathering, give a concise progress update and what's next.\nAvoid repetition across turns: don't restate unchanged plans or sections (like the todo list) verbatim; provide delta updates or only the parts that changed.\nTool batches: You MUST preface each batch with a one-sentence why/what/outcome preamble.\nProgress cadence: After 3 to 5 tool calls, or when you create/edit > ~3 files in a burst, report progress.\nRequirements coverage: Read the user's ask in full and think carefully. Do not omit a requirement. If something cannot be done with available tools, note why briefly and propose a viable alternative.\nWhen using the insert_edit_into_file tool, avoid repeating existing code, instead use a line comment with \\`...existing code...\\` to represent regions of unchanged code.\nSkip filler acknowledgements like \"Sounds good\" or \"Okay, I will…\". Open with a purposeful one-liner about what you're doing next.\nWhen sharing setup or run steps, present terminal commands in fenced code blocks with the correct language tag. Keep commands copyable and on separate lines.\nAvoid definitive claims about the build or runtime setup unless verified from the provided context (or quick tool checks). If uncertain, state what's known from attachments and proceed with minimal steps you can adapt later.\nWhen you create or edit runnable code, run a test yourself to confirm it works; then share optional fenced commands for more advanced runs.\nFor non-trivial code generation, produce a complete, runnable solution: necessary source files, a tiny runner or test/benchmark harness, a minimal `README.md`, and updated dependency manifests (e.g., `package.json`, `requirements.txt`, `pyproject.toml`). Offer quick \"try it\" commands and optional platform-specific speed-ups when relevant.\nYour goal is to act like a pair programmer: be friendly and helpful. If you can do more, do more. Be proactive with your solutions, think about what the user needs and what they want, and implement it proactively.\n\nBefore starting a task, review and follow the guidance in , , and .\nStart your response with a brief acknowledgement, followed by a concise high-level plan outlining your approach.\nDO NOT state your identity or model name unless the user explicitly asks you to. \nYou MUST use the todo list tool to plan and track your progress. NEVER skip this step, and START with this step whenever the task is multi-step. This is essential for maintaining visibility and proper execution of large tasks. Follow the todoListToolInstructions strictly.\nWhen referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\n\n\n\n\nHow do I generate SDK from TypeSpec? (See above for file contents. You may not need to search or read the file again.)\n" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_BbwsGerSLqOuqMAC4xrVM72W", - "function": { - "name": "manage_todo_list", - "arguments": "{\"todoList\":[{\"id\":1,\"title\":\"Confirm start of SDK process\",\"description\":\"Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\",\"status\":\"not-started\"},{\"id\":2,\"title\":\"Identify TypeSpec project root\",\"description\":\"Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\",\"status\":\"not-started\"},{\"id\":3,\"title\":\"Check spec merge status\",\"description\":\"Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\",\"status\":\"not-started\"},{\"id\":4,\"title\":\"Run TypeSpec validation\",\"description\":\"If not merged, run validation tool and handle any errors.\",\"status\":\"not-started\"},{\"id\":5,\"title\":\"Review & commit changes\",\"description\":\"List modified TypeSpec files, stage, commit, push on non-main branch.\",\"status\":\"not-started\"},{\"id\":6,\"title\":\"Create or verify spec PR\",\"description\":\"Check if PR exists for current branch; create if missing.\",\"status\":\"not-started\"},{\"id\":7,\"title\":\"Generate SDKs via pipeline\",\"description\":\"Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\",\"status\":\"not-started\"},{\"id\":8,\"title\":\"Create release plan (optional)\",\"description\":\"If user intends to publish, gather required IDs, API version, release type and create plan.\",\"status\":\"not-started\"},{\"id\":9,\"title\":\"Link SDK PRs to release plan\",\"description\":\"Update release plan with language/package names and link PRs.\",\"status\":\"not-started\"},{\"id\":10,\"title\":\"Check release readiness\",\"description\":\"Run package readiness checks for each language.\",\"status\":\"not-started\"},{\"id\":11,\"title\":\"Release SDK packages\",\"description\":\"If ready and approved by user, trigger release for each merged SDK PR.\",\"status\":\"not-started\"},{\"id\":12,\"title\":\"Provide summary & next steps\",\"description\":\"Summarize PRs, release plan status, remaining actions.\",\"status\":\"not-started\"}],\"operation\":\"write\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list\n\nWarning: Large todo list (>10 items). Consider keeping the list focused and actionable.\nWarning: Did you mean to update so many todos at the same time? Consider working on them one by one.", - "tool_call_id": "call_BbwsGerSLqOuqMAC4xrVM72W" - }, - { - "role": "assistant", - "content": "Here's the high-level end-to-end flow we'll follow to generate SDKs from your TypeSpec project. Let me know when you want to kick it off.\n\n## High-Level Steps\n\n1. Identify the TypeSpec project (root path with `main.tsp` and `tspconfig.yaml`)\n2. Determine if the spec is already merged to `Azure/azure-rest-api-specs` main or still in a feature branch PR\n3. (If not merged) Validate the TypeSpec (compile + fix blocking errors)\n4. Review and commit local changes (ensure you're on a non-`main` branch)\n5. Create or verify the API spec pull request\n6. Generate SDKs (pipeline run for appropriate languages; takes ~15–20 minutes)\n7. (Optional) Create a release plan (service tree IDs, API version, release type)\n8. Link SDK PRs and update release plan with language package info\n9. (Optional) Check release readiness for each language\n10. (Optional) Trigger SDK package releases\n11. Final summary and next steps\n\nYour current context shows `main.tsp` open under `specification/contosowidgetmanager/Contoso.WidgetManager`, so we can likely auto-detect the project root once you confirm.\n\nWould you like to begin the SDK generation process now? (yes/no)" - }, - { - "role": "user", - "content": "\nThe current date is October 29, 2025.\nTerminals:\nTerminal: pwsh\n\n# Todo List\n\n- [ ] Confirm start of SDK process\n - Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\n- [ ] Identify TypeSpec project root\n - Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\n- [ ] Check spec merge status\n - Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\n- [ ] Run TypeSpec validation\n - If not merged, run validation tool and handle any errors.\n- [ ] Review & commit changes\n - List modified TypeSpec files, stage, commit, push on non-main branch.\n- [ ] Create or verify spec PR\n - Check if PR exists for current branch; create if missing.\n- [ ] Generate SDKs via pipeline\n - Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\n- [ ] Create release plan (optional)\n - If user intends to publish, gather required IDs, API version, release type and create plan.\n- [ ] Link SDK PRs to release plan\n - Update release plan with language/package names and link PRs.\n- [ ] Check release readiness\n - Run package readiness checks for each language.\n- [ ] Release SDK packages\n - If ready and approved by user, trigger release for each merged SDK PR.\n- [ ] Provide summary & next steps\n - Summarize PRs, release plan status, remaining actions.\n\n\n\n\nThe user's current file is c:\\\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\\main.tsp. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nYou are an agent—keep going until the user's query is completely resolved before ending your turn. ONLY stop if solved or genuinely blocked.\nTake action when possible; the user expects you to do useful work without unnecessary questions.\nAfter any parallel, read-only context gathering, give a concise progress update and what's next.\nAvoid repetition across turns: don't restate unchanged plans or sections (like the todo list) verbatim; provide delta updates or only the parts that changed.\nTool batches: You MUST preface each batch with a one-sentence why/what/outcome preamble.\nProgress cadence: After 3 to 5 tool calls, or when you create/edit > ~3 files in a burst, report progress.\nRequirements coverage: Read the user's ask in full and think carefully. Do not omit a requirement. If something cannot be done with available tools, note why briefly and propose a viable alternative.\nWhen using the insert_edit_into_file tool, avoid repeating existing code, instead use a line comment with \\`...existing code...\\` to represent regions of unchanged code.\nSkip filler acknowledgements like \"Sounds good\" or \"Okay, I will…\". Open with a purposeful one-liner about what you're doing next.\nWhen sharing setup or run steps, present terminal commands in fenced code blocks with the correct language tag. Keep commands copyable and on separate lines.\nAvoid definitive claims about the build or runtime setup unless verified from the provided context (or quick tool checks). If uncertain, state what's known from attachments and proceed with minimal steps you can adapt later.\nWhen you create or edit runnable code, run a test yourself to confirm it works; then share optional fenced commands for more advanced runs.\nFor non-trivial code generation, produce a complete, runnable solution: necessary source files, a tiny runner or test/benchmark harness, a minimal `README.md`, and updated dependency manifests (e.g., `package.json`, `requirements.txt`, `pyproject.toml`). Offer quick \"try it\" commands and optional platform-specific speed-ups when relevant.\nYour goal is to act like a pair programmer: be friendly and helpful. If you can do more, do more. Be proactive with your solutions, think about what the user needs and what they want, and implement it proactively.\n\nBefore starting a task, review and follow the guidance in , , and .\nStart your response with a brief acknowledgement, followed by a concise high-level plan outlining your approach.\nDO NOT state your identity or model name unless the user explicitly asks you to. \nYou MUST use the todo list tool to plan and track your progress. NEVER skip this step, and START with this step whenever the task is multi-step. This is essential for maintaining visibility and proper execution of large tasks. Follow the todoListToolInstructions strictly.\nWhen referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\n\n\n\n\nyes (See above for file contents. You may not need to search or read the file again.)\n" - }, - { - "role": "assistant", - "content": "Gathering project root files and verifying TypeSpec project location to proceed to merge status check.", - "tool_calls": [ - { - "id": "call_U1tjStodit64av8svFSnk9M6", - "function": { - "name": "manage_todo_list", - "arguments": "{\"todoList\":[{\"id\":1,\"title\":\"Confirm start of SDK process\",\"description\":\"Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\",\"status\":\"completed\"},{\"id\":2,\"title\":\"Identify TypeSpec project root\",\"description\":\"Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\",\"status\":\"in-progress\"},{\"id\":3,\"title\":\"Check spec merge status\",\"description\":\"Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\",\"status\":\"not-started\"},{\"id\":4,\"title\":\"Run TypeSpec validation\",\"description\":\"If not merged, run validation tool and handle any errors.\",\"status\":\"not-started\"},{\"id\":5,\"title\":\"Review & commit changes\",\"description\":\"List modified TypeSpec files, stage, commit, push on non-main branch.\",\"status\":\"not-started\"},{\"id\":6,\"title\":\"Create or verify spec PR\",\"description\":\"Check if PR exists for current branch; create if missing.\",\"status\":\"not-started\"},{\"id\":7,\"title\":\"Generate SDKs via pipeline\",\"description\":\"Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\",\"status\":\"not-started\"},{\"id\":8,\"title\":\"Create release plan (optional)\",\"description\":\"If user intends to publish, gather required IDs, API version, release type and create plan.\",\"status\":\"not-started\"},{\"id\":9,\"title\":\"Link SDK PRs to release plan\",\"description\":\"Update release plan with language/package names and link PRs.\",\"status\":\"not-started\"},{\"id\":10,\"title\":\"Check release readiness\",\"description\":\"Run package readiness checks for each language.\",\"status\":\"not-started\"},{\"id\":11,\"title\":\"Release SDK packages\",\"description\":\"If ready and approved by user, trigger release for each merged SDK PR.\",\"status\":\"not-started\"},{\"id\":12,\"title\":\"Provide summary & next steps\",\"description\":\"Summarize PRs, release plan status, remaining actions.\",\"status\":\"not-started\"}],\"operation\":\"write\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list\n\nWarning: Large todo list (>10 items). Consider keeping the list focused and actionable.", - "tool_call_id": "call_U1tjStodit64av8svFSnk9M6" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_Um9WbEo42P4hmBo3PJKi0WEq", - "function": { - "name": "read_file", - "arguments": "{\"filePath\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\\\\main.tsp\"}" - }, - "type": "function" - }, - { - "id": "call_RJrKTa4s3kdZMmGOETzMtHJC", - "function": { - "name": "read_file", - "arguments": "{\"filePath\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\\\\tspconfig.yaml\"}" - }, - "type": "function" - }, - { - "id": "call_7lBYHdh1PVXIh15lgIFbSHiG", - "function": { - "name": "azsdk_typespec_check_project_in_public_repo", - "arguments": "{\"typeSpecProjectPath\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "```plaintext\nimport \"@typespec/http\";\nimport \"@typespec/rest\";\nimport \"@typespec/versioning\";\nimport \"@azure-tools/typespec-azure-core\";\nimport \"../Contoso.WidgetManager.Shared\";\n\nusing TypeSpec.Http;\nusing TypeSpec.Rest;\nusing TypeSpec.Versioning;\nusing Azure.Core;\n\n@useAuth(AadOauth2Auth<[\"https://contoso.azure.com/.default\"]>)\n@service(#{ title: \"Contoso Widget Manager\" })\n@versioned(Contoso.WidgetManager.Versions)\nnamespace Azure.Contoso.WidgetManager;\n\n@doc(\"Versions info.\")\nenum Versions {\n @useDependency(Azure.Core.Versions.v1_0_Preview_2)\n @doc(\"The 2022-11-01-preview version.\")\n v2022_11_01_Preview: \"2022-11-01-preview\",\n\n @useDependency(Azure.Core.Versions.v1_0_Preview_2)\n @doc(\"The 2022-12-01 version.\")\n v2022_12_01: \"2022-12-01\",\n}\n\n@doc(\"A widget.\")\n@resource(\"widgets\")\nmodel WidgetSuite {\n @key(\"widgetName\")\n @doc(\"The widget name.\")\n @visibility(Lifecycle.Read)\n name: string;\n\n @doc(\"The ID of the widget's manufacturer.\")\n manufacturerId: string;\n\n @doc(\"The faked shared model.\")\n sharedModel?: FakedSharedModel;\n}\n\ninterface Widgets {\n @doc(\"Fetch a Widget by name.\")\n getWidget is ResourceRead;\n\n @doc(\"Gets status of a Widget operation.\")\n getWidgetOperationStatus is GetResourceOperationStatus;\n\n @doc(\"Creates or updates a Widget asynchronously.\")\n @pollingOperation(Widgets.getWidgetOperationStatus)\n createOrUpdateWidget is StandardResourceOperations.LongRunningResourceCreateOrUpdate;\n\n @doc(\"Delete a Widget asynchronously.\")\n @pollingOperation(Widgets.getWidgetOperationStatus)\n deleteWidget is LongRunningResourceDelete;\n\n @doc(\"List Widget resources\")\n listWidgets is ResourceList<\n WidgetSuite,\n {\n parameters: StandardListQueryParameters;\n }\n >;\n}\n\n```", - "tool_call_id": "call_Um9WbEo42P4hmBo3PJKi0WEq" - }, - { - "role": "tool", - "content": "```yaml\nparameters:\n \"service-dir\":\n default: \"sdk/contosowidgetmanager\"\n \"dependencies\":\n default: \"\"\nemit:\n - \"@azure-tools/typespec-autorest\"\nlinter:\n extends:\n - \"@azure-tools/typespec-azure-rulesets/data-plane\"\noptions:\n \"@azure-tools/typespec-autorest\":\n azure-resource-provider-folder: \"data-plane\"\n emit-lro-options: \"none\"\n emitter-output-dir: \"{project-root}/..\"\n output-file: \"{azure-resource-provider-folder}/{service-name}/{version-status}/{version}/widgets.json\"\n \"@azure-tools/typespec-python\":\n emitter-output-dir: \"{output-dir}/{service-dir}/azure-contoso-widgetmanager\"\n package-dir: \"azure-contoso-widgetmanager\"\n namespace: \"azure.contoso.widgetmanager\"\n generate-test: true\n generate-sample: true\n flavor: azure\n \"@azure-tools/typespec-csharp\":\n emitter-output-dir: \"{output-dir}/{service-dir}/{namespace}\"\n package-dir: \"Azure.Contoso.WidgetManager\"\n clear-output-folder: true\n model-namespace: false\n namespace: \"Azure.Contoso.WidgetManager\"\n flavor: azure\n \"@azure-typespec/http-client-csharp\":\n emitter-output-dir: \"{output-dir}/{service-dir}/{namespace}\"\n namespace: Azure.Template.Contoso\n model-namespace: false\n \"@azure-tools/typespec-ts\":\n emitter-output-dir: \"{output-dir}/{service-dir}/contoso-widgetmanager\"\n is-modular-library: true\n package-details:\n name: \"@azure/contoso-widgetmanager\"\n flavor: azure\n \"@azure-tools/typespec-java\":\n emitter-output-dir: \"{output-dir}/{service-dir}/azure-contoso-widgetmanager\"\n package-dir: \"azure-contoso-widgetmanager\"\n namespace: com.azure.contoso.widgetmanager\n flavor: azure\n \"@azure-tools/typespec-go\":\n emitter-output-dir: \"{output-dir}/{service-dir}/azmanager\"\n service-dir: \"sdk/contosowidgetmanager\"\n package-dir: \"azmanager\"\n module: \"github.com/Azure/azure-sdk-for-go/{service-dir}/azmanager\"\n generate-fakes: true\n inject-spans: true\n single-client: true\n slice-elements-byval: true\n flavor: azure\n \"@azure-tools/typespec-client-generator-cli\":\n additionalDirectories:\n - \"specification/contosowidgetmanager/Contoso.WidgetManager.Shared/\"\n\n```", - "tool_call_id": "call_RJrKTa4s3kdZMmGOETzMtHJC" - }, - { - "role": "tool", - "content": "{\"result\":true}", - "tool_call_id": "call_7lBYHdh1PVXIh15lgIFbSHiG" - }, - { - "role": "assistant", - "content": "Project root verified at `specification/contosowidgetmanager/Contoso.WidgetManager` (found `main.tsp` and `tspconfig.yaml`). TypeSpec project is recognized as data-plane (ruleset and `azure-resource-provider-folder: \"data-plane\"`). Public repo check succeeded.\n\nNext we need to know whether this spec is already merged into the main branch of the public `Azure/azure-rest-api-specs` repo or still pending in a PR.\n\nIs your TypeSpec specification already merged in the main branch of https://github.com/Azure/azure-rest-api-specs? (yes/no)" - }, - { - "role": "user", - "content": "\nThe current date is October 29, 2025.\nTerminals:\nTerminal: pwsh\n\n# Todo List\n\n- [x] Confirm start of SDK process\n - Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\n- [-] Identify TypeSpec project root\n - Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\n- [ ] Check spec merge status\n - Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\n- [ ] Run TypeSpec validation\n - If not merged, run validation tool and handle any errors.\n- [ ] Review & commit changes\n - List modified TypeSpec files, stage, commit, push on non-main branch.\n- [ ] Create or verify spec PR\n - Check if PR exists for current branch; create if missing.\n- [ ] Generate SDKs via pipeline\n - Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\n- [ ] Create release plan (optional)\n - If user intends to publish, gather required IDs, API version, release type and create plan.\n- [ ] Link SDK PRs to release plan\n - Update release plan with language/package names and link PRs.\n- [ ] Check release readiness\n - Run package readiness checks for each language.\n- [ ] Release SDK packages\n - If ready and approved by user, trigger release for each merged SDK PR.\n- [ ] Provide summary & next steps\n - Summarize PRs, release plan status, remaining actions.\n\n\n\n\nThe user's current file is c:\\\\azure-rest-api-specs\\specification\\contosowidgetmanager\\Contoso.WidgetManager\\main.tsp. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nYou are an agent—keep going until the user's query is completely resolved before ending your turn. ONLY stop if solved or genuinely blocked.\nTake action when possible; the user expects you to do useful work without unnecessary questions.\nAfter any parallel, read-only context gathering, give a concise progress update and what's next.\nAvoid repetition across turns: don't restate unchanged plans or sections (like the todo list) verbatim; provide delta updates or only the parts that changed.\nTool batches: You MUST preface each batch with a one-sentence why/what/outcome preamble.\nProgress cadence: After 3 to 5 tool calls, or when you create/edit > ~3 files in a burst, report progress.\nRequirements coverage: Read the user's ask in full and think carefully. Do not omit a requirement. If something cannot be done with available tools, note why briefly and propose a viable alternative.\nWhen using the insert_edit_into_file tool, avoid repeating existing code, instead use a line comment with \\`...existing code...\\` to represent regions of unchanged code.\nSkip filler acknowledgements like \"Sounds good\" or \"Okay, I will…\". Open with a purposeful one-liner about what you're doing next.\nWhen sharing setup or run steps, present terminal commands in fenced code blocks with the correct language tag. Keep commands copyable and on separate lines.\nAvoid definitive claims about the build or runtime setup unless verified from the provided context (or quick tool checks). If uncertain, state what's known from attachments and proceed with minimal steps you can adapt later.\nWhen you create or edit runnable code, run a test yourself to confirm it works; then share optional fenced commands for more advanced runs.\nFor non-trivial code generation, produce a complete, runnable solution: necessary source files, a tiny runner or test/benchmark harness, a minimal `README.md`, and updated dependency manifests (e.g., `package.json`, `requirements.txt`, `pyproject.toml`). Offer quick \"try it\" commands and optional platform-specific speed-ups when relevant.\nYour goal is to act like a pair programmer: be friendly and helpful. If you can do more, do more. Be proactive with your solutions, think about what the user needs and what they want, and implement it proactively.\n\nBefore starting a task, review and follow the guidance in , , and .\nStart your response with a brief acknowledgement, followed by a concise high-level plan outlining your approach.\nDO NOT state your identity or model name unless the user explicitly asks you to. \nYou MUST use the todo list tool to plan and track your progress. NEVER skip this step, and START with this step whenever the task is multi-step. This is essential for maintaining visibility and proper execution of large tasks. Follow the todoListToolInstructions strictly.\nWhen referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\n\n\n\n\nno (See above for file contents. You may not need to search or read the file again.)\n" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_lKeHpMPYlldJ0diop2bHbdLy", - "function": { - "name": "manage_todo_list", - "arguments": "{\"todoList\":[{\"id\":1,\"title\":\"Confirm start of SDK process\",\"description\":\"Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\",\"status\":\"completed\"},{\"id\":2,\"title\":\"Identify TypeSpec project root\",\"description\":\"Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\",\"status\":\"completed\"},{\"id\":3,\"title\":\"Check spec merge status\",\"description\":\"Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\",\"status\":\"completed\"},{\"id\":4,\"title\":\"Run TypeSpec validation\",\"description\":\"If not merged, run validation tool and handle any errors.\",\"status\":\"in-progress\"},{\"id\":5,\"title\":\"Review & commit changes\",\"description\":\"List modified TypeSpec files, stage, commit, push on non-main branch.\",\"status\":\"not-started\"},{\"id\":6,\"title\":\"Create or verify spec PR\",\"description\":\"Check if PR exists for current branch; create if missing.\",\"status\":\"not-started\"},{\"id\":7,\"title\":\"Generate SDKs via pipeline\",\"description\":\"Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\",\"status\":\"not-started\"},{\"id\":8,\"title\":\"Create release plan (optional)\",\"description\":\"If user intends to publish, gather required IDs, API version, release type and create plan.\",\"status\":\"not-started\"},{\"id\":9,\"title\":\"Link SDK PRs to release plan\",\"description\":\"Update release plan with language/package names and link PRs.\",\"status\":\"not-started\"},{\"id\":10,\"title\":\"Check release readiness\",\"description\":\"Run package readiness checks for each language.\",\"status\":\"not-started\"},{\"id\":11,\"title\":\"Release SDK packages\",\"description\":\"If ready and approved by user, trigger release for each merged SDK PR.\",\"status\":\"not-started\"},{\"id\":12,\"title\":\"Provide summary & next steps\",\"description\":\"Summarize PRs, release plan status, remaining actions.\",\"status\":\"not-started\"}],\"operation\":\"write\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list\n\nWarning: Large todo list (>10 items). Consider keeping the list focused and actionable.", - "tool_call_id": "call_lKeHpMPYlldJ0diop2bHbdLy" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_rx9loGDfo4sM9hhCc4eWPyAE", - "function": { - "name": "azsdk_run_typespec_validation", - "arguments": "{\"typeSpecProjectRootPath\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "{\"result\":[\"Error: Process must exit before requested information can be determined.\"]}", - "tool_call_id": "call_rx9loGDfo4sM9hhCc4eWPyAE" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_ObOesBYKm6YNDL4v64oQD7Eb", - "function": { - "name": "azsdk_run_typespec_validation", - "arguments": "{\"typeSpecProjectRootPath\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "{\"result\":[\"Running TypeSpecValidation on folder: C:/azure-rest-api-specs/specification/contosowidgetmanager/Contoso.WidgetManager\\r\\n\\r\\nExecuting rule: FolderStructure\\r\\n2025-10-29T15:44:20.217Z simple-git [GitExecutor] [SPAWN] git [ \\u0027rev-parse\\u0027, \\u0027--show-toplevel\\u0027 ]\\r\\n2025-10-29T15:44:20.302Z simple-git [GitExecutor] [HANDLE] retrieving task output complete\\r\\nfolder: C:/azure-rest-api-specs/specification/contosowidgetmanager/Contoso.WidgetManager\\r\\nconfig files: [\\u0022C:/azure-rest-api-specs/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml\\u0022]\\r\\nazure-resource-provider-folder: \\u0022data-plane\\u0022\\r\\nimports:\\r\\n specification\\\\contosowidgetmanager\\r\\n Contoso.Management/employee.tsp: []\\r\\n Contoso.Management/main.tsp: [\\u0022./employee.tsp\\u0022]\\r\\n Contoso.WidgetManager/client.tsp: [\\u0022./main.tsp\\u0022]\\r\\n Contoso.WidgetManager/main.tsp: [\\u0022../Contoso.WidgetManager.Shared\\u0022]\\r\\n Contoso.WidgetManager.Shared/main.tsp: []\\r\\n\\r\\n\\r\\nExecuting rule: NpmPrefix\\r\\n2025-10-29T15:44:20.325Z simple-git [GitExecutor] [SPAWN] git [ \\u0027rev-parse\\u0027, \\u0027--show-toplevel\\u0027 ]\\r\\n2025-10-29T15:44:20.404Z simple-git [GitExecutor] [HANDLE] retrieving task output complete\\r\\nexecFile(\\u0022c:nvm4w\\\\nodejs\\\\node.exe\\u0022, [\\u0022--\\u0022,\\u0022c:\\\\nvm4w\\\\\\\\nodejs\\\\\\\\node_modules\\\\\\\\npm\\\\\\\\bin\\\\\\\\npm-cli.js\\u0022,\\u0022prefix\\u0022])\\r\\nExpected npm prefix: C:/azure-rest-api-specs\\r\\nActual npm prefix: C:/azure-rest-api-specs\\r\\n\\r\\nExecuting rule: EmitAutorest\\r\\nmainTspExists: true\\r\\nemit: [\\u0022@azure-tools/typespec-autorest\\u0022]\\r\\n\\r\\n\\r\\nExecuting rule: FlavorAzure\\r\\n\\u0022@azure-tools/typespec-python\\u0022:\\r\\n flavor: azure\\r\\n\\u0022@azure-tools/typespec-csharp\\u0022:\\r\\n flavor: azure\\r\\n\\u0022@azure-tools/typespec-ts\\u0022:\\r\\n flavor: azure\\r\\n\\u0022@azure-tools/typespec-java\\u0022:\\r\\n flavor: azure\\r\\n\\r\\n\\r\\nExecuting rule: LinterRuleset\\r\\nazure-resource-provider-folder: \\u0022data-plane\\u0022\\r\\nfiles: [\\u0022main.tsp\\u0022,\\u0022client.tsp\\u0022]\\r\\nlinter.extends: [\\u0022@azure-tools/typespec-azure-rulesets/data-plane\\u0022]\\r\\n\\r\\nExecuting rule: Compile\\r\\nexecFile(\\u0022c:nvm4w\\\\nodejs\\\\node.exe\\u0022, [\\u0022--\\u0022,\\u0022c:\\\\nvm4w\\\\\\\\nodejs\\\\\\\\node_modules\\\\\\\\npm\\\\\\\\bin\\\\\\\\npm-cli.js\\u0022,\\u0022exec\\u0022,\\u0022--no\\u0022,\\u0022--\\u0022,\\u0022tsp\\u0022,\\u0022compile\\u0022,\\u0022--list-files\\u0022,\\u0022--warn-as-error\\u0022,\\u0022C:/azure-rest-api-specs/specification/contosowidgetmanager/Contoso.WidgetManager\\u0022])\\r\\nexecFile(\\u0022c:nvm4w\\\\nodejs\\\\node.exe\\u0022, [\\u0022--\\u0022,\\u0022c:\\\\nvm4w\\\\\\\\nodejs\\\\\\\\node_modules\\\\\\\\npm\\\\\\\\bin\\\\\\\\npm-cli.js\\u0022,\\u0022exec\\u0022,\\u0022--no\\u0022,\\u0022--\\u0022,\\u0022tsp\\u0022,\\u0022compile\\u0022,\\u0022--no-emit\\u0022,\\u0022--warn-as-error\\u0022,\\u0022c:\\\\Users\\\\azure-rest-api-specs\\\\\\\\specification\\\\\\\\contosowidgetmanager\\\\\\\\Contoso.WidgetManager\\\\\\\\client.tsp\\u0022])\\r\\n2025-10-29T15:44:28.601Z simple-git [GitExecutor] [SPAWN] git [ \\u0027status\\u0027, \\u0027--porcelain\\u0027, \\u0027-b\\u0027, \\u0027-u\\u0027, \\u0027--null\\u0027, \\u0027--porcelain\\u0027, \\u0027C:/azure-rest-api-specs/specification/contosowidgetmanager\\u0027 ]\\r\\n2025-10-29T15:44:28.987Z simple-git [GitExecutor] [HANDLE] retrieving task output complete\\r\\nTypeSpec compiler v1.2.1\\r\\n\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/widgets.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/examples/Widgets_GetWidgetSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/examples/Widgets_GetWidgetOperationStatusSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/examples/Widgets_CreateOrUpdateWidgetSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/examples/Widgets_DeleteWidgetSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/examples/Widgets_ListWidgetsSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/stable/2022-12-01/widgets.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/stable/2022-12-01/examples/Widgets_GetWidgetSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/stable/2022-12-01/examples/Widgets_GetWidgetOperationStatusSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/stable/2022-12-01/examples/Widgets_CreateOrUpdateWidgetSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/stable/2022-12-01/examples/Widgets_DeleteWidgetSample.json\\u001B[22m\\r\\n \\u001B[2mspecification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/stable/2022-12-01/examples/Widgets_ListWidgetsSample.json\\u001B[22m\\r\\n\\r\\nCompilation completed successfully.\\r\\n\\r\\n- Compiling...\\r\\n\\u001B[32m\\u0393\\u00A3\\u00F6\\u001B[39m Compiling\\r\\n- Running @azure-tools/typespec-autorest...\\r\\n\\u001B[32m\\u0393\\u00A3\\u00F6\\u001B[39m @azure-tools/typespec-autorest \\u001B[32m141ms\\u001B[39m \\u001B[2mspecification/contosowidgetmanager/\\u001B[22m\\r\\n\\r\\nGenerated Swaggers:\\r\\nspecification\\\\contosowidgetmanager\\\\data-plane\\\\Azure.Contoso.WidgetManager\\\\preview\\\\2022-11-01-preview\\\\widgets.json\\r\\nspecification\\\\contosowidgetmanager\\\\data-plane\\\\Azure.Contoso.WidgetManager\\\\stable\\\\2022-12-01\\\\widgets.json\\r\\n\\r\\nOutput folder:\\r\\nspecification\\\\contosowidgetmanager\\\\data-plane\\\\Azure.Contoso.WidgetManager\\r\\n\\r\\nSwaggers matching output folder and filename:\\r\\nspecification\\\\contosowidgetmanager\\\\data-plane\\\\Azure.Contoso.WidgetManager\\\\stable\\\\2022-12-01\\\\widgets.json\\r\\nspecification\\\\contosowidgetmanager\\\\data-plane\\\\Azure.Contoso.WidgetManager\\\\preview\\\\2022-11-01-preview\\\\widgets.json\\r\\n\\r\\nSwaggers excluded via suppressions.yaml:\\r\\n\\r\\n\\r\\nRemaining swaggers:\\r\\nspecification\\\\contosowidgetmanager\\\\data-plane\\\\Azure.Contoso.WidgetManager\\\\stable\\\\2022-12-01\\\\widgets.json\\r\\nspecification\\\\contosowidgetmanager\\\\data-plane\\\\Azure.Contoso.WidgetManager\\\\preview\\\\2022-11-01-preview\\\\widgets.json\\r\\nTypeSpec compiler v1.2.1\\r\\n\\r\\n\\r\\nCompilation completed successfully.\\r\\n\\r\\nRunning git diff on folder C:/azure-rest-api-specs/specification/contosowidgetmanager\\r\\n\\r\\nExecuting rule: Format\\r\\nexecFile(\\u0022c:nvm4w\\\\nodejs\\\\node.exe\\u0022, [\\u0022--\\u0022,\\u0022c:\\\\nvm4w\\\\\\\\nodejs\\\\\\\\node_modules\\\\\\\\npm\\\\\\\\bin\\\\\\\\npm-cli.js\\u0022,\\u0022exec\\u0022,\\u0022--no\\u0022,\\u0022--\\u0022,\\u0022tsp\\u0022,\\u0022format\\u0022,\\u0022../**/*.tsp\\u0022])\\r\\nexecFile(\\u0022c:nvm4w\\\\nodejs\\\\node.exe\\u0022, [\\u0022--\\u0022,\\u0022c:\\\\nvm4w\\\\\\\\nodejs\\\\\\\\node_modules\\\\\\\\npm\\\\\\\\bin\\\\\\\\npm-cli.js\\u0022,\\u0022exec\\u0022,\\u0022--no\\u0022,\\u0022--\\u0022,\\u0022prettier\\u0022,\\u0022--write\\u0022,\\u0022tspconfig.yaml\\u0022])\\r\\n2025-10-29T15:44:33.494Z simple-git [GitExecutor] [SPAWN] git [ \\u0027status\\u0027, \\u0027--porcelain\\u0027, \\u0027-b\\u0027, \\u0027-u\\u0027, \\u0027--null\\u0027, \\u0027--porcelain\\u0027, \\u0027C:/azure-rest-api-specs/specification/contosowidgetmanager\\u0027 ]\\r\\n2025-10-29T15:44:33.706Z simple-git [GitExecutor] [HANDLE] retrieving task output complete\\r\\ntspconfig.yaml 128ms (unchanged)\\r\\nRunning git diff on folder C:/azure-rest-api-specs/specification/contosowidgetmanager\\r\\n\\r\\nExecuting rule: SdkTspConfigValidation\\r\\n[SdkTspConfigValidation]: validation passed.\\r\\n\\r\\n\"]}", - "tool_call_id": "call_ObOesBYKm6YNDL4v64oQD7Eb" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_NMS7hJl4dYld7r1dqiRd8Wvn", - "function": { - "name": "manage_todo_list", - "arguments": "{\"todoList\":[{\"id\":1,\"title\":\"Confirm start of SDK process\",\"description\":\"Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\",\"status\":\"completed\"},{\"id\":2,\"title\":\"Identify TypeSpec project root\",\"description\":\"Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\",\"status\":\"completed\"},{\"id\":3,\"title\":\"Check spec merge status\",\"description\":\"Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\",\"status\":\"completed\"},{\"id\":4,\"title\":\"Run TypeSpec validation\",\"description\":\"If not merged, run validation tool and handle any errors.\",\"status\":\"completed\"},{\"id\":5,\"title\":\"Review & commit changes\",\"description\":\"List modified TypeSpec files, stage, commit, push on non-main branch.\",\"status\":\"in-progress\"},{\"id\":6,\"title\":\"Create or verify spec PR\",\"description\":\"Check if PR exists for current branch; create if missing.\",\"status\":\"not-started\"},{\"id\":7,\"title\":\"Generate SDKs via pipeline\",\"description\":\"Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\",\"status\":\"not-started\"},{\"id\":8,\"title\":\"Create release plan (optional)\",\"description\":\"If user intends to publish, gather required IDs, API version, release type and create plan.\",\"status\":\"not-started\"},{\"id\":9,\"title\":\"Link SDK PRs to release plan\",\"description\":\"Update release plan with language/package names and link PRs.\",\"status\":\"not-started\"},{\"id\":10,\"title\":\"Check release readiness\",\"description\":\"Run package readiness checks for each language.\",\"status\":\"not-started\"},{\"id\":11,\"title\":\"Release SDK packages\",\"description\":\"If ready and approved by user, trigger release for each merged SDK PR.\",\"status\":\"not-started\"},{\"id\":12,\"title\":\"Provide summary & next steps\",\"description\":\"Summarize PRs, release plan status, remaining actions.\",\"status\":\"not-started\"}],\"operation\":\"write\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list\n\nWarning: Large todo list (>10 items). Consider keeping the list focused and actionable.", - "tool_call_id": "call_NMS7hJl4dYld7r1dqiRd8Wvn" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_ZugfNs7d7PbVFhcbcgW4cWlA", - "function": { - "name": "azsdk_get_modified_typespec_projects", - "arguments": "{\"repoRootPath\":\"c:\\\\azure-rest-api-specs\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "{\"result\":[\"specification/contosowidgetmanager/Contoso.Management\",\"specification/contosowidgetmanager/Contoso.WidgetManager\"]}", - "tool_call_id": "call_ZugfNs7d7PbVFhcbcgW4cWlA" - }, - { - "role": "assistant", - "content": "TypeSpec validation passed; generated preview and stable swagger + examples. Modified TypeSpec project directories detected:\n- `specification/contosowidgetmanager/Contoso.Management`\n- `specification/contosowidgetmanager/Contoso.WidgetManager`\n\nCurrent branch: `testing-contoso-sdk-gen` (non-main) so we can commit directly.\n\nPlease review those changes locally (you can expand `git diff` in the editor). Do you want me to stage and commit all changes in both modified project directories and push them? (yes/no)\n\nIf yes, I’ll:\n1. Add all changed files under `specification/contosowidgetmanager/Contoso.*`\n2. Prompt you for a commit message\n3. Push to `testing-contoso-sdk-gen`\n4. Move on to creating/verifying the spec PR\n\nLet me know your choice." - }, - { - "role": "user", - "content": "\nThe current date is October 29, 2025.\nTerminals:\nTerminal: pwsh\n\n# Todo List\n\n- [x] Confirm start of SDK process\n - Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\n- [x] Identify TypeSpec project root\n - Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\n- [x] Check spec merge status\n - Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\n- [x] Run TypeSpec validation\n - If not merged, run validation tool and handle any errors.\n- [-] Review & commit changes\n - List modified TypeSpec files, stage, commit, push on non-main branch.\n- [ ] Create or verify spec PR\n - Check if PR exists for current branch; create if missing.\n- [ ] Generate SDKs via pipeline\n - Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\n- [ ] Create release plan (optional)\n - If user intends to publish, gather required IDs, API version, release type and create plan.\n- [ ] Link SDK PRs to release plan\n - Update release plan with language/package names and link PRs.\n- [ ] Check release readiness\n - Run package readiness checks for each language.\n- [ ] Release SDK packages\n - If ready and approved by user, trigger release for each merged SDK PR.\n- [ ] Provide summary & next steps\n - Summarize PRs, release plan status, remaining actions.\n\n\n\n\nThe user's current file is c:\\azure-rest-api-specs\\.vscode\\mcp.json. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nYou are an agent—keep going until the user's query is completely resolved before ending your turn. ONLY stop if solved or genuinely blocked.\nTake action when possible; the user expects you to do useful work without unnecessary questions.\nAfter any parallel, read-only context gathering, give a concise progress update and what's next.\nAvoid repetition across turns: don't restate unchanged plans or sections (like the todo list) verbatim; provide delta updates or only the parts that changed.\nTool batches: You MUST preface each batch with a one-sentence why/what/outcome preamble.\nProgress cadence: After 3 to 5 tool calls, or when you create/edit > ~3 files in a burst, report progress.\nRequirements coverage: Read the user's ask in full and think carefully. Do not omit a requirement. If something cannot be done with available tools, note why briefly and propose a viable alternative.\nWhen using the insert_edit_into_file tool, avoid repeating existing code, instead use a line comment with \\`...existing code...\\` to represent regions of unchanged code.\nSkip filler acknowledgements like \"Sounds good\" or \"Okay, I will…\". Open with a purposeful one-liner about what you're doing next.\nWhen sharing setup or run steps, present terminal commands in fenced code blocks with the correct language tag. Keep commands copyable and on separate lines.\nAvoid definitive claims about the build or runtime setup unless verified from the provided context (or quick tool checks). If uncertain, state what's known from attachments and proceed with minimal steps you can adapt later.\nWhen you create or edit runnable code, run a test yourself to confirm it works; then share optional fenced commands for more advanced runs.\nFor non-trivial code generation, produce a complete, runnable solution: necessary source files, a tiny runner or test/benchmark harness, a minimal `README.md`, and updated dependency manifests (e.g., `package.json`, `requirements.txt`, `pyproject.toml`). Offer quick \"try it\" commands and optional platform-specific speed-ups when relevant.\nYour goal is to act like a pair programmer: be friendly and helpful. If you can do more, do more. Be proactive with your solutions, think about what the user needs and what they want, and implement it proactively.\n\nBefore starting a task, review and follow the guidance in , , and .\nStart your response with a brief acknowledgement, followed by a concise high-level plan outlining your approach.\nDO NOT state your identity or model name unless the user explicitly asks you to. \nYou MUST use the todo list tool to plan and track your progress. NEVER skip this step, and START with this step whenever the task is multi-step. This is essential for maintaining visibility and proper execution of large tasks. Follow the todoListToolInstructions strictly.\nWhen referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\n\n\n\n\neverything is committed (See above for file contents. You may not need to search or read the file again.)\n" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_tppU34G9J6yknLhLxKBQ6M0j", - "function": { - "name": "manage_todo_list", - "arguments": "{\"todoList\":[{\"id\":1,\"title\":\"Confirm start of SDK process\",\"description\":\"Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\",\"status\":\"completed\"},{\"id\":2,\"title\":\"Identify TypeSpec project root\",\"description\":\"Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\",\"status\":\"completed\"},{\"id\":3,\"title\":\"Check spec merge status\",\"description\":\"Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\",\"status\":\"completed\"},{\"id\":4,\"title\":\"Run TypeSpec validation\",\"description\":\"If not merged, run validation tool and handle any errors.\",\"status\":\"completed\"},{\"id\":5,\"title\":\"Review & commit changes\",\"description\":\"List modified TypeSpec files, stage, commit, push on non-main branch.\",\"status\":\"completed\"},{\"id\":6,\"title\":\"Create or verify spec PR\",\"description\":\"Check if PR exists for current branch; create if missing.\",\"status\":\"in-progress\"},{\"id\":7,\"title\":\"Generate SDKs via pipeline\",\"description\":\"Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\",\"status\":\"not-started\"},{\"id\":8,\"title\":\"Create release plan (optional)\",\"description\":\"If user intends to publish, gather required IDs, API version, release type and create plan.\",\"status\":\"not-started\"},{\"id\":9,\"title\":\"Link SDK PRs to release plan\",\"description\":\"Update release plan with language/package names and link PRs.\",\"status\":\"not-started\"},{\"id\":10,\"title\":\"Check release readiness\",\"description\":\"Run package readiness checks for each language.\",\"status\":\"not-started\"},{\"id\":11,\"title\":\"Release SDK packages\",\"description\":\"If ready and approved by user, trigger release for each merged SDK PR.\",\"status\":\"not-started\"},{\"id\":12,\"title\":\"Provide summary & next steps\",\"description\":\"Summarize PRs, release plan status, remaining actions.\",\"status\":\"not-started\"}],\"operation\":\"write\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list\n\nWarning: Large todo list (>10 items). Consider keeping the list focused and actionable.", - "tool_call_id": "call_tppU34G9J6yknLhLxKBQ6M0j" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_ZJyf5wvHyGGRp7p3QWDKG6EO", - "function": { - "name": "azsdk_get_pull_request_link_for_current_branch", - "arguments": "{\"repoPath\":\"c:\\\\azure-rest-api-specs\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "{\"result\":\"Pull request found: https://github.com/Azure/azure-rest-api-specs/pull/38387\\u0022{\\\\u0022pullRequestNumber\\\\u0022:38387,\\\\u0022Author\\\\u0022:null,\\\\u0022Labels\\\\u0022:[{\\\\u0022Id\\\\u0022:5353717489,\\\\u0022Url\\\\u0022:\\\\u0022https://api.github.com/repos/Azure/azure-rest-api-specs/labels/TypeSpec\\\\u0022,\\\\u0022Name\\\\u0022:\\\\u0022TypeSpec\\\\u0022,\\\\u0022NodeId\\\\u0022:\\\\u0022LA_kwDOAlSEjc8AAAABPxs-8Q\\\\u0022,\\\\u0022Color\\\\u0022:\\\\u00220052CC\\\\u0022,\\\\u0022Description\\\\u0022:\\\\u0022Authored with TypeSpec\\\\u0022,\\\\u0022Default\\\\u0022:false},{\\\\u0022Id\\\\u0022:8716331499,\\\\u0022Url\\\\u0022:\\\\u0022https://api.github.com/repos/Azure/azure-rest-api-specs/labels/Created%20By%20Copilot\\\\u0022,\\\\u0022Name\\\\u0022:\\\\u0022Created By Copilot\\\\u0022,\\\\u0022NodeId\\\\u0022:\\\\u0022LA_kwDOAlSEjc8AAAACB4ip6w\\\\u0022,\\\\u0022Color\\\\u0022:\\\\u0022bfdadc\\\\u0022,\\\\u0022Description\\\\u0022:\\\\u0022This indicates that PR was created by GitHub copilot agent\\\\u0022,\\\\u0022Default\\\\u0022:false}],\\\\u0022Status\\\\u0022:\\\\u0022open\\\\u0022,\\\\u0022AssignedTo\\\\u0022:\\\\u0022\\\\u0022,\\\\u0022Url\\\\u0022:\\\\u0022https://github.com/Azure/azure-rest-api-specs/pull/38387\\\\u0022,\\\\u0022IsMerged\\\\u0022:false,\\\\u0022IsMergeable\\\\u0022:true,\\\\u0022Checks\\\\u0022:[\\\\u0022Name: SDK Validation - Python, Status: completed, Output: [1 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490980), Conclusion: failure, Link: https://github.com/Azure/azure-rest-api-specs/runs/53495233105\\\\u0022,\\\\u0022Name: SDK Validation - .NET, Status: completed, Output: [1 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490977), Conclusion: failure, Link: https://github.com/Azure/azure-rest-api-specs/runs/53495223866\\\\u0022,\\\\u0022Name: SDK Validation - Go, Status: completed, Output: [1 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490978), Conclusion: failure, Link: https://github.com/Azure/azure-rest-api-specs/runs/53495170336\\\\u0022,\\\\u0022Name: SDK Validation - Java, Status: completed, Output: [1 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490976), Conclusion: failure, Link: https://github.com/Azure/azure-rest-api-specs/runs/53495164506\\\\u0022,\\\\u0022Name: SDK Validation - JS, Status: completed, Output: [1 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490979), Conclusion: failure, Link: https://github.com/Azure/azure-rest-api-specs/runs/53495162016\\\\u0022,\\\\u0022Name: spec - api-doc-preview, Status: completed, Output: [0 errors / 0 warnings](https://dev.azure.com/azure-sdk/590cfd2a-581c-4dcb-a12e-6568ce786175/_build/results?buildId=5490981), Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/runs/53495002753\\\\u0022,\\\\u0022Name: TypeSpec APIView, Status: completed, Output: [3 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490975), Conclusion: failure, Link: https://github.com/Azure/azure-rest-api-specs/runs/53494978185\\\\u0022,\\\\u0022Name: Swagger PrettierCheck, Status: completed, Output: [0 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490973), Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/runs/53494946473\\\\u0022,\\\\u0022Name: Swagger APIView, Status: completed, Output: [0 errors / 1 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490974), Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/runs/53494941487\\\\u0022,\\\\u0022Name: license/cla, Status: completed, Output: This check verifies that the author has agreed to a CLA with Microsoft., Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/runs/53494919968\\\\u0022,\\\\u0022Name: public.rest-api-specs, Status: completed, Output: [0 errors / 0 warnings](https://dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_build/results?buildId=5490972), Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/runs/53494746780\\\\u0022,\\\\u0022Name: ARM Auto SignOff, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237048/job/53494737696\\\\u0022,\\\\u0022Name: Set Breaking Change(Cross-Version) Status / Breaking Change(Cross-Version) - Set Status, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237016/job/53494726179\\\\u0022,\\\\u0022Name: Set Avocado Status / ${{ inputs.required_check_name }} - Set Status, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237176/job/53494725897\\\\u0022,\\\\u0022Name: Set BreakingChange Status / ${{ inputs.required_check_name }} - Set Status, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237251/job/53494725402\\\\u0022,\\\\u0022Name: Set Breaking Change(Cross-Version) Status / ${{ inputs.required_check_name }} - Set Status, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237258/job/53494724810\\\\u0022,\\\\u0022Name: Set SemanticValidation Status / ${{ inputs.required_check_name }} - Set Status, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237180/job/53494724759\\\\u0022,\\\\u0022Name: Set LintDiff Status / ${{ inputs.required_check_name }} - Set Status, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237182/job/53494724713\\\\u0022,\\\\u0022Name: Set ModelValidation Status / ${{ inputs.required_check_name }} - Set Status, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237168/job/53494724586\\\\u0022,\\\\u0022Name: tsp-client / test (windows, 22), Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237134/job/53494724456\\\\u0022,\\\\u0022Name: tsp-client / test (ubuntu, 20), Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237134/job/53494724438\\\\u0022,\\\\u0022Name: Set LintDiff Status / Swagger LintDiff - Set Status, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237120/job/53494724363\\\\u0022,\\\\u0022Name: Swagger BreakingChange - Analyze Code, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237087/job/53494724354\\\\u0022,\\\\u0022Name: Set SemanticValidation Status / Swagger SemanticValidation - Set Status, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237095/job/53494724308\\\\u0022,\\\\u0022Name: Set ModelValidation Status / Swagger ModelValidation - Set Status, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237099/job/53494724302\\\\u0022,\\\\u0022Name: TypeSpec Migration Validation, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237080/job/53494724244\\\\u0022,\\\\u0022Name: Swagger SemanticValidation - Analyze Code, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237083/job/53494724212\\\\u0022,\\\\u0022Name: Swagger LintDiff - Analyze Code, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237076/job/53494724113\\\\u0022,\\\\u0022Name: TypeSpec Requirement, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237104/job/53494724061\\\\u0022,\\\\u0022Name: Set Avocado Status / Swagger Avocado - Set Status, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237069/job/53494724039\\\\u0022,\\\\u0022Name: SpellCheck, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237046/job/53494723978\\\\u0022,\\\\u0022Name: TypeSpec Validation, Status: completed, Output: , Conclusion: failure, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237089/job/53494723968\\\\u0022,\\\\u0022Name: Set BreakingChange Status / Swagger BreakingChange - Set Status, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237047/job/53494723944\\\\u0022,\\\\u0022Name: Protected Files, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237037/job/53494723904\\\\u0022,\\\\u0022Name: Swagger Avocado - Analyze Code, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752236975/job/53494723898\\\\u0022,\\\\u0022Name: Sdk Suppressions, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237041/job/53494723884\\\\u0022,\\\\u0022Name: Summarize PR Impact, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237034/job/53494723845\\\\u0022,\\\\u0022Name: Swagger ModelValidation - Analyze Code, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237052/job/53494723837\\\\u0022,\\\\u0022Name: Summarize Checks, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237024/job/53494723775\\\\u0022,\\\\u0022Name: TypeSpec Migration Validation, Status: completed, Output: , Conclusion: skipped, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752236986/job/53494723765\\\\u0022,\\\\u0022Name: Breaking Change(Cross-Version) - Analyze Code, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237002/job/53494723706\\\\u0022,\\\\u0022Name: Summarize Checks, Status: completed, Output: , Conclusion: success, Link: https://github.com/Azure/azure-rest-api-specs/actions/runs/18752236892/job/53494723431\\\\u0022,\\\\u0022Total checks found: 42\\\\u0022,\\\\u0022Failed checks: 7\\\\u0022,\\\\u0022Pending required checks to merge the PR: 7.\\\\u0022],\\\\u0022Comments\\\\u0022:[\\\\u0022Comment by github-actions[bot]: \\\\\\\\u003Ch2\\\\\\\\u003ENext Steps to Merge\\\\\\\\u003C/h2\\\\\\\\u003E\\\\\\\\u231B Please wait. Next steps to merge this PR are being evaluated by automation. \\\\\\\\u231B\\\\\\\\u003Cbr /\\\\\\\\u003E\\\\\\\\u003Cbr /\\\\\\\\u003EComment generated by \\\\\\\\u003Ca href=\\\\\\\\u0022https://github.com/Azure/azure-rest-api-specs/actions/runs/18752237024\\\\\\\\u0022\\\\\\\\u003Esummarize-checks\\\\\\\\u003C/a\\\\\\\\u003E workflow run.\\\\\\\\n\\\\\\\\u003C!-- NextStepsToMerge --\\\\\\\\u003E\\\\u0022,\\\\u0022Comment by github-actions[bot]: \\\\\\\\u003Ch2\\\\\\\\u003ENext Steps to Merge\\\\\\\\u003C/h2\\\\\\\\u003ENext steps that must be taken to merge this PR: \\\\\\\\u003Cbr/\\\\\\\\u003E\\\\\\\\u003Cul\\\\\\\\u003E\\\\\\\\u003Cli\\\\\\\\u003E\\\\\\\\u274C The required check named \\\\\\\\u003Ccode\\\\\\\\u003ETypeSpec Validation\\\\\\\\u003C/code\\\\\\\\u003E has failed. Refer to the check in the PR\\\\\\\\u0027s \\\\\\\\u0027Checks\\\\\\\\u0027 tab for details on how to fix it and consult the \\\\\\\\u003Ca href=\\\\\\\\u0022https://aka.ms/ci-fix\\\\\\\\u0022\\\\\\\\u003Eaka.ms/ci-fix\\\\\\\\u003C/a\\\\\\\\u003E guide\\\\\\\\u003C/li\\\\\\\\u003E\\\\\\\\u003C/ul\\\\\\\\u003E\\\\\\\\u003Cbr /\\\\\\\\u003E\\\\\\\\u003Cbr /\\\\\\\\u003EComment generated by \\\\\\\\u003Ca href=\\\\\\\\u0022https://github.com/Azure/azure-rest-api-specs/actions/runs/18752402670\\\\\\\\u0022\\\\\\\\u003Esummarize-checks\\\\\\\\u003C/a\\\\\\\\u003E workflow run.\\\\\\\\n\\\\\\\\u003C!-- NextStepsToMerge --\\\\\\\\u003E\\\\u0022],\\\\u0022ApiViews\\\\u0022:[]}\\u0022\\r\\n\"}", - "tool_call_id": "call_ZJyf5wvHyGGRp7p3QWDKG6EO" - }, - { - "role": "assistant", - "content": "Pull request detected: https://github.com/Azure/azure-rest-api-specs/pull/38387 (open, number 38387). Local TypeSpec validation succeeded, so we can move to SDK generation.\n\nData-plane project versions:\n- Preview: 2022-11-01-preview → release type beta\n- Stable: 2022-12-01 → release type stable\n\nSDK languages for data-plane: Python, .NET, JavaScript, Java.\n\nNext step is to trigger SDK generation pipeline runs (one per language per API version you want). Each run will take ~15–20 minutes; we can:\nA) Generate only the latest stable (2022-12-01) first\nB) Generate both stable and preview\nC) Generate only preview (beta) for early testing\n\nPlease choose (A/B/C) and confirm you want to start SDK generation now. If you pick B, we’ll run 8 pipeline jobs (4 languages × 2 versions). Let me know your choice and I’ll kick them off." - }, - { - "role": "user", - "content": "\nThe current date is October 29, 2025.\nTerminals:\nTerminal: pwsh\n\n# Todo List\n\n- [x] Confirm start of SDK process\n - Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\n- [x] Identify TypeSpec project root\n - Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\n- [x] Check spec merge status\n - Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\n- [x] Run TypeSpec validation\n - If not merged, run validation tool and handle any errors.\n- [x] Review & commit changes\n - List modified TypeSpec files, stage, commit, push on non-main branch.\n- [-] Create or verify spec PR\n - Check if PR exists for current branch; create if missing.\n- [ ] Generate SDKs via pipeline\n - Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\n- [ ] Create release plan (optional)\n - If user intends to publish, gather required IDs, API version, release type and create plan.\n- [ ] Link SDK PRs to release plan\n - Update release plan with language/package names and link PRs.\n- [ ] Check release readiness\n - Run package readiness checks for each language.\n- [ ] Release SDK packages\n - If ready and approved by user, trigger release for each merged SDK PR.\n- [ ] Provide summary & next steps\n - Summarize PRs, release plan status, remaining actions.\n\n\n\n\nThe user's current file is c:\\\\azure-rest-api-specs\\.vscode\\mcp.json. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nYou are an agent—keep going until the user's query is completely resolved before ending your turn. ONLY stop if solved or genuinely blocked.\nTake action when possible; the user expects you to do useful work without unnecessary questions.\nAfter any parallel, read-only context gathering, give a concise progress update and what's next.\nAvoid repetition across turns: don't restate unchanged plans or sections (like the todo list) verbatim; provide delta updates or only the parts that changed.\nTool batches: You MUST preface each batch with a one-sentence why/what/outcome preamble.\nProgress cadence: After 3 to 5 tool calls, or when you create/edit > ~3 files in a burst, report progress.\nRequirements coverage: Read the user's ask in full and think carefully. Do not omit a requirement. If something cannot be done with available tools, note why briefly and propose a viable alternative.\nWhen using the insert_edit_into_file tool, avoid repeating existing code, instead use a line comment with \\`...existing code...\\` to represent regions of unchanged code.\nSkip filler acknowledgements like \"Sounds good\" or \"Okay, I will…\". Open with a purposeful one-liner about what you're doing next.\nWhen sharing setup or run steps, present terminal commands in fenced code blocks with the correct language tag. Keep commands copyable and on separate lines.\nAvoid definitive claims about the build or runtime setup unless verified from the provided context (or quick tool checks). If uncertain, state what's known from attachments and proceed with minimal steps you can adapt later.\nWhen you create or edit runnable code, run a test yourself to confirm it works; then share optional fenced commands for more advanced runs.\nFor non-trivial code generation, produce a complete, runnable solution: necessary source files, a tiny runner or test/benchmark harness, a minimal `README.md`, and updated dependency manifests (e.g., `package.json`, `requirements.txt`, `pyproject.toml`). Offer quick \"try it\" commands and optional platform-specific speed-ups when relevant.\nYour goal is to act like a pair programmer: be friendly and helpful. If you can do more, do more. Be proactive with your solutions, think about what the user needs and what they want, and implement it proactively.\n\nBefore starting a task, review and follow the guidance in , , and .\nStart your response with a brief acknowledgement, followed by a concise high-level plan outlining your approach.\nDO NOT state your identity or model name unless the user explicitly asks you to. \nYou MUST use the todo list tool to plan and track your progress. NEVER skip this step, and START with this step whenever the task is multi-step. This is essential for maintaining visibility and proper execution of large tasks. Follow the todoListToolInstructions strictly.\nWhen referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\n\n\n\n\nC (See above for file contents. You may not need to search or read the file again.)\n" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_qUh4SOUTHBBaOMqPa0l4vjzB", - "function": { - "name": "manage_todo_list", - "arguments": "{\"todoList\":[{\"id\":1,\"title\":\"Confirm start of SDK process\",\"description\":\"Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\",\"status\":\"completed\"},{\"id\":2,\"title\":\"Identify TypeSpec project root\",\"description\":\"Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\",\"status\":\"completed\"},{\"id\":3,\"title\":\"Check spec merge status\",\"description\":\"Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\",\"status\":\"completed\"},{\"id\":4,\"title\":\"Run TypeSpec validation\",\"description\":\"If not merged, run validation tool and handle any errors.\",\"status\":\"completed\"},{\"id\":5,\"title\":\"Review & commit changes\",\"description\":\"List modified TypeSpec files, stage, commit, push on non-main branch.\",\"status\":\"completed\"},{\"id\":6,\"title\":\"Create or verify spec PR\",\"description\":\"Check if PR exists for current branch; create if missing.\",\"status\":\"completed\"},{\"id\":7,\"title\":\"Generate SDKs via pipeline\",\"description\":\"Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\",\"status\":\"in-progress\"},{\"id\":8,\"title\":\"Create release plan (optional)\",\"description\":\"If user intends to publish, gather required IDs, API version, release type and create plan.\",\"status\":\"not-started\"},{\"id\":9,\"title\":\"Link SDK PRs to release plan\",\"description\":\"Update release plan with language/package names and link PRs.\",\"status\":\"not-started\"},{\"id\":10,\"title\":\"Check release readiness\",\"description\":\"Run package readiness checks for each language.\",\"status\":\"not-started\"},{\"id\":11,\"title\":\"Release SDK packages\",\"description\":\"If ready and approved by user, trigger release for each merged SDK PR.\",\"status\":\"not-started\"},{\"id\":12,\"title\":\"Provide summary & next steps\",\"description\":\"Summarize PRs, release plan status, remaining actions.\",\"status\":\"not-started\"}],\"operation\":\"write\"}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "Successfully wrote todo list\n\nWarning: Large todo list (>10 items). Consider keeping the list focused and actionable.", - "tool_call_id": "call_qUh4SOUTHBBaOMqPa0l4vjzB" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_NBOq7xPkJ4VzTyj7pV7dlDNe", - "function": { - "name": "azsdk_run_generate_sdk", - "arguments": "{\"typespecProjectRoot\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\",\"apiVersion\":\"2022-11-01-preview\",\"sdkReleaseType\":\"beta\",\"language\":\"Python\",\"pullRequestNumber\":38387}" - }, - "type": "function" - }, - { - "id": "call_vFVpuz6a3OFQ9RJQosP1RMqc", - "function": { - "name": "azsdk_run_generate_sdk", - "arguments": "{\"typespecProjectRoot\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\",\"apiVersion\":\"2022-11-01-preview\",\"sdkReleaseType\":\"beta\",\"language\":\".NET\",\"pullRequestNumber\":38387}" - }, - "type": "function" - }, - { - "id": "call_8Vt0eof9NmHx4pm13ofJxBlP", - "function": { - "name": "azsdk_run_generate_sdk", - "arguments": "{\"typespecProjectRoot\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\",\"apiVersion\":\"2022-11-01-preview\",\"sdkReleaseType\":\"beta\",\"language\":\"JavaScript\",\"pullRequestNumber\":38387}" - }, - "type": "function" - }, - { - "id": "call_fzufuktnPmeHUyzLZmTaoPIP", - "function": { - "name": "azsdk_run_generate_sdk", - "arguments": "{\"typespecProjectRoot\":\"c:\\\\azure-rest-api-specs\\\\specification\\\\contosowidgetmanager\\\\Contoso.WidgetManager\",\"apiVersion\":\"2022-11-01-preview\",\"sdkReleaseType\":\"beta\",\"language\":\"Java\",\"pullRequestNumber\":38387}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "{\"status\":\"Success\",\"details\":[\"Azure DevOps pipeline https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513110 has been initiated to generate the SDK. Build ID is 5513110. Once the pipeline job completes, an SDK pull request for Python will be created.\"]}", - "tool_call_id": "call_NBOq7xPkJ4VzTyj7pV7dlDNe" - }, - { - "role": "tool", - "content": "{\"status\":\"Success\",\"details\":[\"Azure DevOps pipeline https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513111 has been initiated to generate the SDK. Build ID is 5513111. Once the pipeline job completes, an SDK pull request for .NET will be created.\"]}", - "tool_call_id": "call_vFVpuz6a3OFQ9RJQosP1RMqc" - }, - { - "role": "tool", - "content": "{\"status\":\"Success\",\"details\":[\"Azure DevOps pipeline https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513113 has been initiated to generate the SDK. Build ID is 5513113. Once the pipeline job completes, an SDK pull request for JavaScript will be created.\"]}", - "tool_call_id": "call_8Vt0eof9NmHx4pm13ofJxBlP" - }, - { - "role": "tool", - "content": "{\"status\":\"Success\",\"details\":[\"Azure DevOps pipeline https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513114 has been initiated to generate the SDK. Build ID is 5513114. Once the pipeline job completes, an SDK pull request for Java will be created.\"]}", - "tool_call_id": "call_fzufuktnPmeHUyzLZmTaoPIP" - }, - { - "role": "assistant", - "content": "Started preview (beta) SDK generation for API version 2022-11-01-preview:\n\nBuilds in progress:\n- Python: Build 5513110\n- .NET: Build 5513111\n- JavaScript: Build 5513113\n- Java: Build 5513114\n\nEach pipeline will create a corresponding SDK PR when finished. Typical duration: ~15–20 minutes. I can poll statuses when you're ready.\n\nNext options:\n1. Wait and later ask me to check pipeline status\n2. Also kick off stable version generation (2022-12-01) now\n3. Cancel or adjust scope (e.g., skip a language)\n\nLet me know what you’d like next (e.g., “check status”, “generate stable”, or “pause”)." - }, - { - "role": "user", - "content": "\nThe current date is October 29, 2025.\nTerminals:\nTerminal: pwsh\n\n# Todo List\n\n- [x] Confirm start of SDK process\n - Ask user if they want to begin the TypeSpec → SDK generation workflow now (yes/no).\n- [x] Identify TypeSpec project root\n - Use open file `main.tsp` parent directory as project root; verify presence of `tspconfig.yaml`.\n- [x] Check spec merge status\n - Ask if TypeSpec spec is already merged in Azure/azure-rest-api-specs main branch.\n- [x] Run TypeSpec validation\n - If not merged, run validation tool and handle any errors.\n- [x] Review & commit changes\n - List modified TypeSpec files, stage, commit, push on non-main branch.\n- [x] Create or verify spec PR\n - Check if PR exists for current branch; create if missing.\n- [-] Generate SDKs via pipeline\n - Trigger SDK generation pipeline for appropriate languages (Mgmt or Data Plane) and monitor status.\n- [ ] Create release plan (optional)\n - If user intends to publish, gather required IDs, API version, release type and create plan.\n- [ ] Link SDK PRs to release plan\n - Update release plan with language/package names and link PRs.\n- [ ] Check release readiness\n - Run package readiness checks for each language.\n- [ ] Release SDK packages\n - If ready and approved by user, trigger release for each merged SDK PR.\n- [ ] Provide summary & next steps\n - Summarize PRs, release plan status, remaining actions.\n\n\n\n\nThe user's current file is c:\\\\azure-rest-api-specs\\.vscode\\mcp.json. \n\n\nBelow is the information about the current repository. You can use this information when you need to calculate diffs or compare changes with the default branch.\nRepository name: azure-rest-api-specs\nOwner: jeo02\nCurrent branch: testing-contoso-sdk-gen\n\n\n\nYou are an agent—keep going until the user's query is completely resolved before ending your turn. ONLY stop if solved or genuinely blocked.\nTake action when possible; the user expects you to do useful work without unnecessary questions.\nAfter any parallel, read-only context gathering, give a concise progress update and what's next.\nAvoid repetition across turns: don't restate unchanged plans or sections (like the todo list) verbatim; provide delta updates or only the parts that changed.\nTool batches: You MUST preface each batch with a one-sentence why/what/outcome preamble.\nProgress cadence: After 3 to 5 tool calls, or when you create/edit > ~3 files in a burst, report progress.\nRequirements coverage: Read the user's ask in full and think carefully. Do not omit a requirement. If something cannot be done with available tools, note why briefly and propose a viable alternative.\nWhen using the insert_edit_into_file tool, avoid repeating existing code, instead use a line comment with \\`...existing code...\\` to represent regions of unchanged code.\nSkip filler acknowledgements like \"Sounds good\" or \"Okay, I will…\". Open with a purposeful one-liner about what you're doing next.\nWhen sharing setup or run steps, present terminal commands in fenced code blocks with the correct language tag. Keep commands copyable and on separate lines.\nAvoid definitive claims about the build or runtime setup unless verified from the provided context (or quick tool checks). If uncertain, state what's known from attachments and proceed with minimal steps you can adapt later.\nWhen you create or edit runnable code, run a test yourself to confirm it works; then share optional fenced commands for more advanced runs.\nFor non-trivial code generation, produce a complete, runnable solution: necessary source files, a tiny runner or test/benchmark harness, a minimal `README.md`, and updated dependency manifests (e.g., `package.json`, `requirements.txt`, `pyproject.toml`). Offer quick \"try it\" commands and optional platform-specific speed-ups when relevant.\nYour goal is to act like a pair programmer: be friendly and helpful. If you can do more, do more. Be proactive with your solutions, think about what the user needs and what they want, and implement it proactively.\n\nBefore starting a task, review and follow the guidance in , , and .\nStart your response with a brief acknowledgement, followed by a concise high-level plan outlining your approach.\nDO NOT state your identity or model name unless the user explicitly asks you to. \nYou MUST use the todo list tool to plan and track your progress. NEVER skip this step, and START with this step whenever the task is multi-step. This is essential for maintaining visibility and proper execution of large tasks. Follow the todoListToolInstructions strictly.\nWhen referring to a filename or symbol in the user's workspace, wrap it in backticks.\n\n\n\n\n\nCheck the status of the four sdks that were generated. Only check each pipeline once. (See above for file contents. You may not need to search or read the file again.)\n" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_gw9oDtMgs8t6sPncktLBj7qf", - "function": { - "name": "azsdk_get_pipeline_status", - "arguments": "{\"buildId\":5513110}" - }, - "type": "function" - }, - { - "id": "call_OyjZvuHsdkjCaEVUBxpQs8xN", - "function": { - "name": "azsdk_get_pipeline_status", - "arguments": "{\"buildId\":5513111}" - }, - "type": "function" - }, - { - "id": "call_VxtN4PiaKFrniJGWdvfaS7o9", - "function": { - "name": "azsdk_get_pipeline_status", - "arguments": "{\"buildId\":5513113}" - }, - "type": "function" - }, - { - "id": "call_TKMYWtPFUJoijMz4pUkaiiTS", - "function": { - "name": "azsdk_get_pipeline_status", - "arguments": "{\"buildId\":5513114}" - }, - "type": "function" - } - ] - }, - { - "role": "tool", - "content": "{\"message\":\"Pipeline run link: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513110\",\"result\":\"NotStarted\"}", - "tool_call_id": "call_gw9oDtMgs8t6sPncktLBj7qf" - }, - { - "role": "tool", - "content": "{\"message\":\"Pipeline run link: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513111\",\"result\":\"NotStarted\"}", - "tool_call_id": "call_OyjZvuHsdkjCaEVUBxpQs8xN" - }, - { - "role": "tool", - "content": "{\"message\":\"Pipeline run link: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513113\",\"result\":\"NotStarted\"}", - "tool_call_id": "call_VxtN4PiaKFrniJGWdvfaS7o9" - }, - { - "role": "tool", - "content": "{\"message\":\"Pipeline run link: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5513114\",\"result\":\"NotStarted\"}", - "tool_call_id": "call_TKMYWtPFUJoijMz4pUkaiiTS" - }, - { - "role": "assistant", - "content": "Status check: all four preview SDK generation pipelines are still in NotStarted state (queued):\n\n- Python: 5513110\n- .NET: 5513111\n- JavaScript: 5513113\n- Java: 5513114\n\nThey haven't begun execution yet. Likely waiting for an agent; this can happen if the pool is busy.\n\nNext options:\n1. Re-check in a few minutes (say: “check again”)\n2. Start stable version generation in parallel while these queue (say: “generate stable”)\n3. Cancel preview run (not recommended unless misconfigured)\n4. Proceed to release plan prep while we wait (if you intend to publish)\n\nWhat would you like to do next?" - } -] \ No newline at end of file