Skip to content

Commit 7511fa4

Browse files
helen229helen229
authored
Refresh Azure.Sdk.Tools.Mock handler coverage to match live MCP tool list (#15852) (#15854)
* Add MCP tool-coverage drift check for Azure.Sdk.Tools.Mock (#15852) - New eng/scripts/Get-McpToolInventory.ps1 boots the live Azure.Sdk.Tools.Cli MCP server (via 'azsdk list -o json'), enumerates the IMockToolHandler implementations under Azure.Sdk.Tools.Mock, and reports the diff in three buckets: both / live-only / mock-only. - Cross-references mock-tier eval YAMLs under tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals/ when present; gracefully no-ops when that folder hasn't landed yet (PR #15811). - '-CheckOnly' exits non-zero on (a) any stale handler that no longer maps to a live tool, or (b) any tool referenced by a mock-tier eval without a handler -- intended for the CI job tracked in #15829. - Documents the drift workflow in Azure.Sdk.Tools.Mock/README.md so a contributor flagged by the script knows how to add a handler. No stale handlers detected against the current live tool set. * Add mock handlers for remaining live MCP tools; drop eval scanning from inventory script (#15852) - 13 new handler files covering 63 live tools that previously fell back to the default response (APIView, Codeowners, EngSys, GitHub, Package, Pipeline, ReleasePlan, TypeSpec, Verify, Core, Example). - Get-McpToolInventory.ps1: pure live-vs-mock parity (removes Vally eval cross-reference); -CheckOnly fails if either bucket is non-empty. - README: updated sync workflow to reflect parity-only check. * Simplify Get-McpToolInventory.ps1: no parameters, always exits non-zero on drift (#15852) * Fix 3 release-plan handler response types to match live tools (#15852) Addresses Copilot review on PR #15854: - azsdk_get_kpi_attestation_status: ReleaseWorkflowResponse -> ReleasePlanListResponse - azsdk_get_service_details_by_typespec_path: ReleaseWorkflowResponse -> ProductInfoResponse - azsdk_update_language_exclusion_justification: ReleaseWorkflowResponse -> DefaultCommandResponse * Drop Get-McpToolInventory.ps1 (#15852) Per review discussion: the script only checked that an IMockToolHandler exists with the right ToolName; it could not detect handlers that exist but just return the placeholder DefaultCommandResponse. That blind spot makes the script of limited value. A unit test in Cli.Tests is a better fit for actual drift enforcement and is tracked as a follow-up. README updated to drop the script reference. * Update Mock README: drop reference to removed inventory script (#15852)
1 parent 19b2e07 commit 7511fa4

12 files changed

Lines changed: 1053 additions & 0 deletions

File tree

‎tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/APIView/ApiViewHandlers.cs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Sdk.Tools.Cli.Models;
5+
using Azure.Sdk.Tools.Cli.Models.Responses;
6+
7+
namespace Azure.Sdk.Tools.Mock.Handlers.APIView;
8+
9+
/// <summary>
10+
/// Mock handler for azsdk_apiview_get_comments. Returns a small fixed comment payload so
11+
/// callers can exercise the "consume APIView feedback" path deterministically.
12+
/// </summary>
13+
public class ApiViewGetCommentsHandler : IMockToolHandler
14+
{
15+
public string ToolName => "azsdk_apiview_get_comments";
16+
17+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new APIViewResponse
18+
{
19+
Message = "Retrieved APIView comments",
20+
Language = arguments?.GetValueOrDefault("language")?.ToString() ?? ".NET",
21+
PackageName = arguments?.GetValueOrDefault("packageName")?.ToString() ?? "Azure.Template.Contoso",
22+
Result = new[]
23+
{
24+
new
25+
{
26+
id = "comment-1",
27+
line = 42,
28+
text = "Consider renaming this property for clarity.",
29+
author = "reviewer@microsoft.com",
30+
resolved = false
31+
}
32+
}
33+
};
34+
}
35+
36+
/// <summary>
37+
/// Mock handler for azsdk_apiview_get_review_url. Returns a deterministic review URL for the
38+
/// requested package + language.
39+
/// </summary>
40+
public class ApiViewGetReviewUrlHandler : IMockToolHandler
41+
{
42+
public string ToolName => "azsdk_apiview_get_review_url";
43+
44+
public CommandResponse Handle(Dictionary<string, object?>? arguments)
45+
{
46+
var language = arguments?.GetValueOrDefault("language")?.ToString() ?? "dotnet";
47+
var package = arguments?.GetValueOrDefault("packageName")?.ToString() ?? "Azure.Template.Contoso";
48+
return new APIViewResponse
49+
{
50+
Message = "APIView URL resolved",
51+
Language = language,
52+
PackageName = package,
53+
Result = $"https://apiview.dev/Assemblies/Review/mock-{language}-{package}".ToLowerInvariant()
54+
};
55+
}
56+
}
57+
58+
/// <summary>
59+
/// Mock handler for azsdk_apiview_request_copilot_review. Returns a deterministic job ID
60+
/// callers can poll with azsdk_apiview_get_copilot_review.
61+
/// </summary>
62+
public class ApiViewRequestCopilotReviewHandler : IMockToolHandler
63+
{
64+
public string ToolName => "azsdk_apiview_request_copilot_review";
65+
66+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new APIViewResponse
67+
{
68+
Message = "Copilot review submitted",
69+
Language = arguments?.GetValueOrDefault("language")?.ToString() ?? ".NET",
70+
PackageName = arguments?.GetValueOrDefault("packageName")?.ToString() ?? "Azure.Template.Contoso",
71+
Result = "mock-copilot-job-00000001"
72+
};
73+
}
74+
75+
/// <summary>
76+
/// Mock handler for azsdk_apiview_get_copilot_review. Returns a "completed" review with a
77+
/// single sample comment.
78+
/// </summary>
79+
public class ApiViewGetCopilotReviewHandler : IMockToolHandler
80+
{
81+
public string ToolName => "azsdk_apiview_get_copilot_review";
82+
83+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new APIViewResponse
84+
{
85+
Message = "Copilot review complete",
86+
Result = new
87+
{
88+
jobId = arguments?.GetValueOrDefault("jobId")?.ToString() ?? "mock-copilot-job-00000001",
89+
status = "Completed",
90+
comments = new[]
91+
{
92+
new { line = 24, severity = "info", text = "Mock Copilot suggestion: tighten the parameter type." }
93+
}
94+
}
95+
};
96+
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Config/ConfigHandlers.cs‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Sdk.Tools.Cli.Models;
5+
using Azure.Sdk.Tools.Cli.Models.Responses;
6+
using Azure.Sdk.Tools.Cli.Models.Responses.Codeowners;
7+
8+
namespace Azure.Sdk.Tools.Mock.Handlers.Config;
9+
10+
/// <summary>Mock handler for azsdk_check_service_label.</summary>
11+
public class CheckServiceLabelHandler : IMockToolHandler
12+
{
13+
public string ToolName => "azsdk_check_service_label";
14+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new ServiceLabelResponse
15+
{
16+
Label = arguments?.GetValueOrDefault("label")?.ToString() ?? "Contoso.WidgetManager",
17+
Status = "Exists"
18+
};
19+
}
20+
21+
/// <summary>Mock handler for azsdk_create_service_label.</summary>
22+
public class CreateServiceLabelHandler : IMockToolHandler
23+
{
24+
public string ToolName => "azsdk_create_service_label";
25+
public CommandResponse Handle(Dictionary<string, object?>? arguments)
26+
{
27+
var label = arguments?.GetValueOrDefault("label")?.ToString() ?? "Contoso.WidgetManager";
28+
return new ServiceLabelResponse
29+
{
30+
Label = label,
31+
Status = "Created",
32+
PullRequestUrl = $"https://github.com/Azure/azure-sdk-tools/pull/99001"
33+
};
34+
}
35+
}
36+
37+
/// <summary>Mock handler for azsdk_engsys_codeowner_view.</summary>
38+
public class CodeownerViewHandler : IMockToolHandler
39+
{
40+
public string ToolName => "azsdk_engsys_codeowner_view";
41+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new CodeownersViewResponse
42+
{
43+
Packages =
44+
[
45+
new PackageResponse
46+
{
47+
WorkItemId = 70001,
48+
PackageName = "Azure.Template.Contoso",
49+
Language = ".NET",
50+
PackageType = "client",
51+
Owners =
52+
[
53+
new OwnerResponse { GitHubAlias = "contoso-owner-1" },
54+
new OwnerResponse { GitHubAlias = "contoso-owner-2" }
55+
],
56+
Labels = ["Contoso.WidgetManager"]
57+
}
58+
]
59+
};
60+
}
61+
62+
/// <summary>Mock handler for azsdk_engsys_codeowner_check_package.</summary>
63+
public class CodeownerCheckPackageHandler : IMockToolHandler
64+
{
65+
public string ToolName => "azsdk_engsys_codeowner_check_package";
66+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new CheckPackageResponse
67+
{
68+
DirectoryPath = arguments?.GetValueOrDefault("directoryPath")?.ToString() ?? "sdk/contoso/Azure.Template.Contoso",
69+
Owners = ["contoso-owner-1", "contoso-owner-2"],
70+
PRLabels = ["Contoso.WidgetManager"],
71+
ServiceOwners = ["service-team-lead"],
72+
ServiceLabels = ["Service Attention", "Contoso.WidgetManager"]
73+
};
74+
}
75+
76+
internal static class CodeownersModifyMockResponses
77+
{
78+
public static CodeownersModifyResponse OkWithMessage() => new()
79+
{
80+
View = new CodeownersViewResponse
81+
{
82+
Packages =
83+
[
84+
new PackageResponse
85+
{
86+
WorkItemId = 70001,
87+
PackageName = "Azure.Template.Contoso",
88+
Language = ".NET",
89+
PackageType = "client",
90+
Owners = [new OwnerResponse { GitHubAlias = "contoso-owner-1" }],
91+
Labels = ["Contoso.WidgetManager"]
92+
}
93+
]
94+
}
95+
};
96+
}
97+
98+
/// <summary>Mock handler for azsdk_engsys_codeowner_add_package_owner.</summary>
99+
public class CodeownerAddPackageOwnerHandler : IMockToolHandler
100+
{
101+
public string ToolName => "azsdk_engsys_codeowner_add_package_owner";
102+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => CodeownersModifyMockResponses.OkWithMessage();
103+
}
104+
105+
/// <summary>Mock handler for azsdk_engsys_codeowner_add_package_label.</summary>
106+
public class CodeownerAddPackageLabelHandler : IMockToolHandler
107+
{
108+
public string ToolName => "azsdk_engsys_codeowner_add_package_label";
109+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => CodeownersModifyMockResponses.OkWithMessage();
110+
}
111+
112+
/// <summary>Mock handler for azsdk_engsys_codeowner_add_label_owner.</summary>
113+
public class CodeownerAddLabelOwnerHandler : IMockToolHandler
114+
{
115+
public string ToolName => "azsdk_engsys_codeowner_add_label_owner";
116+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => CodeownersModifyMockResponses.OkWithMessage();
117+
}
118+
119+
/// <summary>Mock handler for azsdk_engsys_codeowner_remove_package_owner.</summary>
120+
public class CodeownerRemovePackageOwnerHandler : IMockToolHandler
121+
{
122+
public string ToolName => "azsdk_engsys_codeowner_remove_package_owner";
123+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => CodeownersModifyMockResponses.OkWithMessage();
124+
}
125+
126+
/// <summary>Mock handler for azsdk_engsys_codeowner_remove_package_label.</summary>
127+
public class CodeownerRemovePackageLabelHandler : IMockToolHandler
128+
{
129+
public string ToolName => "azsdk_engsys_codeowner_remove_package_label";
130+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => CodeownersModifyMockResponses.OkWithMessage();
131+
}
132+
133+
/// <summary>Mock handler for azsdk_engsys_codeowner_remove_label_owner.</summary>
134+
public class CodeownerRemoveLabelOwnerHandler : IMockToolHandler
135+
{
136+
public string ToolName => "azsdk_engsys_codeowner_remove_label_owner";
137+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => CodeownersModifyMockResponses.OkWithMessage();
138+
}
139+
140+
/// <summary>Mock handler for azsdk_engsys_codeowner_update_cache.</summary>
141+
public class CodeownerUpdateCacheHandler : IMockToolHandler
142+
{
143+
public string ToolName => "azsdk_engsys_codeowner_update_cache";
144+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new DefaultCommandResponse
145+
{
146+
Message = "CODEOWNERS cache refreshed (mock)",
147+
Result = new { packagesRefreshed = 1, labelOwnersRefreshed = 1 }
148+
};
149+
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Core/UpgradeHandler.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Sdk.Tools.Cli.Models;
5+
6+
namespace Azure.Sdk.Tools.Mock.Handlers.Core;
7+
8+
/// <summary>
9+
/// Mock handler for azsdk_upgrade. Always reports the current "mock" version is up to date so
10+
/// callers exercising the upgrade flow don't trigger a real download.
11+
/// </summary>
12+
public class UpgradeHandler : IMockToolHandler
13+
{
14+
public string ToolName => "azsdk_upgrade";
15+
16+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new UpgradeResponse
17+
{
18+
OldVersion = "0.0.0-mock",
19+
NewVersion = "0.0.0-mock",
20+
Message = "azsdk is already up to date (mock).",
21+
RestartRequired = false
22+
};
23+
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/EngSys/EngSysHandlers.cs‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Sdk.Tools.Cli.Models;
5+
using Azure.Sdk.Tools.Cli.Models.Responses;
6+
7+
namespace Azure.Sdk.Tools.Mock.Handlers.EngSys;
8+
9+
/// <summary>Mock handler for azsdk_analyze_log_file. Returns a single fake build error.</summary>
10+
public class AnalyzeLogFileHandler : IMockToolHandler
11+
{
12+
public string ToolName => "azsdk_analyze_log_file";
13+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new LogAnalysisResponse
14+
{
15+
Summary = "Detected one CS0246 compile error in the build log.",
16+
SuggestedFix = "Add a `using Azure.Core;` directive at the top of the file.",
17+
Errors =
18+
[
19+
new LogEntry
20+
{
21+
File = "src/Contoso.Widgets/Generated/WidgetsClient.cs",
22+
Line = 42,
23+
Message = "error CS0246: The type or namespace name 'WidgetOptions' could not be found"
24+
}
25+
]
26+
};
27+
}
28+
29+
/// <summary>Mock handler for azsdk_get_failed_test_cases.</summary>
30+
public class GetFailedTestCasesHandler : IMockToolHandler
31+
{
32+
public string ToolName => "azsdk_get_failed_test_cases";
33+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new FailedTestRunListResponse
34+
{
35+
Items =
36+
[
37+
new FailedTestRunResponse
38+
{
39+
RunId = 100001,
40+
TestCaseTitle = "Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget",
41+
Outcome = "Failed",
42+
ErrorMessage = "Expected status 200, got 404.",
43+
StackTrace = "at Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget()"
44+
}
45+
]
46+
};
47+
}
48+
49+
/// <summary>Mock handler for azsdk_get_failed_test_case_data.</summary>
50+
public class GetFailedTestCaseDataHandler : IMockToolHandler
51+
{
52+
public string ToolName => "azsdk_get_failed_test_case_data";
53+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new FailedTestRunResponse
54+
{
55+
RunId = 100001,
56+
TestCaseTitle = arguments?.GetValueOrDefault("testCaseTitle")?.ToString()
57+
?? "Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget",
58+
Outcome = "Failed",
59+
ErrorMessage = "Expected status 200, got 404.",
60+
StackTrace = "at Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget()",
61+
Uri = "https://dev.azure.com/azure-sdk/internal/_test/cases?id=100001"
62+
};
63+
}
64+
65+
/// <summary>Mock handler for azsdk_get_failed_test_run_data.</summary>
66+
public class GetFailedTestRunDataHandler : IMockToolHandler
67+
{
68+
public string ToolName => "azsdk_get_failed_test_run_data";
69+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new FailedTestRunListResponse
70+
{
71+
Items =
72+
[
73+
new FailedTestRunResponse
74+
{
75+
RunId = int.TryParse(arguments?.GetValueOrDefault("runId")?.ToString(), out var id) ? id : 100001,
76+
TestCaseTitle = "Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget",
77+
Outcome = "Failed",
78+
ErrorMessage = "Expected status 200, got 404.",
79+
StackTrace = "at Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget()"
80+
}
81+
]
82+
};
83+
}
84+
85+
/// <summary>Mock handler for azsdk_cleanup_ai_agents.</summary>
86+
public class CleanupAiAgentsHandler : IMockToolHandler
87+
{
88+
public string ToolName => "azsdk_cleanup_ai_agents";
89+
public CommandResponse Handle(Dictionary<string, object?>? arguments) => new DefaultCommandResponse
90+
{
91+
Message = "AI agents cleaned up (mock)",
92+
Result = new { agentsDeleted = 3, threadsDeleted = 5 }
93+
};
94+
}

0 commit comments

Comments
 (0)