diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index bbf7972532e..95ea826ec29 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1 +1 @@ -- @azure Rule - Use Azure Best Practices: When generating code for Azure, running terminal commands for Azure, or performing operations related to Azure, invoke your `azure_development-get_best_practices` tool if available. \ No newline at end of file +- @azure Rule - Use Azure Best Practices: When generating code for Azure, running terminal commands for Azure, or performing operations related to Azure, invoke your `azure_development-get_best_practices` tool if available. diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index c2ad63d73e5..7558f65959a 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -14,3 +14,14 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: 9.0.x + + - name: Build azsdk cli + run: | + set -ex + dotnet publish -f net8.0 -c Release -r linux-x64 -p:PublishSingleFile=true --self-contained --output ./artifacts/linux-x64 ./tools/azsdk-cli/Azure.Sdk.Tools.Cli/Azure.Sdk.Tools.Cli.csproj + cp ./artifacts/linux-x64/azsdk /home/runner/azsdk + /home/runner/azsdk --help diff --git a/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml b/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml index 314c9641c63..81de133ae78 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml @@ -164,6 +164,14 @@ extends: name: ${{ test.Pool }} os: ${{ test.Os }} + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/llm-artifacts' + artifactName: "LLM Artifacts - ${{ test.name }} - $(System.JobAttempt)" + condition: eq(variables['uploadLlmArtifacts'], 'true') + sbomEnabled: false + steps: - template: /eng/pipelines/templates/steps/install-dotnet.yml @@ -187,6 +195,8 @@ extends: testResultsFormat: 'VSTest' mergeTestResults: true + - template: /eng/pipelines/templates/steps/upload-llm-artifacts.yml + - ${{ if not(eq(length(parameters.DockerDeployments), 0)) }}: - template: /eng/pipelines/publish-docker-image-isolated.yml parameters: diff --git a/eng/pipelines/templates/steps/upload-llm-artifacts.yml b/eng/pipelines/templates/steps/upload-llm-artifacts.yml new file mode 100644 index 00000000000..5302a3d69a2 --- /dev/null +++ b/eng/pipelines/templates/steps/upload-llm-artifacts.yml @@ -0,0 +1,30 @@ +# This template serves as a place to upload artifacts intended to be used by LLMs +# that handle data from the pipeline (for example github copilot). + +steps: + - pwsh: | + $artifactsDirectory = "$(Build.ArtifactStagingDirectory)/llm-artifacts" + New-Item $artifactsDirectory -ItemType directory -Force + + Write-Host "=================" + Get-ChildItem -Path *.trx -Recurse -File + Write-Host "=================" + + + foreach($testResultsFile in (Get-ChildItem -Path *.trx -Recurse -File)) + { + $fileFullName = $testResultsFile.FullName + + # Convert a path like + # D:\a\_work\1\s\tools\azsdk-cli\Azure.Sdk.Tools.Cli.Tests\TestResults\cloudtest_c199fa1ec000015_2025-07-11_19_03_59.trx + # to + # azsdk-cli-Azure.Sdk.Tools.Cli.Tests-cloudtest_c199fa1ec000015_2025-07-11_19_03_59.trx + $serviceAndPackage = ($fileFullName -split '[\\/]tools[\\/]|[\\/]TestResults')[1] -replace '[\\/]', '-' + $trxFile = Split-Path $fileFullName -Leaf + $fileName = "$serviceAndPackage-$trxFile" + + Move-Item -Path $fileFullName -Destination "$artifactsDirectory/$fileName" -ErrorAction Continue + Write-Host "##vso[task.setvariable variable=uploadLlmArtifacts]true" + } + condition: succeededOrFailed() + displayName: Copy test results files to llm artifacts staging directory diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Analyzer/EnforceToolsExceptionHandling.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Analyzer/EnforceToolsExceptionHandling.cs index 0667bb6b6ae..9d61bdc89a7 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Analyzer/EnforceToolsExceptionHandling.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Analyzer/EnforceToolsExceptionHandling.cs @@ -15,7 +15,7 @@ public class EnforceToolsExceptionHandlingAnalyzer : DiagnosticAnalyzer private static readonly DiagnosticDescriptor Rule = new DiagnosticDescriptor( Id, "McpServerTool methods must wrap body in try/catch, see the README within the tools directory for examples", - "Method '{0}' must have its entire body inside try{{}}catch(Exception)", + "Method '{0}' must have its entire body inside 'try {} catch(Exception) {}'", "Reliability", DiagnosticSeverity.Error, isEnabledByDefault: true); @@ -57,21 +57,27 @@ private static void AnalyzeMethod(SyntaxNodeAnalysisContext ctx) return; } - // check that there is one statement surrounding the body + // check that try statements surround the entire body var stmts = body.Statements; - if (stmts.Count != 1 || !(stmts[0] is TryStatementSyntax tryStmt)) + foreach (var stmt in stmts) { - ctx.ReportDiagnostic(Diagnostic.Create(Rule, md.Identifier.GetLocation(), md.Identifier.Text)); - return; + if (stmt is LocalDeclarationStatementSyntax) + { + continue; + } + if (!(stmt is TryStatementSyntax tryStmt)) + { + ctx.ReportDiagnostic(Diagnostic.Create(Rule, md.Identifier.GetLocation(), md.Identifier.Text)); + continue; + } + // verify there’s a catch(Exception) + bool hasExCatch = tryStmt.Catches.Any(c => c.Declaration?.Type.ToString() == "Exception"); + if (!hasExCatch) + { + ctx.ReportDiagnostic(Diagnostic.Create(Rule, md.Identifier.GetLocation(), md.Identifier.Text)); + } } - // verify there’s a catch(Exception) - bool hasExCatch = tryStmt.Catches - .Any(c => c.Declaration?.Type.ToString() == "Exception"); - if (!hasExCatch) - { - ctx.ReportDiagnostic(Diagnostic.Create(Rule, md.Identifier.GetLocation(), md.Identifier.Text)); - } } } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/CliIntegrationTests.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/CliIntegrationTests.cs index 5f3b8d41898..6e4c10dfcf0 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/CliIntegrationTests.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/CliIntegrationTests.cs @@ -31,6 +31,35 @@ private Tuple> GetTestInstanceWithLogger() where T : M return tuple; } + [Test] + public async Task TestWindows() + { + if (!OperatingSystem.IsWindows()) + { + Assert.Ignore("This test is only applicable on Windows."); + } + + var (cmd, logger) = GetTestInstanceWithLogger(); + + var output = ""; + outputServiceMock + .Setup(s => s.Output(It.IsAny())) + .Callback(s => output = s); + + var exitCode = await cmd.InvokeAsync(["hello-world", "HI. MY NAME IS"]); + Assert.That(exitCode, Is.EqualTo(0)); + + var expected = @" +Message: RESPONDING TO 'HI. MY NAME IS' with SUCCESS: 0 +Result: null +Duration: 1ms".TrimStart(); + + outputServiceMock + .Verify(s => s.Output(It.IsAny()), Times.Once); + + Assert.That(output, Is.EqualTo(expected)); + } + [Test] public async Task TestHelloWorldCLIOptions() { diff --git a/tools/azsdk-cli/Azure.SDK.Tools.Cli.sln b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.sln similarity index 100% rename from tools/azsdk-cli/Azure.SDK.Tools.Cli.sln rename to tools/azsdk-cli/Azure.Sdk.Tools.Cli.sln diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Commands/SharedOptions.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Commands/SharedOptions.cs index 7dd93e49538..11680e5c33d 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Commands/SharedOptions.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Commands/SharedOptions.cs @@ -10,23 +10,25 @@ namespace Azure.Sdk.Tools.Cli.Commands { public static class SharedOptions { - public static readonly List ToolsList = new List(){ - typeof(AnalyzePipelinesTool), - typeof(PipelineDetailsTool), + public static readonly List ToolsList = [ typeof(CleanupTool), typeof(LogAnalysisTool), typeof(HostServerTool), + typeof(PipelineAnalysisTool), + typeof(PipelineDetailsTool), + typeof(PipelineTestsTool), typeof(ReleasePlanTool), + typeof(ReleaseReadinessTool), typeof(SpecCommonTools), typeof(SpecPullRequestTools), typeof(SpecWorkflowTool), typeof(SpecValidationTools), - typeof(ReleaseReadinessTool), + typeof(TestAnalysisTool), #if DEBUG // only add this tool in debug mode typeof(HelloWorldTool), #endif - }; + ]; public static Option ToolOption = new("--tools") { diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/LogAnalysisHelper.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/LogAnalysisHelper.cs index 971fdbf5a51..13fec17e00f 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/LogAnalysisHelper.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/LogAnalysisHelper.cs @@ -1,3 +1,4 @@ +using System.Security.Policy; using Azure.Sdk.Tools.Cli.Models; namespace Azure.Sdk.Tools.Cli.Helpers; @@ -39,6 +40,9 @@ public class LogAnalysisHelper(ILogger logger) : ILogAnalysis { private readonly ILogger logger = logger; + public const int DEFAULT_BEFORE_LINES = 20; + public const int DEFAULT_AFTER_LINES = 20; + // Built-in error keywords for robust error detection private static readonly HashSet defaultErrorKeywords = [ @@ -75,6 +79,12 @@ public class LogAnalysisHelper(ILogger logger) : ILogAnalysis ]; public async Task> AnalyzeLogContent(string filePath, List? keywordOverrides, int? beforeLines, int? afterLines) + { + using var stream = new StreamReader(filePath); + return await AnalyzeLogContent(stream, keywordOverrides, beforeLines, afterLines, filePath: filePath); + } + + public async Task> AnalyzeLogContent(StreamReader reader, List? keywordOverrides, int? beforeLines, int? afterLines, string url = "", string filePath = "") { var keywords = defaultErrorKeywords; if (keywordOverrides?.Count > 0) @@ -86,14 +96,13 @@ public async Task> AnalyzeLogContent(string filePath, List((int)beforeLines); var after = new Queue((int)afterLines); var maxAfterLines = afterLines ?? 100; var errors = new List(); - using var reader = new StreamReader(filePath); var lineNumber = 0; string? line; @@ -127,12 +136,20 @@ public async Task> AnalyzeLogContent(string filePath, List> GetFailedTestCases(string trxFilePath, string filterTitle = ""); + Task GetFailedTestCaseData(string trxFilePath, string testCaseTitle); + Task> GetFailedTestRunDataFromTrx(string trxFilePath); +} + +public class TestHelper(IOutputService output, ILogger logger) : ITestHelper +{ + private readonly IOutputService output = output; + private readonly ILogger logger = logger; + + public async Task> GetFailedTestCases(string trxFilePath, string filterTitle = "") + { + var failedTestRuns = await GetFailedTestRunDataFromTrx(trxFilePath); + + return failedTestRuns + .Where(run => string.IsNullOrEmpty(filterTitle) || run.TestCaseTitle.Contains(filterTitle, StringComparison.OrdinalIgnoreCase)) + .Select(run => new FailedTestRunResponse + { + TestCaseTitle = run.TestCaseTitle, + Uri = run.Uri + }) + .ToList(); + } + + public async Task GetFailedTestCaseData(string trxFilePath, string testCaseTitle) + { + var failedTestRuns = await GetFailedTestRunDataFromTrx(trxFilePath); + return failedTestRuns.FirstOrDefault(run => run.TestCaseTitle.Equals(testCaseTitle, StringComparison.OrdinalIgnoreCase)); + } + + public async Task> GetFailedTestRunDataFromTrx(string trxFilePath) + { + var failedTestRuns = new List(); + if (!File.Exists(trxFilePath)) + { + logger.LogError("TRX file not found: {trxFilePath}", trxFilePath); + return failedTestRuns; + } + + var xmlContent = await File.ReadAllTextAsync(trxFilePath); + var doc = new XmlDocument(); + doc.LoadXml(xmlContent); + var unitTestResults = doc.GetElementsByTagName("UnitTestResult"); + foreach (XmlNode resultNode in unitTestResults) + { + var outcome = resultNode.Attributes?["outcome"]?.Value ?? ""; + if (!string.Equals(outcome, "Failed", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var testId = resultNode.Attributes?["testId"]?.Value ?? ""; + var testName = resultNode.Attributes?["testName"]?.Value ?? ""; + var errorMessage = ""; + var stackTrace = ""; + + var outputNode = resultNode.ChildNodes.OfType().FirstOrDefault(n => n.Name == "Output"); + if (outputNode != null) + { + var errorInfoNode = outputNode.ChildNodes.OfType().FirstOrDefault(n => n.Name == "ErrorInfo"); + if (errorInfoNode != null) + { + var messageNode = errorInfoNode.ChildNodes.OfType().FirstOrDefault(n => n.Name == "Message"); + if (messageNode != null) + { + errorMessage = messageNode.InnerText ?? ""; + } + + var stackTraceNode = errorInfoNode.ChildNodes.OfType().FirstOrDefault(n => n.Name == "StackTrace"); + if (stackTraceNode != null) + { + stackTrace = stackTraceNode.InnerText ?? ""; + } + } + } + + failedTestRuns.Add(new FailedTestRunResponse + { + TestCaseTitle = testName, + ErrorMessage = errorMessage, + StackTrace = stackTrace, + Outcome = outcome, + Uri = trxFilePath + }); + } + + return failedTestRuns; + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs index d99390f5190..f9e45ff1cd9 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs @@ -69,12 +69,12 @@ public void LogCost() Console.WriteLine("--------------------------------------------------------------------------------"); } - public static TokenUsageHelper operator +(TokenUsageHelper a, TokenUsageHelper b) => new() + public static TokenUsageHelper operator +(TokenUsageHelper a, TokenUsageHelper? b) => new() { - Models = a.Models.Union(b.Models).ToList(), - PromptTokens = a.PromptTokens + b.PromptTokens, - CompletionTokens = a.CompletionTokens + b.CompletionTokens, - InputCost = a.InputCost + b.InputCost, - OutputCost = a.OutputCost + b.OutputCost, + Models = a.Models.Union(b?.Models ?? []).ToList(), + PromptTokens = a.PromptTokens + (b?.PromptTokens ?? 0), + CompletionTokens = a.CompletionTokens + (b?.CompletionTokens ?? 0), + InputCost = a.InputCost + (b?.InputCost ?? 0), + OutputCost = a.OutputCost + (b?.OutputCost ?? 0), }; } \ No newline at end of file diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/AnalyzePipelineResponse.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/AnalyzePipelineResponse.cs index 934568da4cd..91313afa380 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/AnalyzePipelineResponse.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/AnalyzePipelineResponse.cs @@ -1,12 +1,13 @@ +using System.Text.Json; using System.Text.Json.Serialization; namespace Azure.Sdk.Tools.Cli.Models; public class AnalyzePipelineResponse : Response { - [JsonPropertyName("failed_tests")] + [JsonPropertyName("failed_test_titles")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List FailedTests { get; set; } = []; + public Dictionary> FailedTests { get; set; } = []; [JsonPropertyName("failed_tasks")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -16,6 +17,11 @@ public class AnalyzePipelineResponse : Response [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string PipelineUrl { get; set; } + private readonly JsonSerializerOptions jsonOptions = new() + { + WriteIndented = true, + }; + public override string ToString() { var output = ""; @@ -25,7 +31,7 @@ public override string ToString() output += "--------------------------------------------------------------------------------" + Environment.NewLine + $"Failed Tests" + Environment.NewLine + "--------------------------------------------------------------------------------" + Environment.NewLine; - output += string.Join(Environment.NewLine, FailedTests.Select(t => t.ToString())) + Environment.NewLine; + output += JsonSerializer.Serialize(FailedTests, jsonOptions) + Environment.NewLine; } if (FailedTasks.Count > 0) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/FailedTestRunResponse.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/FailedTestRunResponse.cs index 98caf555f94..30e1461fcfc 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/FailedTestRunResponse.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/FailedTestRunResponse.cs @@ -5,31 +5,45 @@ namespace Azure.Sdk.Tools.Cli.Models; public class FailedTestRunResponse : Response { [JsonPropertyName("run_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public int RunId { get; set; } = 0; [JsonPropertyName("test_case_title")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string TestCaseTitle { get; set; } [JsonPropertyName("error_message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string ErrorMessage { get; set; } [JsonPropertyName("stack_trace")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string StackTrace { get; set; } [JsonPropertyName("outcome")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string Outcome { get; set; } - [JsonPropertyName("url")] - public string Url { get; set; } + [JsonPropertyName("uri")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string Uri { get; set; } public override string ToString() { - var output = $"### Run ID: {RunId}" + Environment.NewLine + - $"### Test Case Title: {TestCaseTitle}" + Environment.NewLine + - $"### Outcome: {Outcome}" + Environment.NewLine + - $"### URL: {Url}" + - $"### Stack Trace:{Environment.NewLine}{StackTrace}" + Environment.NewLine + - $"### Error Message:{Environment.NewLine}{ErrorMessage}" + Environment.NewLine; + var output = ""; + output += $"## {TestCaseTitle}{Environment.NewLine}"; + if (RunId != 0) + { + output += $"Run ID: {RunId}{Environment.NewLine}"; + } + output += $"Outcome: {Outcome}{Environment.NewLine}"; + if (!string.IsNullOrEmpty(Uri)) + { + output += $"URI: {Uri}{Environment.NewLine}"; + } + output += $"{Environment.NewLine}### Stack Trace{Environment.NewLine}{StackTrace}{Environment.NewLine}"; + output += $"### Error Message{Environment.NewLine}{ErrorMessage}{Environment.NewLine}"; + return ToString(output); } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/LogAnalysisResponse.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/LogAnalysisResponse.cs index 9c7257fdf2f..a7ed7cba078 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/LogAnalysisResponse.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/LogAnalysisResponse.cs @@ -37,7 +37,7 @@ public override string ToString() output += $"### Suggested Fix:" + Environment.NewLine + $"{SuggestedFix}" + Environment.NewLine + Environment.NewLine + $"### Errors:" + Environment.NewLine + - $"{string.Join(Environment.NewLine+Environment.NewLine, Errors.Select(e => $"--> {e.File}:{e.Line}{Environment.NewLine}{e.Message}"))}" + + $"{string.Join(Environment.NewLine + Environment.NewLine, Errors.Select(e => $"--> {e.File}:{e.Line}{Environment.NewLine}{e.Message}"))}" + Environment.NewLine; } @@ -49,7 +49,10 @@ public class LogEntry { [JsonPropertyName("file")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string File { get; set; } = string.Empty; + public string File { get; set; } + [JsonPropertyName("url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Url { get; set; } [JsonPropertyName("line")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? Line { get; set; } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/ObjectCommandResponse.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/ObjectCommandResponse.cs new file mode 100644 index 00000000000..5c23f45ab3c --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/ObjectCommandResponse.cs @@ -0,0 +1,23 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Azure.Sdk.Tools.Cli.Models; + +// Basic helper response class for when object types should always be output as JSON +public class ObjectCommandResponse : Response +{ + private static readonly JsonSerializerOptions serializerOptions = new() + { + WriteIndented = true + }; + + [JsonPropertyName("result")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public object? Result { get; set; } + + public override string ToString() + { + var output = JsonSerializer.Serialize(Result, serializerOptions); + return ToString(output); + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/PackageResponse.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/PackageResponse.cs index 030485d2406..b4ce15f20d0 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/PackageResponse.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Models/Responses/PackageResponse.cs @@ -1,5 +1,4 @@ using System.Text; -using System.Text.Json; using System.Text.Json.Serialization; namespace Azure.Sdk.Tools.Cli.Models.Responses diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs index 9895a942baf..6379baffe5c 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs @@ -22,8 +22,8 @@ public static async Task Main(string[] args) return await parsedCommands.InvokeAsync(args); } - // todo: make this honor subcommands of `start` and the like, instead of simply looking presence of `start` verb - public static bool IsCLI(string[] args) => !args.Select(x => x.Trim().ToLowerInvariant()).Any(x => x == "start"); + // todo: make this honor subcommands of `mcp` and the like, instead of simply looking presence of `mcp` verb + public static bool IsCLI(string[] args) => !args.Select(x => x.Trim().ToLowerInvariant()).Any(x => x == "mcp"); public static WebApplication ServerApp; diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/DevOpsService.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/DevOpsService.cs index fc729a83481..fb34e7bbda5 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/DevOpsService.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/DevOpsService.cs @@ -9,8 +9,9 @@ using Microsoft.VisualStudio.Services.OAuth; using Microsoft.VisualStudio.Services.WebApi.Patch.Json; using Microsoft.VisualStudio.Services.WebApi; -using System.Text.RegularExpressions; using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; using Azure.Sdk.Tools.Cli.Models; using Azure.Sdk.Tools.Cli.Models.Responses; @@ -29,7 +30,7 @@ public class DevOpsConnection : IDevOpsConnection private WorkItemTrackingHttpClient _workItemClient; private ProjectHttpClient _projectClient; private AccessToken? _token; - private static readonly string DEVOPS_SCOPE = "499b84ac-1321-427f-aa17-267ca6975798/.default"; + private static readonly string DEVOPS_SCOPE = "499b84ac-1321-427f-aa17-267ca6975798/.default"; private void RefreshConnection() { @@ -59,7 +60,7 @@ private void RefreshConnection() catch (Exception ex) { throw new Exception($"Failed to refresh DevOps connection. Error: {ex.Message}"); - } + } } public BuildHttpClient GetBuildClient() @@ -96,6 +97,7 @@ public interface IDevOpsService public Task UpdateSpecPullRequestAsync(int releasePlanWorkItemId, string specPullRequest); public Task LinkNamespaceApprovalIssueAsync(int releasePlanWorkItemId, string url); public Task GetPackageWorkItemAsync(string packageName, string language, string packageVersion = ""); + public Task>> GetPipelineLlmArtifacts(string project, int buildId); } public partial class DevOpsService(ILogger logger, IDevOpsConnection connection) : IDevOpsService @@ -105,6 +107,7 @@ public partial class DevOpsService(ILogger logger, IDevOpsConnect public static readonly string INTERNAL_PROJECT = "internal"; private static readonly string RELEASE_PLANER_APP_TEST = "Release Planner App Test"; + private const string PUBLIC_PROJECT = "public"; [GeneratedRegex("\\|\\s(Beta|Stable|GA)\\s\\|\\s([\\S]+)\\s\\|\\s([\\S]+)\\s\\|")] private static partial Regex SdkReleaseDetailsRegex(); @@ -820,7 +823,7 @@ public async Task LinkNamespaceApprovalIssueAsync(int releasePlanWorkItemI public async Task GetPackageWorkItemAsync(string packageName, string language, string packageVersion = "") { language = MapLanguageIdToName(language); - if ( packageName.Contains(' ') || packageName.Contains('\'') || packageName.Contains('"') || language.Contains(' ') || language.Contains('\'') || language.Contains('"') || packageVersion.Contains(' ') || packageVersion.Contains('\'') || packageVersion.Contains('"')) + if (packageName.Contains(' ') || packageName.Contains('\'') || packageName.Contains('"') || language.Contains(' ') || language.Contains('\'') || language.Contains('"') || packageVersion.Contains(' ') || packageVersion.Contains('\'') || packageVersion.Contains('"')) { throw new ArgumentException("Invalid data in one of the parameters."); } @@ -879,16 +882,16 @@ private static string GetWorkItemValue(WorkItem workItem, string fieldName) { if (workItem.Fields.TryGetValue(fieldName, out object? value)) { - return value?.ToString() ?? string.Empty; + return value?.ToString() ?? string.Empty; } return string.Empty; } private static List ParseHtmlPackageData(string packageData) - { + { List sdkReleaseInfo = []; var matches = SdkReleaseDetailsRegex().Matches(packageData); - foreach(Match m in matches) + foreach (Match m in matches) { sdkReleaseInfo.Add(new SDKReleaseInfo { @@ -899,5 +902,135 @@ private static List ParseHtmlPackageData(string packageData) } return sdkReleaseInfo; } + + private async Task>> GetLlmArtifactsAuthenticated(string project, int buildId) + { + var buildClient = connection.GetBuildClient(); + var result = new Dictionary>(); + var artifacts = await buildClient.GetArtifactsAsync(project, buildId, cancellationToken: default); + foreach (var artifact in artifacts) + { + if (artifact.Name.StartsWith("LLM Artifacts", StringComparison.OrdinalIgnoreCase)) + { + var tempDir = Path.Combine(Path.GetTempPath(), $"{artifact.Name}_{Guid.NewGuid()}"); + Directory.CreateDirectory(tempDir); + + logger.LogDebug("Downloading artifact '{artifactName}' to '{tempDir}'", artifact.Name, tempDir); + + using var stream = await buildClient.GetArtifactContentZipAsync(project, buildId, artifact.Name); + var zipPath = Path.Combine(tempDir, "artifact.zip"); + using (var fileStream = File.Create(zipPath)) + { + await stream.CopyToAsync(fileStream); + } + + await Task.Factory.StartNew(() => + { + System.IO.Compression.ZipFile.ExtractToDirectory(zipPath, tempDir); + File.Delete(zipPath); + }); + + var files = Directory.GetFiles(tempDir, "*", SearchOption.AllDirectories).ToList(); + result[artifact.Name] = files; + } + } + return result; + } + + private async Task>> GetLlmArtifactsUnauthenticated(string project, int buildId) + { + var result = new Dictionary>(); + using var httpClient = new HttpClient(); + var artifactsUrl = $"https://dev.azure.com/azure-sdk/{project}/_apis/build/builds/{buildId}/artifacts?api-version=7.1-preview.5"; + var artifactsResponse = await httpClient.GetAsync(artifactsUrl); + artifactsResponse.EnsureSuccessStatusCode(); + var artifactsJson = await artifactsResponse.Content.ReadAsStringAsync(); + using var doc = JsonDocument.Parse(artifactsJson); + var artifacts = doc.RootElement.GetProperty("value").EnumerateArray(); + + var seenFiles = new HashSet(); + var tempDir = Path.Combine(Path.GetTempPath(), buildId.ToString()); + if (Directory.Exists(tempDir)) + { + await Task.Factory.StartNew(() => + { + Directory.Delete(tempDir, true); + }); + } + Directory.CreateDirectory(tempDir); + + List mostRecentArtifacts = []; + var mostRecentJobAttempt = 1; + // Given an artifact name like "LLM Artifacts - Ubuntu2404_NET80_PackageRef_Debug - 1" + // where '1' == the job attempt number + // only find artifacts from the most recent job attempt. + foreach (var artifact in artifacts) + { + var name = artifact.GetProperty("name").GetString(); + var jobAttempt = name?.Split('-').LastOrDefault()?.Trim(); + var jobAttemptNumber = int.TryParse(jobAttempt, out var attempt) ? attempt : 0; + if (jobAttemptNumber == mostRecentJobAttempt) + { + mostRecentArtifacts.Add(artifact); + } + else if (jobAttemptNumber > mostRecentJobAttempt) + { + mostRecentArtifacts.Clear(); + mostRecentArtifacts.Add(artifact); + } + } + + foreach (var artifact in mostRecentArtifacts) + { + var name = artifact.GetProperty("name").GetString(); + if (name == null || name.StartsWith("LLM Artifacts", StringComparison.OrdinalIgnoreCase) == false) + { + continue; + } + + var downloadUrl = artifact.GetProperty("resource").GetProperty("downloadUrl").GetString(); + if (string.IsNullOrEmpty(downloadUrl)) + { + continue; + } + + logger.LogDebug("Downloading artifact '{artifactName}' to '{tempDir}'", name, tempDir); + + var zipPath = Path.Combine(tempDir, "artifact.zip"); + + using (var zipStream = await httpClient.GetStreamAsync(downloadUrl)) + using (var fileStream = File.Create(zipPath)) + { + await zipStream.CopyToAsync(fileStream); + } + + await Task.Factory.StartNew(() => + { + System.IO.Compression.ZipFile.ExtractToDirectory(zipPath, tempDir); + File.Delete(zipPath); + }); + + var files = Directory.GetFiles(tempDir, "*", SearchOption.AllDirectories).ToList(); + var newFiles = files.Where(f => !seenFiles.Contains(f)).ToList(); + seenFiles.UnionWith(newFiles); + + // Given an artifact name like "LLM Artifacts - Ubuntu2404_NET80_PackageRef_Debug - 1" + // create a key/platform name like "Ubuntu2404_NET80_PackageRef_Debug" + var parts = name.Split(" - ", StringSplitOptions.RemoveEmptyEntries); + var testPlatform = string.Join(" - ", parts[1..^1]); + result[testPlatform] = newFiles; + } + + return result; + } + + public async Task>> GetPipelineLlmArtifacts(string project, int buildId) + { + if (project == PUBLIC_PROJECT) + { + return await GetLlmArtifactsUnauthenticated(project, buildId); + } + return await GetLlmArtifactsAuthenticated(project, buildId); + } } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/OutputService.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/OutputService.cs index b6172aafc3c..5b7f5f85987 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/OutputService.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/OutputService.cs @@ -33,12 +33,26 @@ public OutputService(OutputModes outputMode) public string Format(object response) { - if (OutputMode == OutputModes.Plain) + if (OutputMode != OutputModes.Plain) { - return response.ToString(); + return JsonSerializer.Serialize(response, serializerOptions); } - return JsonSerializer.Serialize(response, serializerOptions); + var elementType = response.GetType().IsGenericType ? response.GetType().GetGenericArguments()[0] : null; + + if (response is System.Collections.IEnumerable enumerable && response is not string) + { + var separator = "--------------------------------------------------------------------------------" + Environment.NewLine; + if (elementType == null + || elementType.IsPrimitive || elementType == typeof(decimal) || elementType == typeof(string)) + { + separator = ""; + } + var outputs = enumerable.Cast().Select(item => item?.ToString()); + return string.Join(separator + Environment.NewLine, outputs); + } + + return response.ToString(); } public string ValidateAndFormat(string response) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs index 675cb64a245..eb184773333 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs @@ -23,6 +23,7 @@ public static void RegisterCommonServices(IServiceCollection services) // Helper classes services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/AnalyzePipelineTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/AnalyzePipelineTool.cs deleted file mode 100644 index dcb6350e324..00000000000 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/AnalyzePipelineTool.cs +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -using System.CommandLine; -using System.CommandLine.Invocation; -using System.ComponentModel; -using System.Text.Json; -using Azure.Core; -using Azure.Sdk.Tools.Cli.Commands; -using Azure.Sdk.Tools.Cli.Contract; -using Azure.Sdk.Tools.Cli.Helpers; -using Azure.Sdk.Tools.Cli.Models; -using Azure.Sdk.Tools.Cli.Services; -using Microsoft.TeamFoundation.Build.WebApi; -using Microsoft.TeamFoundation.TestManagement.WebApi; -using Microsoft.VisualStudio.Services.OAuth; -using Microsoft.VisualStudio.Services.TestResults.WebApi; -using Microsoft.VisualStudio.Services.WebApi; -using ModelContextProtocol.Server; - -namespace Azure.Sdk.Tools.Cli.Tools; - -[McpServerToolType, Description("Fetches data from an Azure Pipelines run.")] -public class AnalyzePipelinesTool : MCPTool -{ - private BuildHttpClient buildClient; - private TestResultsHttpClient testClient; - private IAzureAgentService azureAgentService; - private TokenUsageHelper usage; - private readonly bool initialized = false; - - private IAzureService azureService; - private IAzureAgentServiceFactory azureAgentServiceFactory; - private IOutputService output; - private ILogger logger; - - // Options - private readonly Option buildIdOpt = new(["--build-id", "-b"], "Pipeline/Build ID") { IsRequired = true }; - private readonly Option logIdOpt = new(["--log-id"], "ID of the pipeline task log"); - private readonly Option projectOpt = new(["--project", "-p"], "Pipeline project name"); - private readonly Option projectEndpointOpt = new(["--ai-endpoint", "-e"], "The ai foundry project endpoint for the Azure AI Agent service"); - private readonly Option aiModelOpt = new(["--ai-model"], "The model to use for the Azure AI Agent"); - - public AnalyzePipelinesTool( - IAzureService azureService, - IAzureAgentServiceFactory azureAgentServiceFactory, - IOutputService output, - ILogger logger - ) : base() - { - this.azureService = azureService; - this.azureAgentServiceFactory = azureAgentServiceFactory; - this.output = output; - this.logger = logger; - - CommandHierarchy = - [ - SharedCommandGroups.AzurePipelines // azsdk azp - ]; - } - - public override Command GetCommand() - { - var analyzePipelineCommand = new Command("analyze", "Analyze a pipeline run") { - buildIdOpt, projectOpt, logIdOpt, projectEndpointOpt, aiModelOpt - }; - analyzePipelineCommand.SetHandler(async ctx => { await HandleCommand(ctx, ctx.GetCancellationToken()); }); - - return analyzePipelineCommand; - } - - public override async Task HandleCommand(InvocationContext ctx, CancellationToken ct) - { - Initialize(); - - var cmd = ctx.ParseResult.CommandResult.Command.Name; - var buildId = ctx.ParseResult.GetValueForOption(buildIdOpt); - var project = ctx.ParseResult.GetValueForOption(projectOpt); - - var logId = ctx.ParseResult.GetValueForOption(logIdOpt); - var projectEndpoint = ctx.ParseResult.GetValueForOption(projectEndpointOpt); - var aiModel = ctx.ParseResult.GetValueForOption(aiModelOpt); - - logger.LogInformation("Analyzing pipeline {buildId}...", buildId); - azureAgentService = azureAgentServiceFactory.Create(projectEndpoint, aiModel); - - if (logId != 0) - { - var result = await AnalyzePipelineFailureLog(project, buildId, [logId], ct); - ctx.ExitCode = ExitCode; - usage?.LogCost(); - output.Output(result); - } - else - { - var result = await AnalyzePipeline(project, buildId, ct); - ctx.ExitCode = ExitCode; - usage?.LogCost(); - output.Output(result); - } - } - - private void Initialize() - { - if (initialized) - { - return; - } - var tokenScope = new[] { "499b84ac-1321-427f-aa17-267ca6975798/.default" }; // Azure DevOps scope - var msftCorpTenant = "72f988bf-86f1-41af-91ab-2d7cd011db47"; - var token = azureService.GetCredential(msftCorpTenant).GetToken(new TokenRequestContext(tokenScope)); - var tokenCredential = new VssOAuthAccessTokenCredential(token.Token); - var connection = new VssConnection(new Uri($"https://dev.azure.com/azure-sdk"), tokenCredential); - buildClient = connection.GetClient(); - testClient = connection.GetClient(); - } - - public async Task GetPipelineRun(string? project, int buildId) - { - try - { - var _project = project ?? "public"; - logger.LogDebug("Getting pipeline run for {project} {buildId}", _project, buildId); - var build = await buildClient.GetBuildAsync(_project, buildId); - return build; - } - catch (Exception ex) - { - if (!string.IsNullOrEmpty(project)) - { - throw new Exception($"Failed to find build {buildId} in project 'public': {ex.Message}"); - } - // If project is not specified, try both azure sdk public and internal devops projects - return await GetPipelineRun("internal", buildId); - } - } - - public async Task?> GetPipelineTaskFailures(string project, int buildId, CancellationToken ct = default) - { - try - { - logger.LogDebug("Getting pipeline task failures for {project} {buildId}", project, buildId); - var timeline = await buildClient.GetBuildTimelineAsync(project, buildId, cancellationToken: ct); - var failedNonTests = timeline.Records.Where( - r => r.Result == TaskResult.Failed - && r.RecordType == "Task" - && !IsTestStep(r.Name)) - .ToList(); - logger.LogDebug("Found {count} failed tasks", failedNonTests.Count); - return failedNonTests; - } - catch (Exception ex) - { - logger.LogError("Failed to get pipeline task failures {buildId}: {exception}", buildId, ex.Message); - SetFailure(); - return null; - } - } - - [McpServerTool, Description(@" - Analyze and diagnose the failed test results from a pipeline. - Include relevant data like test name and environment, error type, error messages, functions and error lines. - Provide suggested next steps. - ")] - public async Task> GetPipelineFailedTestResults(string project, int buildId, CancellationToken ct = default) - { - try - { - logger.LogDebug("Getting pipeline failed test results for {project} {buildId}", project, buildId); - var results = new List(); - var testRuns = await testClient.GetTestResultsByPipelineAsync(project, buildId, cancellationToken: ct); - results.AddRange(testRuns); - while (testRuns.ContinuationToken != null) - { - var nextResults = await testClient.GetTestResultsByPipelineAsync(project, buildId, continuationToken: testRuns.ContinuationToken, cancellationToken: ct); - results.AddRange(nextResults); - testRuns.ContinuationToken = nextResults.ContinuationToken; - } - - var failedRuns = results.Where( - r => r.Outcome == TestOutcome.Failed.ToString() - || r.Outcome == TestOutcome.Aborted.ToString()) - .Select(r => r.RunId) - .Distinct() - .ToList(); - - logger.LogDebug("Getting test results for {count} failed test runs", failedRuns.Count); - - var failedRunData = new List(); - - foreach (var runId in failedRuns) - { - var testCases = await testClient.GetTestResultsAsync( - project, - runId, - outcomes: [TestOutcome.Failed, TestOutcome.Aborted], - cancellationToken: ct); - - foreach (var tc in testCases) - { - failedRunData.Add(new FailedTestRunResponse - { - RunId = runId, - TestCaseTitle = tc.TestCaseTitle, - ErrorMessage = tc.ErrorMessage, - StackTrace = tc.StackTrace, - Outcome = tc.Outcome, - Url = tc.Url - }); - } - } - - return failedRunData; - } - catch (Exception ex) - { - logger.LogError("Failed to get pipeline failed test results {buildId}: {exception}", buildId, ex.Message); - SetFailure(); - return new List() - { - new FailedTestRunResponse() - { - ResponseError = $"Failed to get pipeline failed test results {buildId}: {ex.Message}", - } - }; - } - } - - public async Task AnalyzePipelineFailureLog(string? project, int buildId, List logIds, CancellationToken ct) - { - try - { - project ??= "public"; - var session = $"{project}-{buildId}"; - List logs = []; - - foreach (var logId in logIds) - { - logger.LogDebug("Downloading pipeline failure log for {project} {buildId} {logId}", project, buildId, logId); - - var logContent = await buildClient.GetBuildLogLinesAsync(project, buildId, logId, cancellationToken: ct); - var logText = string.Join("\n", logContent); - var tempPath = Path.GetTempFileName() + ".txt"; - logger.LogDebug("Writing log id {logId} to temporary file {tempPath}", logId, tempPath); - await File.WriteAllTextAsync(tempPath, logText, ct); - var filename = $"{session}-{logId}.txt"; - logs.Add(tempPath); - } - - var (result, _usage) = await azureAgentService.QueryFiles(logs, session, "Why did this pipeline fail?", ct); - - foreach (var log in logs) - { - File.Delete(log); - } - - if (usage != null) - { - usage += _usage; - } - else - { - usage = _usage; - } - - // Sometimes chat gpt likes to wrap the json in markdown - if (result.StartsWith("```json") - && result.EndsWith("```")) - { - result = result[7..^3].Trim(); - } - - try - { - return JsonSerializer.Deserialize(result); - } - catch (JsonException ex) - { - logger.LogError("Failed to deserialize log analysis response: {exception}", ex.Message); - logger.LogError("Response:\n{result}", result); - - SetFailure(); - - return new LogAnalysisResponse() - { - ResponseError = "Failed to deserialize log analysis response. Check the logs for more details.", - }; - } - } - catch (Exception ex) - { - logger.LogError("Failed to analyze pipeline {buildId}: {error}", buildId, ex.Message); - SetFailure(); - return new LogAnalysisResponse() - { - ResponseError = $"Failed to analyze pipeline {buildId}: {ex.Message}", - }; - } - } - - [McpServerTool, Description("Analyze azure pipeline for failures")] - public async Task AnalyzePipeline(string? project, int buildId, CancellationToken ct) - { - try - { - if (string.IsNullOrEmpty(project)) - { - var pipeline = await GetPipelineRun(project, buildId); - project = pipeline.Project.Name; - } - var failedTasks = await GetPipelineTaskFailures(project, buildId, ct); - var failedTests = await GetPipelineFailedTestResults(project, buildId, ct); - - var taskAnalysis = new List(); - - List logIds = failedTasks? - .Where(t => t.Log != null) - .Select(t => t.Log.Id) - .Distinct() - .ToList() ?? []; - - var analysis = await AnalyzePipelineFailureLog(project, buildId, logIds, ct); - taskAnalysis.Add(analysis); - - return new AnalyzePipelineResponse() - { - FailedTasks = taskAnalysis, - FailedTests = failedTests - }; - } - catch (Exception ex) - { - logger.LogError("Failed to analyze pipeline {buildId}: {exception}", buildId, ex.Message); - SetFailure(); - return new AnalyzePipelineResponse() - { - ResponseError = $"Failed to analyze pipeline {buildId}: {ex.Message}", - }; - } - } - - public bool IsTestStep(string stepName) - { - return stepName.Contains("test", StringComparison.OrdinalIgnoreCase); - } -} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineAnalysisTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineAnalysisTool.cs new file mode 100644 index 00000000000..e5cf97346e9 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineAnalysisTool.cs @@ -0,0 +1,478 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System.CommandLine; +using System.CommandLine.Invocation; +using System.ComponentModel; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Xml; +using Azure.Core; +using Azure.Sdk.Tools.Cli.Commands; +using Azure.Sdk.Tools.Cli.Contract; +using Azure.Sdk.Tools.Cli.Helpers; +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Services; +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.TeamFoundation.TestManagement.WebApi; +using Microsoft.VisualStudio.Services.OAuth; +using Microsoft.VisualStudio.Services.TestResults.WebApi; +using Microsoft.VisualStudio.Services.WebApi; +using ModelContextProtocol.Server; + +namespace Azure.Sdk.Tools.Cli.Tools; + +[McpServerToolType, Description("Fetches data from an Azure Pipelines run.")] +public class PipelineAnalysisTool : MCPTool +{ + private readonly IAzureService azureService; + private readonly IDevOpsService devopsService; + private readonly IAzureAgentServiceFactory azureAgentServiceFactory; + private readonly ILogAnalysisHelper logAnalysisHelper; + private ITestHelper testHelper; + private readonly IOutputService output; + private readonly ILogger logger; + + private const string PUBLIC_PROJECT = "public"; + private const string INTERNAL_PROJECT = "internal"; + + private IAzureAgentService azureAgentService; + private TokenUsageHelper usage; + private bool initialized = false; + + private readonly HttpClient httpClient = new(); + private BuildHttpClient _buildClient; + private BuildHttpClient buildClient + { + get + { + Initialize(); + return _buildClient; + } + } + private TestResultsHttpClient _testClient; + private TestResultsHttpClient testClient + { + get + { + Initialize(); + return _testClient; + } + } + + // Options + private readonly Argument buildIdArg = new("Pipeline/Build ID"); + private readonly Option logIdOpt = new(["--log-id"], "ID of the pipeline task log"); + private readonly Option projectOpt = new(["--project", "-p"], "Pipeline project name"); + private readonly Option analyzeWithAgentOpt = new(["--agent", "-a"], () => false, "Analyze logs with RAG via upstream ai agent"); + private readonly Option projectEndpointOpt = new(["--ai-endpoint", "-e"], "The ai foundry project endpoint for the Azure AI Agent service"); + private readonly Option aiModelOpt = new(["--ai-model"], "The model to use for the Azure AI Agent"); + + public PipelineAnalysisTool( + IAzureService azureService, + IDevOpsService devopsService, + IAzureAgentServiceFactory azureAgentServiceFactory, + ILogAnalysisHelper logAnalysisHelper, + ITestHelper testHelper, + IOutputService output, + ILogger logger + ) : base() + { + this.azureService = azureService; + this.devopsService = devopsService; + this.azureAgentServiceFactory = azureAgentServiceFactory; + this.logAnalysisHelper = logAnalysisHelper; + this.testHelper = testHelper; + this.output = output; + this.logger = logger; + + CommandHierarchy = + [ + SharedCommandGroups.AzurePipelines // azsdk azp + ]; + } + + public override Command GetCommand() + { + var analyzePipelineCommand = new Command("analyze", "Analyze a pipeline run") { + buildIdArg, projectOpt, logIdOpt, analyzeWithAgentOpt, projectEndpointOpt, aiModelOpt + }; + analyzePipelineCommand.SetHandler(async ctx => { await HandleCommand(ctx, ctx.GetCancellationToken()); }); + + return analyzePipelineCommand; + } + + public override async Task HandleCommand(InvocationContext ctx, CancellationToken ct) + { + var buildId = ctx.ParseResult.GetValueForArgument(buildIdArg); + var project = ctx.ParseResult.GetValueForOption(projectOpt); + + var logId = ctx.ParseResult.GetValueForOption(logIdOpt); + var analyzeWithAgent = ctx.ParseResult.GetValueForOption(analyzeWithAgentOpt); + var projectEndpoint = ctx.ParseResult.GetValueForOption(projectEndpointOpt); + var aiModel = ctx.ParseResult.GetValueForOption(aiModelOpt); + + logger.LogInformation("Analyzing pipeline {buildId}...", buildId); + azureAgentService = azureAgentServiceFactory.Create(projectEndpoint, aiModel); + + if (logId != 0) + { + var result = await AnalyzePipelineFailureLogs(project, buildId, [logId], analyzeWithAgent, ct); + ctx.ExitCode = ExitCode; + usage?.LogCost(); + output.Output(result); + } + else + { + var result = await AnalyzePipeline(project, buildId, analyzeWithAgent, ct); + ctx.ExitCode = ExitCode; + usage?.LogCost(); + output.Output(result); + } + } + + private void Initialize(bool auth = true) + { + if (initialized) + { + return; + } + + if (auth) + { + var tokenScope = new[] { "499b84ac-1321-427f-aa17-267ca6975798/.default" }; // Azure DevOps scope + var msftCorpTenant = "72f988bf-86f1-41af-91ab-2d7cd011db47"; + var token = azureService.GetCredential(msftCorpTenant).GetToken(new TokenRequestContext(tokenScope)); + var tokenCredential = new VssOAuthAccessTokenCredential(token.Token); + var connection = new VssConnection(new Uri($"https://dev.azure.com/azure-sdk"), tokenCredential); + _buildClient = connection.GetClient(); + _testClient = connection.GetClient(); + } + else + { + var connection = new VssConnection(new Uri($"https://dev.azure.com/azure-sdk"), null); + _buildClient = connection.GetClient(); + _testClient = connection.GetClient(); + } + + initialized = true; + } + + private async Task GetPipelineProject(int buildId, string? project = null) + { + if (project == PUBLIC_PROJECT || string.IsNullOrEmpty(project)) + { + var pipelineUrl = $"https://dev.azure.com/azure-sdk/{PUBLIC_PROJECT}/_apis/build/builds/{buildId}?api-version=7.1"; + logger.LogDebug("Getting pipeline details from {url} via http", pipelineUrl); + var response = await httpClient.GetAsync(pipelineUrl); + // If project is not specified, try both public and internal projects + if (string.IsNullOrEmpty(project) && !response.IsSuccessStatusCode) + { + return await GetPipelineProject(buildId, INTERNAL_PROJECT); + } + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadAsStringAsync(); + using var doc = JsonDocument.Parse(json); + var projectName = doc.RootElement.GetProperty("project").GetProperty("name").GetString(); + if (string.IsNullOrEmpty(projectName)) + { + throw new Exception($"Failed to parse project name from build details for build {buildId}"); + } + return projectName; + } + + var _pipelineUrl = $"https://dev.azure.com/azure-sdk/{project}/_apis/build/builds/{buildId}?api-version=7.1"; + logger.LogDebug("Getting pipeline details from {url} via sdk", _pipelineUrl); + var build = await buildClient.GetBuildAsync(project, buildId); + return build.Project.Name; + } + + public async Task> GetPipelineFailureLogIds(string project, int buildId, CancellationToken ct = default) + { + logger.LogDebug("Getting pipeline task failures for {project} {buildId}", project, buildId); + + if (project != PUBLIC_PROJECT) + { + var timeline = await buildClient.GetBuildTimelineAsync(project, buildId, cancellationToken: ct); + var _failedTasks = timeline.Records.Where( + r => r.Result == TaskResult.Failed + && r.RecordType == "Task" + && !IsTestStep(r.Name)) + .ToList(); + logger.LogDebug("Found {count} failed tasks", _failedTasks.Count); + return _failedTasks.Select(t => t.Log?.Id ?? 0).Where(id => id != 0).Distinct().ToList(); + } + + var timelineUrl = $"https://dev.azure.com/azure-sdk/{project}/_apis/build/builds/{buildId}/timeline?api-version=7.1"; + logger.LogDebug("Getting timeline records from {url}", timelineUrl); + var response = await httpClient.GetAsync(timelineUrl, ct); + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadAsStringAsync(ct); + if (string.IsNullOrEmpty(json)) + { + throw new Exception($"No timeline records found for build {buildId} in project {project}"); + } + + using var doc = JsonDocument.Parse(json); + var failedTasks = doc.RootElement.GetProperty("records") + .EnumerateArray() + .Where(r => + r.GetProperty("result").GetString() == "failed" && + r.GetProperty("type").GetString() == "Task" && + !IsTestStep(r.GetProperty("name").GetString())).ToList(); + + List logIds = []; + foreach (var task in failedTasks) + { + if (task.TryGetProperty("log", out var logProp) && logProp.TryGetProperty("id", out var idProp)) + { + var id = idProp.GetInt32(); + if (id != 0) + { + logIds.Add(id); + } + } + } + + logger.LogDebug("Found {count} failed tasks", failedTasks.Count); + return logIds; + } + + public async Task> GetPipelineFailedTestResults(string project, int buildId, CancellationToken ct = default) + { + try + { + logger.LogDebug("Getting pipeline failed test results for {project} {buildId}", project, buildId); + var results = new List(); + + var testRuns = await testClient.GetTestResultsByPipelineAsync(project, buildId, cancellationToken: ct); + results.AddRange(testRuns); + while (testRuns.ContinuationToken != null) + { + var nextResults = await testClient.GetTestResultsByPipelineAsync(project, buildId, continuationToken: testRuns.ContinuationToken, cancellationToken: ct); + results.AddRange(nextResults); + testRuns.ContinuationToken = nextResults.ContinuationToken; + } + + var failedRuns = results.Where( + r => r.Outcome == TestOutcome.Failed.ToString() + || r.Outcome == TestOutcome.Aborted.ToString()) + .Select(r => r.RunId) + .Distinct() + .ToList(); + + logger.LogDebug("Getting test results for {count} failed test runs", failedRuns.Count); + + var failedRunData = new List(); + + foreach (var runId in failedRuns) + { + var testCases = await testClient.GetTestResultsAsync( + project, + runId, + outcomes: [TestOutcome.Failed, TestOutcome.Aborted], + cancellationToken: ct); + + foreach (var tc in testCases) + { + failedRunData.Add(new FailedTestRunResponse + { + RunId = runId, + TestCaseTitle = tc.TestCaseTitle, + ErrorMessage = tc.ErrorMessage, + StackTrace = tc.StackTrace, + Outcome = tc.Outcome, + Uri = tc.Url + }); + } + } + + return failedRunData; + } + catch (Exception ex) + { + logger.LogError("Failed to get pipeline failed test results {buildId}: {exception}", buildId, ex.Message); + logger.LogError("Stack Trace:"); + logger.LogError("{stackTrace}", ex.StackTrace); + SetFailure(); + return + [ + new FailedTestRunResponse() + { + ResponseError = $"Failed to get pipeline failed test results {buildId}: {ex.Message}", + } + ]; + } + } + + public async Task GetBuildLogLinesUnauthenticated(string project, int buildId, int logId, CancellationToken ct = default) + { + var logUrl = $"https://dev.azure.com/azure-sdk/{project}/_apis/build/builds/{buildId}/logs/{logId}?api-version=7.1"; + logger.LogDebug("Fetching log file from {url}", logUrl); + var response = await httpClient.GetAsync(logUrl, ct); + response.EnsureSuccessStatusCode(); + var logContent = await response.Content.ReadAsStringAsync(ct); + return logContent; + } + + public async Task AnalyzePipelineFailureLogs(string? project, int buildId, List logIds, bool analyzeWithAgent, CancellationToken ct) + { + try + { + project ??= PUBLIC_PROJECT; + var session = $"{project}-{buildId}"; + List logs = []; + + foreach (var logId in logIds) + { + string logText; + logger.LogDebug("Downloading pipeline failure log for {project} {buildId} {logId}", project, buildId, logId); + + if (project == PUBLIC_PROJECT) + { + logText = await GetBuildLogLinesUnauthenticated(project, buildId, logId, ct); + } + else + { + var logContent = await buildClient.GetBuildLogLinesAsync(project, buildId, logId, cancellationToken: ct); + logText = string.Join("\n", logContent); + } + + var tempPath = Path.GetTempFileName() + ".txt"; + logger.LogDebug("Writing log id {logId} to temporary file {tempPath}", logId, tempPath); + await File.WriteAllTextAsync(tempPath, logText, ct); + var filename = $"{session}-{logId}.txt"; + logs.Add(tempPath); + } + + if (!analyzeWithAgent) + { + LogAnalysisResponse response = new() { Errors = [] }; + foreach (var log in logs) + { + var localLogResult = await logAnalysisHelper.AnalyzeLogContent(log, null, null, null); + response.Errors.AddRange(localLogResult); + } + return response; + } + + var (result, _usage) = await azureAgentService.QueryFiles(logs, session, "Why did this pipeline fail?", ct); + // Sometimes chat gpt likes to wrap the json in markdown + if (result.StartsWith("```json") + && result.EndsWith("```")) + { + result = result[7..^3].Trim(); + } + + foreach (var log in logs) + { + File.Delete(log); + } + + + usage = usage != null ? usage + _usage : _usage; + + try + { + return JsonSerializer.Deserialize(result); + } + catch (JsonException ex) + { + logger.LogError("Failed to deserialize log analysis response: {exception}", ex.Message); + logger.LogError("Response:\n{result}", result); + + SetFailure(); + + return new LogAnalysisResponse() + { + ResponseError = "Failed to deserialize log analysis response. Check the logs for more details.", + }; + } + } + catch (Exception ex) + { + logger.LogError("Failed to analyze pipeline {buildId}: {error}", buildId, ex.Message); + logger.LogError("Stack Trace:"); + logger.LogError("{stackTrace}", ex.StackTrace); + SetFailure(); + return new LogAnalysisResponse() + { + ResponseError = $"Failed to analyze pipeline {buildId}: {ex.Message}", + }; + } + } + + [McpServerTool, Description("Analyze azure pipeline for failures. Set analyzeWithAgent to false unless requested otherwise by the user")] + public async Task AnalyzePipeline(int buildId, bool analyzeWithAgent, CancellationToken ct) + { + try + { + return await AnalyzePipeline(null, buildId, analyzeWithAgent, ct); + } + catch (Exception ex) + { + logger.LogError("Failed to analyze pipeline {buildId}: {exception}", buildId, ex.Message); + logger.LogError("Stack Trace:"); + logger.LogError("{stackTrace}", ex.StackTrace); + SetFailure(); + return new AnalyzePipelineResponse() + { + ResponseError = $"Failed to analyze pipeline {buildId}: {ex.Message}", + }; + } + } + + public async Task AnalyzePipeline(string? project, int buildId, bool analyzeWithAgent, CancellationToken ct) + { + try + { + if (string.IsNullOrEmpty(project)) + { + project = await GetPipelineProject(buildId, project); + } + + var failureLogIds = await GetPipelineFailureLogIds(project, buildId, ct); + var analysis = await AnalyzePipelineFailureLogs(project, buildId, failureLogIds, analyzeWithAgent, ct); + + List failedTests = []; + var failedTestArtifacts = await devopsService.GetPipelineLlmArtifacts(project, buildId); + + foreach (var testFiles in failedTestArtifacts) + { + foreach (var file in testFiles.Value) + { + var failed = await testHelper.GetFailedTestCases(file); + failedTests.AddRange(failed); + } + } + + var failedTestsByUri = failedTests + .GroupBy(ft => ft.Uri) + .ToDictionary( + g => g.Key, + g => g.Select(ft => ft.TestCaseTitle).ToList() + ); + + return new AnalyzePipelineResponse() + { + FailedTasks = [analysis], + FailedTests = failedTestsByUri + }; + } + catch (Exception ex) + { + logger.LogError("Failed to analyze pipeline {buildId}: {exception}", buildId, ex.Message); + logger.LogError("Stack Trace:"); + logger.LogError("{stackTrace}", ex.StackTrace); + SetFailure(); + return new AnalyzePipelineResponse() + { + ResponseError = $"Failed to analyze pipeline {buildId}: {ex.Message}", + }; + } + } + + public bool IsTestStep(string stepName) + { + return stepName.Contains("test", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineDetailsTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineDetailsTool.cs index 94bc39c396c..99de657b5f5 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineDetailsTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineDetailsTool.cs @@ -3,11 +3,10 @@ using System.CommandLine; using System.CommandLine.Invocation; using System.ComponentModel; -using System.Text; -using System.Text.Json; using Azure.Core; using Azure.Sdk.Tools.Cli.Commands; using Azure.Sdk.Tools.Cli.Contract; +using Azure.Sdk.Tools.Cli.Models; using Azure.Sdk.Tools.Cli.Services; using Microsoft.TeamFoundation.Build.WebApi; using Microsoft.VisualStudio.Services.OAuth; @@ -27,7 +26,7 @@ public class PipelineDetailsTool : MCPTool private ILogger logger; // Options - private readonly Option buildIdOpt = new(["--build-id", "-b"], "Pipeline/Build ID") { IsRequired = true }; + private readonly Argument buildIdArg = new("Pipeline/Build ID"); private readonly Option projectOpt = new(["--project", "-p"], "Pipeline project name"); public PipelineDetailsTool( @@ -48,7 +47,7 @@ ILogger logger public override Command GetCommand() { - var pipelineRunCommand = new Command("pipeline", "Get details for a pipeline run") { buildIdOpt, projectOpt }; + var pipelineRunCommand = new Command("pipeline", "Get details for a pipeline run") { buildIdArg, projectOpt }; pipelineRunCommand.SetHandler(async ctx => { await HandleCommand(ctx, ctx.GetCancellationToken()); }); return pipelineRunCommand; @@ -59,11 +58,11 @@ public override async Task HandleCommand(InvocationContext ctx, CancellationToke Initialize(); var cmd = ctx.ParseResult.CommandResult.Command.Name; - var buildId = ctx.ParseResult.GetValueForOption(buildIdOpt); + var buildId = ctx.ParseResult.GetValueForArgument(buildIdArg); var project = ctx.ParseResult.GetValueForOption(projectOpt); logger.LogInformation("Getting pipeline run {buildId}...", buildId); - var result = await GetPipelineRun(project, buildId); + var result = await GetPipelineRun(buildId, project); ctx.ExitCode = ExitCode; output.Output(result); } @@ -81,24 +80,36 @@ private void Initialize() buildClient = connection.GetClient(); } - [McpServerTool, Description("Gets details for a pipeline run")] - public async Task GetPipelineRun(string? project, int buildId) + [McpServerTool, Description("Gets details for a pipeline run. Do not specify project unless asked")] + public async Task GetPipelineRun(int buildId, string? project = null) { try { - var _project = project ?? "public"; - logger.LogDebug("Getting pipeline run for {project} {buildId}", _project, buildId); - var build = await buildClient.GetBuildAsync(_project, buildId); - return build; + if (!string.IsNullOrEmpty(project)) + { + logger.LogDebug("Getting pipeline run for {project} {buildId}", project, buildId); + var build = await buildClient.GetBuildAsync(project, buildId); + return new ObjectCommandResponse { Result = build }; + } } catch (Exception ex) { - if (!string.IsNullOrEmpty(project)) + logger.LogError(ex, "Failed to get build {buildId} in project {project}", buildId, project); + return new ObjectCommandResponse { - throw new Exception($"Failed to find build {buildId} in project 'public': {ex.Message}"); - } - // If project is not specified, try both azure sdk public and internal devops projects - return await GetPipelineRun("internal", buildId); + ResponseError = $"Failed to get build {buildId} in project {project}: {ex.Message}" + }; + } + + try + { + var build = await buildClient.GetBuildAsync("public", buildId); + return new ObjectCommandResponse { Result = build }; + } + catch (Exception) + { + var build = await buildClient.GetBuildAsync("internal", buildId); + return new ObjectCommandResponse { Result = build }; } } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineTestsTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineTestsTool.cs new file mode 100644 index 00000000000..5e11d72f388 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/AzurePipelines/PipelineTestsTool.cs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System.CommandLine; +using System.CommandLine.Invocation; +using System.ComponentModel; +using Azure.Core; +using Azure.Sdk.Tools.Cli.Commands; +using Azure.Sdk.Tools.Cli.Contract; +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Services; +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.VisualStudio.Services.OAuth; +using Microsoft.VisualStudio.Services.WebApi; +using ModelContextProtocol.Server; + +namespace Azure.Sdk.Tools.Cli.Tools; + +[McpServerToolType, Description("Fetches test data from Azure Pipelines")] +public class PipelineTestsTool : MCPTool +{ + private BuildHttpClient buildClient; + private readonly bool initialized = false; + + private IAzureService azureService; + private IDevOpsService devopsService; + private IOutputService output; + private ILogger logger; + + private readonly Argument buildIdArg = new("Pipeline/Build ID"); + + private const string PUBLIC_PROJECT = "public"; + + public PipelineTestsTool( + IAzureService azureService, + IDevOpsService devopsService, + IOutputService output, + ILogger logger + ) : base() + { + this.azureService = azureService; + this.devopsService = devopsService; + this.output = output; + this.logger = logger; + + CommandHierarchy = + [ + SharedCommandGroups.AzurePipelines // azsdk azp + ]; + } + + public override Command GetCommand() + { + var testResultsCommand = new Command("test-results", "Get test results for a pipeline run") { buildIdArg }; + testResultsCommand.SetHandler(async ctx => { await HandleCommand(ctx, ctx.GetCancellationToken()); }); + + return testResultsCommand; + } + + public override async Task HandleCommand(InvocationContext ctx, CancellationToken ct) + { + Initialize(); + var buildId = ctx.ParseResult.GetValueForArgument(buildIdArg); + + logger.LogInformation("Getting test results for pipeline {buildId}...", buildId); + var result = await GetPipelineLlmArtifacts(buildId); + ctx.ExitCode = ExitCode; + output.Output(result); + } + + private void Initialize() + { + if (initialized) + { + return; + } + var tokenScope = new[] { "499b84ac-1321-427f-aa17-267ca6975798/.default" }; // Azure DevOps scope + var token = azureService.GetCredential().GetToken(new TokenRequestContext(tokenScope)); + var tokenCredential = new VssOAuthAccessTokenCredential(token.Token); + var connection = new VssConnection(new Uri($"https://dev.azure.com/azure-sdk"), tokenCredential); + buildClient = connection.GetClient(); + } + + [McpServerTool, Description("Downloads artifacts intended for LLM analysis from a pipeline run")] + public async Task GetPipelineLlmArtifacts(int buildId) + { + string project = ""; + try + { + var build = await GetPipelineRun(buildId); + project = build.Project.Name; + logger.LogInformation("Fetching artifacts for build {buildId} in project {project}", buildId, project); + var result = await devopsService.GetPipelineLlmArtifacts(project, buildId); + return new ObjectCommandResponse { Result = result }; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to get pipeline artifacts for build {buildId} in project {project}", buildId, project); + SetFailure(); + return new ObjectCommandResponse + { + ResponseError = $"Failed to get pipeline artifacts for build {buildId} in project {project}", + }; + } + } + + private async Task GetPipelineRun(int buildId, string? project = null) + { + if (!string.IsNullOrEmpty(project)) + { + return await buildClient.GetBuildAsync(project, buildId); + } + try + { + return await buildClient.GetBuildAsync("public", buildId); + } + catch (Exception) + { + return await buildClient.GetBuildAsync("internal", buildId); + } + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/HostServer/HostServerTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/HostServer/HostServerTool.cs index 6d6e088d81c..9e16d53da06 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/HostServer/HostServerTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/HostServer/HostServerTool.cs @@ -19,7 +19,7 @@ public HostServerTool(ILogger logger) public override Command GetCommand() { - Command command = new Command("start"); + Command command = new Command("mcp", "Run in MCP (model context protocol) server mode"); command.SetHandler(async ctx => { diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Test/TestAnalysisTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Test/TestAnalysisTool.cs new file mode 100644 index 00000000000..1fad3b901d3 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Test/TestAnalysisTool.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System.CommandLine; +using System.CommandLine.Invocation; +using System.ComponentModel; +using System.Xml; +using Azure.Sdk.Tools.Cli.Contract; +using Azure.Sdk.Tools.Cli.Helpers; +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Services; +using ModelContextProtocol.Server; + +namespace Azure.Sdk.Tools.Cli.Tools; + +[McpServerToolType, Description("Processes and analyzes test results from TRX files")] +public class TestAnalysisTool(ITestHelper testHelper, IOutputService output, ILogger logger) : MCPTool() +{ + // Options + private readonly Option trxPathOpt = new(["--trx-file"], "Path to the TRX file for failed test runs") { IsRequired = true }; + private readonly Option filterOpt = new(["--filter-title"], "Test case title to filter results"); + private readonly Option titlesOpt = new(["--titles"], "Only return test case titles, not full details"); + + public override Command GetCommand() + { + var analyzeTestCommand = new Command("test-results", "Analyze test results") { + trxPathOpt, filterOpt, titlesOpt + }; + analyzeTestCommand.SetHandler(async ctx => { await HandleCommand(ctx, ctx.GetCancellationToken()); }); + return analyzeTestCommand; + } + + public override async Task HandleCommand(InvocationContext ctx, CancellationToken ct) + { + var cmd = ctx.ParseResult.CommandResult.Command.Name; + var trxPath = ctx.ParseResult.GetValueForOption(trxPathOpt); + var filterTitle = ctx.ParseResult.GetValueForOption(filterOpt); + var titlesOnly = ctx.ParseResult.GetValueForOption(titlesOpt); + + if (titlesOnly) + { + var testTitles = await GetFailedTestCases(trxPath); + ctx.ExitCode = ExitCode; + output.Output(testTitles); + return; + } + + if (!string.IsNullOrEmpty(filterTitle)) + { + var testCase = await GetFailedTestCaseData(trxPath, filterTitle); + ctx.ExitCode = ExitCode; + output.Output(testCase); + return; + } + + var testResult = await GetFailedTestRunDataFromTrx(trxPath); + ctx.ExitCode = ExitCode; + output.Output(testResult); + return; + } + + [McpServerTool, Description("Get titles of failed test cases from a TRX file")] + public async Task> GetFailedTestCases(string trxFilePath) + { + try + { + return await testHelper.GetFailedTestCases(trxFilePath); + } + catch (Exception ex) + { + logger.LogError("Failed to process TRX file {trxFilePath}: {exception}", trxFilePath, ex.Message); + logger.LogError("Stack Trace: {stackTrace}", ex.StackTrace); + SetFailure(); + return []; + } + } + + [McpServerTool, Description("Get details for a failed test from a TRX file")] + public async Task GetFailedTestCaseData(string trxFilePath, string testCaseTitle) + { + try + { + var failedTestRuns = await testHelper.GetFailedTestRunDataFromTrx(trxFilePath); + var testRun = failedTestRuns.FirstOrDefault(run => run.TestCaseTitle.Equals(testCaseTitle, StringComparison.OrdinalIgnoreCase)); + if (testRun == null) + { + return new FailedTestRunResponse + { + ResponseError = $"No failed test run found for test case title: {testCaseTitle}" + }; + } + return testRun; + } + catch (Exception ex) + { + logger.LogError("Failed to process TRX file {trxFilePath}: {exception}", trxFilePath, ex.Message); + logger.LogError("Stack Trace: {stackTrace}", ex.StackTrace); + SetFailure(); + return new FailedTestRunResponse + { + ResponseError = $"Failed to process TRX file {trxFilePath}: {ex.Message}" + }; + } + } + + [McpServerTool, Description("Get failed test run data from a TRX file")] + public async Task> GetFailedTestRunDataFromTrx(string trxFilePath) + { + try + { + return await testHelper.GetFailedTestRunDataFromTrx(trxFilePath); + } + catch (Exception ex) + { + logger.LogError("Failed to process TRX file {trxFilePath}: {exception}", trxFilePath, ex.Message); + logger.LogError("Stack Trace: {stackTrace}", ex.StackTrace); + SetFailure(); + return + [ + new FailedTestRunResponse + { + ResponseError = $"Failed to process TRX file {trxFilePath}: {ex.Message}" + } + ]; + } + } +} diff --git a/tools/azsdk-cli/CONTRIBUTING.md b/tools/azsdk-cli/CONTRIBUTING.md new file mode 100644 index 00000000000..0b1802ca9ab --- /dev/null +++ b/tools/azsdk-cli/CONTRIBUTING.md @@ -0,0 +1,50 @@ +# Contributing to azsdk-cli + +## Prerequisites + +- [.NET 8.0 SDK or later](https://dotnet.microsoft.com/download) + +## Setup + +1. Clone the repository: + ```sh + git clone https://github.com/Azure/azure-sdk-tools.git + cd azure-sdk-tools/tools/azsdk-cli + ``` + +2. Restore dependencies: + ```sh + dotnet restore + ``` + +## Build + +To build the project: + +```sh +dotnet build +``` + +## Run + +To run the CLI locally: + +```sh +dotnet run --project Azure.Sdk.Tools.Cli -- --help +``` + +## Test + +To run the tests: + +```sh +dotnet test +``` + +## Package + +To create a self contained binary: + +```sh +dotnet publish -f net8.0 -c Release -r linux-x64 -p:PublishSingleFile=true --self-contained --output ./artifacts/linux-x64 ./tools/azsdk-cli/Azure.Sdk.Tools.Cli/Azure.Sdk.Tools.Cli.csproj +``` \ No newline at end of file