Add Azure DevOps extension to report errors

TestFx.sln

@@ -222,6 +222,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTest.Engine.UnitTests", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTest.SourceGeneration.UnitTests", "test\UnitTests\MSTest.SourceGeneration.UnitTests\MSTest.SourceGeneration.UnitTests.csproj", "{E6C0466E-BE8D-C04F-149A-FD98438F1413}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Testing.Extensions.AzureDevOps", "src\Platform\Microsoft.Testing.Extensions.AzureDevOps\Microsoft.Testing.Extensions.AzureDevOps.csproj", "{F608D3A3-125B-CD88-1D51-8714ED142029}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -524,6 +526,10 @@ Global
 		{E6C0466E-BE8D-C04F-149A-FD98438F1413}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E6C0466E-BE8D-C04F-149A-FD98438F1413}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{E6C0466E-BE8D-C04F-149A-FD98438F1413}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F608D3A3-125B-CD88-1D51-8714ED142029}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F608D3A3-125B-CD88-1D51-8714ED142029}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F608D3A3-125B-CD88-1D51-8714ED142029}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F608D3A3-125B-CD88-1D51-8714ED142029}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -615,6 +621,7 @@ Global
 		{7BA0E74E-798E-4399-2EDE-A23BD5DA78CA} = {E7F15C9C-3928-47AD-8462-64FD29FFCA54}
 		{2C0DFAC0-5D58-D172-ECE4-CBB78AD03435} = {BB874DF1-44FE-415A-B634-A6B829107890}
 		{E6C0466E-BE8D-C04F-149A-FD98438F1413} = {BB874DF1-44FE-415A-B634-A6B829107890}
+		{F608D3A3-125B-CD88-1D51-8714ED142029} = {6AEE1440-FDF0-4729-8196-B24D0E333550}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9}

eng/Versions.props

@@ -1,9 +1,9 @@
 <Project>
   <PropertyGroup Label="Version settings">
     <!-- MSTest version -->
-    <VersionPrefix>3.9.0</VersionPrefix>
+    <VersionPrefix>3.10.0</VersionPrefix>
     <!-- Testing Platform version -->
-    <TestingPlatformVersionPrefix>1.7.0</TestingPlatformVersionPrefix>
+    <TestingPlatformVersionPrefix>1.8.0</TestingPlatformVersionPrefix>
     <PreReleaseVersionLabel>preview</PreReleaseVersionLabel>
   </PropertyGroup>
   <PropertyGroup Label="MSTest prod dependencies - darc updated">

src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzDoEscaper.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Platform;
+
+namespace Microsoft.Testing.Extensions.Reporting;
+
+internal static class AzDoEscaper
+{
+    public static string Escape(string value)
+    {
+        if (RoslynString.IsNullOrEmpty(value))
+        {
+            return value;
+        }
+
+        var result = new StringBuilder(value.Length);
+        foreach (char c in value)
+        {
+            switch (c)
+            {
+                case ';':
+                    result.Append("%3B");
+                    break;
+                case '\r':
+                    result.Append("%0D");
+                    break;
+                case '\n':
+                    result.Append("%0A");
+                    break;
+                default:
+                    result.Append(c);
+                    break;
+            }
+        }
+
+        return result.ToString();
+    }
+}

src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsCommandLineOptions.cs

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Testing.Extensions.Reporting;
+
+internal static class AzureDevOpsCommandLineOptions
+{
+    public const string AzureDevOpsOptionName = "report-azdo";
+    public const string AzureDevOpsReportSeverity = "report-azdo-severity";
+}

src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsCommandLineProvider.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Extensions.AzureDevOps.Resources;
+using Microsoft.Testing.Platform.CommandLine;
+using Microsoft.Testing.Platform.Extensions;
+using Microsoft.Testing.Platform.Extensions.CommandLine;
+using Microsoft.Testing.Platform.Helpers;
+
+namespace Microsoft.Testing.Extensions.Reporting;
+
+internal sealed class AzureDevOpsCommandLineProvider : ICommandLineOptionsProvider
+{
+    private static readonly string[] SeverityOptions = ["error", "warning"];
+
+    public string Uid => nameof(AzureDevOpsCommandLineProvider);
+
+    public string Version => AppVersion.DefaultSemVer;
+
+    public string DisplayName => AzureDevOpsResources.DisplayName;
+
+    public string Description => AzureDevOpsResources.Description;
+
+    public Task<bool> IsEnabledAsync() => Task.FromResult(true);
+
+    public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
+        =>
+        [
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName, AzureDevOpsResources.OptionDescription, ArgumentArity.Zero, false),
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false),
+        ];
+
+    public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
+    {
+        if (commandOption.Name == AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity)
+        {
+            if (!SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase))
+            {
+                return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
+            }
+        }
+
+        return ValidationResult.ValidTask;
+    }
+
+    public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
+        => ValidationResult.ValidTask;
+}

src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsExtensions.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Extensions.Reporting;
+using Microsoft.Testing.Extensions.TrxReport.Abstractions;
+using Microsoft.Testing.Platform.Builder;
+using Microsoft.Testing.Platform.Extensions;
+using Microsoft.Testing.Platform.Services;
+
+namespace Microsoft.Testing.Extensions;
+
+/// <summary>
+/// Provides extension methods for adding Azure DevOps reporting support to the test application builder.
+/// </summary>
+public static class AzureDevOpsExtensions
+{
+    /// <summary>
+    /// Adds  support to the test application builder.
+    /// </summary>
+    /// <param name="builder">The test application builder.</param>
+    public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
+    {
+        var compositeTestSessionAzDoService =
+           new CompositeExtensionFactory<AzureDevOpsReporter>(serviceProvider =>
+               new AzureDevOpsReporter(
+                   serviceProvider.GetCommandLineOptions(),
+                   serviceProvider.GetEnvironment(),
+                   serviceProvider.GetOutputDevice()));
+
+        builder.TestHost.AddDataConsumer(compositeTestSessionAzDoService);
+
+        builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
+    }
+}

src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsReporter.cs

@@ -0,0 +1,167 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Extensions.AzureDevOps;
+using Microsoft.Testing.Extensions.AzureDevOps.Resources;
+using Microsoft.Testing.Extensions.Reporting;
+using Microsoft.Testing.Platform;
+using Microsoft.Testing.Platform.CommandLine;
+using Microsoft.Testing.Platform.Extensions.Messages;
+using Microsoft.Testing.Platform.Extensions.OutputDevice;
+using Microsoft.Testing.Platform.Extensions.TestHost;
+using Microsoft.Testing.Platform.Helpers;
+using Microsoft.Testing.Platform.OutputDevice;
+
+namespace Microsoft.Testing.Extensions.TrxReport.Abstractions;
+
+internal sealed class AzureDevOpsReporter :
+    IDataConsumer,
+    IDataProducer,
+    IOutputDeviceDataProducer
+{
+    private readonly IOutputDevice _outputDisplay;
+
+    private static readonly char[] NewlineCharacters = new char[] { '\r', '\n' };
+    private readonly ICommandLineOptions _commandLine;
+    private readonly IEnvironment _environment;
+    private string _severity = "error";
+
+    public AzureDevOpsReporter(
+        ICommandLineOptions commandLine,
+        IEnvironment environment,
+        IOutputDevice outputDisplay)
+    {
+        _commandLine = commandLine;
+        _environment = environment;
+        _outputDisplay = outputDisplay;
+    }
+
+    public Type[] DataTypesConsumed { get; } =
+    [
+        typeof(TestNodeUpdateMessage)
+    ];
+
+    public Type[] DataTypesProduced { get; } = [typeof(SessionFileArtifact)];
+
+    /// <inheritdoc />
+    public string Uid { get; } = nameof(AzureDevOpsReporter);
+
+    /// <inheritdoc />
+    public string Version { get; } = AppVersion.DefaultSemVer;
+
+    /// <inheritdoc />
+    public string DisplayName { get; } = AzureDevOpsResources.DisplayName;
+
+    /// <inheritdoc />
+    public string Description { get; } = AzureDevOpsResources.Description;
+
+    /// <inheritdoc />
+    public Task<bool> IsEnabledAsync()
+    {
+        bool isEnabled = _commandLine.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName)
+            && string.Equals(_environment.GetEnvironmentVariable("TF_BUILD"), "true", StringComparison.OrdinalIgnoreCase);
+
+        if (isEnabled)
+        {
+            bool found = _commandLine.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, out string[]? arguments);
+            if (found && arguments?.Length > 0)
+            {
+                _severity = arguments[0].ToLowerInvariant();
+            }
+        }
+
+        return Task.FromResult(isEnabled);
+    }
+
+    public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken)
+    {
+        if (cancellationToken.IsCancellationRequested)
+        {
+            return;
+        }
+
+        if (value is not TestNodeUpdateMessage nodeUpdateMessage)
+        {
+            return;
+        }
+
+        TestNodeStateProperty nodeState = nodeUpdateMessage.TestNode.Properties.Single<TestNodeStateProperty>();
+
+        switch (nodeState)
+        {
+            case FailedTestNodeStateProperty failed:
+                await WriteExceptionAsync(failed.Explanation, failed.Exception);
+                break;
+            case ErrorTestNodeStateProperty error:
+                await WriteExceptionAsync(error.Explanation, error.Exception);
+                break;
+            case CancelledTestNodeStateProperty cancelled:
+                await WriteExceptionAsync(cancelled.Explanation, cancelled.Exception);
+                break;
+            case TimeoutTestNodeStateProperty timeout:
+                await WriteExceptionAsync(timeout.Explanation, timeout.Exception);
+                break;
+        }
+
+        return;
+    }
+
+    private async Task WriteExceptionAsync(string? explanation, Exception? exception)
+    {
+        if (exception == null || exception.StackTrace == null)
+        {
+            return;
+        }
+
+        string message = explanation ?? exception.Message;
+
+        if (message == null)
+        {
+            return;
+        }
+
+        string stackTrace = exception.StackTrace;
+        int index = stackTrace.IndexOfAny(NewlineCharacters);
+        string firstLine = index == -1 ? stackTrace : stackTrace.Substring(0, index);
+        if (firstLine != null)
+        {
+            (string Code, string File, int LineNumber)? location = GetStackFrameLocation(firstLine);
+            if (location != null)
+            {
+                string root = RootFinder.Find();
+                string file = location.Value.File;
+                string relativePath = file.StartsWith(root, StringComparison.CurrentCultureIgnoreCase) ? file.Substring(root.Length) : file;
+                string relativeNormalizedPath = relativePath.Replace('\\', '/');
+
+                string err = AzDoEscaper.Escape(message);
+
+                string line = $"##vso[task.logissue type={_severity};sourcepath={relativeNormalizedPath};linenumber={location.Value.LineNumber};columnnumber=1]{err}";
+                await _outputDisplay.DisplayAsync(this, new FormattedTextOutputDeviceData(line));
+            }
+        }
+    }
+
+    internal /* for testing */ static (string Code, string File, int LineNumber)? GetStackFrameLocation(string stackTraceLine)
+    {
+        Match match = StackTraceHelper.GetFrameRegex().Match(stackTraceLine);
+        if (!match.Success)
+        {
+            return null;
+        }
+
+        bool weHaveFilePathAndCodeLine = !RoslynString.IsNullOrWhiteSpace(match.Groups["code"].Value);
+        if (!weHaveFilePathAndCodeLine)
+        {
+            return null;
+        }
+
+        if (RoslynString.IsNullOrWhiteSpace(match.Groups["file"].Value))
+        {
+            return null;
+        }
+
+        int line = int.TryParse(match.Groups["line"].Value, out int value) ? value : 0;
+
+        return (match.Groups["code"].Value, match.Groups["file"].Value, line);
+    }
+}

src/Platform/Microsoft.Testing.Extensions.AzureDevOps/BannedSymbols.txt

@@ -0,0 +1,10 @@
+T:System.ArgumentNullException; Use 'ArgumentGuard' instead
+P:System.DateTime.Now; Use 'IClock' instead
+P:System.DateTime.UtcNow; Use 'IClock' instead
+M:System.Threading.Tasks.Task.Run(System.Action); Use 'ITask' instead
+M:System.Threading.Tasks.Task.WhenAll(System.Threading.Tasks.Task[]); Use 'ITask' instead
+M:System.Threading.Tasks.Task.WhenAll(System.Collections.Generic.IEnumerable{System.Threading.Tasks.Task}); Use 'ITask' instead
+M:System.String.IsNullOrEmpty(System.String); Use 'RoslynString.IsNullOrEmpty' instead
+M:System.String.IsNullOrWhiteSpace(System.String); Use 'RoslynString.IsNullOrWhiteSpace' instead
+M:System.Diagnostics.Debug.Assert(System.Boolean); Use 'RoslynDebug.Assert' instead
+M:System.Diagnostics.Debug.Assert(System.Boolean,System.String); Use 'RoslynDebug.Assert' instead