AgentRunner using ResultReporter

Author: CharliePoole

src/NUnitCommon/nunit.agent.core.tests/AgentDirectRunnerTests.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using System.Linq;
 using System.Text;
+using NUnit.Common;
 using NUnit.Framework;
 using NUnit.TestData.Assemblies;
 
@@ -28,6 +29,7 @@ private static void RunTestUnderTestBed(string testAssembly)
 #else
             string agentExe = Path.ChangeExtension(agentAssembly, ".exe");
 #endif
+            MockAssembly.DisplayCounts();
 
             var startInfo = new ProcessStartInfo(agentExe);
             startInfo.Arguments = testAssembly;
@@ -39,16 +41,10 @@ private static void RunTestUnderTestBed(string testAssembly)
             if (process is not null)
             {
                 process.WaitForExit();
-
-                string output = process.StandardOutput.ReadToEnd();
-                int index = output.IndexOf("Test Run Summary");
-                if (index > 0)
-                    output = output.Substring(index);
-
-                Console.WriteLine(output);
                 Console.WriteLine($"Agent process exited with rc={process.ExitCode}");
 
-                if (index < 0)
+                string output = process.StandardOutput.ReadToEnd();
+                if (!output.Contains("Test Run Summary"))
                     Assert.Fail("No Summary Report found");
             }
         }

src/NUnitCommon/nunit.agent.core.tests/Drivers/NUnitFrameworkDriverTests.cs

@@ -107,7 +107,7 @@ public void RunTestsAction_AfterLoad_ReturnsRunnableSuite()
             Assert.That(result.GetAttribute("runstate"), Is.EqualTo("Runnable"));
             Assert.That(result.GetAttribute("testcasecount"), Is.EqualTo(MockAssembly.Tests.ToString()));
             Assert.That(result.GetAttribute("result"), Is.EqualTo("Failed"));
-            Assert.That(result.GetAttribute("passed"), Is.EqualTo(MockAssembly.PassedInAttribute.ToString()));
+            Assert.That(result.GetAttribute("passed"), Is.EqualTo(MockAssembly.Passed.ToString()));
             Assert.That(result.GetAttribute("failed"), Is.EqualTo(MockAssembly.Failed.ToString()));
             Assert.That(result.GetAttribute("skipped"), Is.EqualTo(MockAssembly.Skipped.ToString()));
             Assert.That(result.GetAttribute("inconclusive"), Is.EqualTo(MockAssembly.Inconclusive.ToString()));

src/NUnitCommon/nunit.agent.core.tests/Runners/TestAgentRunnerTests.cs

@@ -114,7 +114,7 @@ private static void CheckRunResult(TestEngineResult result)
         private static void CheckRunResult(XmlNode result)
         {
             CheckBasicResult(result);
-            Assert.That(result.GetAttribute("passed", 0), Is.EqualTo(MockAssembly.PassedInAttribute));
+            Assert.That(result.GetAttribute("passed", 0), Is.EqualTo(MockAssembly.Passed));
             Assert.That(result.GetAttribute("failed", 0), Is.EqualTo(MockAssembly.Failed));
             Assert.That(result.GetAttribute("skipped", 0), Is.EqualTo(MockAssembly.Skipped));
             Assert.That(result.GetAttribute("inconclusive", 0), Is.EqualTo(MockAssembly.Inconclusive));

src/NUnitCommon/nunit.agent.core/AgentDirectRunner.cs

@@ -1,9 +1,7 @@
 // Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt
 
 using System;
-using System.Globalization;
 using System.IO;
-using System.Reflection;
 using System.Xml;
 using NUnit.TextDisplay;
 using NUnit.Engine;
@@ -17,7 +15,6 @@ public class AgentDirectRunner
         private AgentOptions _options;
 
         private ColorConsoleWriter OutWriter { get; } = new ColorConsoleWriter();
-        private int ReportIndex { get; set; }
 
         public AgentDirectRunner(AgentOptions options)
         {
@@ -30,7 +27,7 @@ public void ExecuteTestsDirectly()
             {
                 var testFile = _options.Files[0];
 
-                WriteHeader(OutWriter);
+                ResultReporter.WriteHeader(OutWriter);
 
                 TestPackage package = new TestPackage(testFile).SubPackages[0];
 
@@ -40,18 +37,9 @@ public void ExecuteTestsDirectly()
                 var runner = new LocalTestRunner(package);
 #endif
                 var xmlResult = runner.Run(null, TestFilter.Empty).Xml;
+                var summary = new ResultSummary(xmlResult);
 
-                WriteRunSettingsReport(xmlResult);
-
-                var result = xmlResult.GetAttribute("result");
-                if (result == "Failed" || result == "Warning")
-                    WriteErrorsFailuresAndWarningsReport(xmlResult);
-
-                int notrunCount = int.Parse(xmlResult.GetAttribute("skipped") ?? "0");
-                if (notrunCount > 0)
-                    WriteNotRunReport(xmlResult);
-
-                WriteSummaryReport(xmlResult);
+                ResultReporter.ReportResults(summary, OutWriter);
 
                 var pathToResultFile = Path.Combine(_options.WorkDirectory, "TestResult.xml");
                 WriteResultFile(xmlResult, pathToResultFile);
@@ -66,241 +54,6 @@ public void ExecuteTestsDirectly()
             Environment.Exit(AgentExitCodes.OK);
         }
 
-        private static void WriteHeader(ExtendedTextWriter outWriter)
-        {
-            var ea = Assembly.GetEntryAssembly();
-            if (ea is not null)
-            {
-                var title = GetAttribute<AssemblyTitleAttribute>(ea)?.Title;
-                var version = GetAttribute<AssemblyFileVersionAttribute>(ea)?.Version;
-                var copyright = GetAttribute<AssemblyCopyrightAttribute>(ea)?.Copyright;
-
-                outWriter.WriteLine(ColorStyle.Header, $"{title} {version}");
-                if (copyright is not null)
-                    outWriter.WriteLine(ColorStyle.SubHeader, copyright);
-                outWriter.WriteLine(ColorStyle.SubHeader, DateTime.Now.ToString(CultureInfo.CurrentCulture.DateTimeFormat.FullDateTimePattern));
-                outWriter.WriteLine();
-            }
-        }
-
-        private static TAttr? GetAttribute<TAttr>(Assembly assembly)
-            where TAttr : Attribute
-        {
-            return assembly?.GetCustomAttribute<TAttr>();
-        }
-
-        internal void WriteRunSettingsReport(XmlNode resultNode)
-        {
-            var settings = resultNode.SelectNodes("settings/setting");
-
-            if (settings is not null && settings.Count > 0)
-            {
-                OutWriter.WriteLine(ColorStyle.SectionHeader, "Run Settings");
-                settings.ForEachNode(WriteSettingsNode);
-                OutWriter.WriteLine();
-            }
-        }
-
-        private void WriteSettingsNode(XmlNode node)
-        {
-            var items = node.SelectNodes("item");
-            var name = node.GetAttribute("name");
-            var val = node.GetAttribute("value") ?? string.Empty;
-
-            if (items is not null)
-            {
-                OutWriter.WriteLabelLine($"    {name}:", items.Count > 0 ? string.Empty : $" {val}");
-
-                items.ForEachNode((item) =>
-                {
-                    var key = item.GetAttribute("key");
-                    var value = item.GetAttribute("value");
-                    OutWriter.WriteLine(ColorStyle.Value, $"        {key} -> {value}");
-                });
-            }
-        }
-
-        public void WriteSummaryReport(XmlNode resultNode)
-        {
-            var overallResult = resultNode.GetAttribute("result") ?? "Unknown";
-            if (overallResult == "Skipped")
-                overallResult = "Warning";
-
-            ColorStyle resultColor = overallResult == "Passed"
-                ? ColorStyle.Pass
-                : overallResult == "Failed" || overallResult == "Unknown"
-                    ? ColorStyle.Failure
-                    : overallResult == "Warning"
-                        ? ColorStyle.Warning
-                        : ColorStyle.Output;
-
-            OutWriter.WriteLine(ColorStyle.SectionHeader, "Test Run Summary");
-            OutWriter.WriteLabelLine(overallResult, resultColor);
-
-            int cases = resultNode.GetAttribute("testcasecount", 0);
-            int passed = resultNode.GetAttribute("passed", 0);
-            int failed = resultNode.GetAttribute("failed", 0);
-            int warnings = resultNode.GetAttribute("warnings", 0);
-            int inconclusive = resultNode.GetAttribute("inconclusive", 0);
-            int skipped = resultNode.GetAttribute("skipped", 0);
-
-            WriteSummaryCount("  Test Cases: ", cases);
-            WriteSummaryCount(", Passed: ", passed);
-            WriteSummaryCount(", Failed: ", failed, ColorStyle.Failure);
-            WriteSummaryCount(", Warnings: ", warnings, ColorStyle.Warning);
-            WriteSummaryCount(", Inconclusive: ", inconclusive);
-            WriteSummaryCount(", Skipped: ", skipped);
-            OutWriter.WriteLine();
-
-            var duration = resultNode.GetAttribute("duration", 0.0);
-            var startTime = resultNode.GetAttribute("start-time", DateTime.MinValue);
-            var endTime = resultNode.GetAttribute("end-time", DateTime.MaxValue);
-
-            OutWriter.WriteLabelLine("  Start time: ", startTime.ToString("u"));
-            OutWriter.WriteLabelLine("    End time: ", endTime.ToString("u"));
-            OutWriter.WriteLabelLine("    Duration: ", string.Format(NumberFormatInfo.InvariantInfo, "{0:0.000} seconds", duration));
-            OutWriter.WriteLine();
-        }
-
-        public void WriteErrorsFailuresAndWarningsReport(XmlNode resultNode)
-        {
-            ReportIndex = 0;
-            OutWriter.WriteLine(ColorStyle.SectionHeader, "Errors, Failures and Warnings");
-            OutWriter.WriteLine();
-
-            WriteErrorsFailuresAndWarnings(resultNode);
-        }
-
-        private void WriteErrorsFailuresAndWarnings(XmlNode resultNode)
-        {
-            const string OLD_NUNIT_CHILD_HAD_ERRORS_MESSAGE = "One or more child tests had errors";
-            const string OLD_NUNIT_CHILD_HAD_WARNINGS_MESSAGE = "One or more child tests had errors";
-
-            string resultState = resultNode.GetAttribute("result") ?? string.Empty;
-
-            switch (resultNode.Name)
-            {
-                case "test-case":
-                    if (resultState == "Failed" || resultState == "Warning")
-                        DisplayResultItem(resultNode);
-                    return;
-
-                // Not present in this agent but retain for use with other agents
-                case "test-run":
-                    resultNode.ForEachChildNode(WriteErrorsFailuresAndWarnings);
-                    break;
-
-                case "test-suite":
-                    if (resultState == "Failed" || resultState == "Warning")
-                    {
-                        var suiteType = resultNode.GetAttribute("type");
-                        if (suiteType == "Theory")
-                        {
-                            // Report failure of the entire theory and then go on
-                            // to list the individual cases that failed
-                            DisplayResultItem(resultNode);
-                        }
-                        else
-                        {
-                            // Where did this happen? Default is in the current test.
-                            var site = resultNode.GetAttribute("site");
-
-                            // Correct a problem in some framework versions, whereby warnings and some failures
-                            // are promulgated to the containing suite without setting the FailureSite.
-                            if (site is null)
-                            {
-                                if (resultNode.SelectSingleNode("reason/message")?.InnerText == OLD_NUNIT_CHILD_HAD_WARNINGS_MESSAGE ||
-                                    resultNode.SelectSingleNode("failure/message")?.InnerText == OLD_NUNIT_CHILD_HAD_ERRORS_MESSAGE)
-                                {
-                                    site = "Child";
-                                }
-                                else
-                                    site = "Test";
-                            }
-
-                            // Only report errors in the current test method, setup or teardown
-                            if (site == "SetUp" || site == "TearDown" || site == "Test")
-                            {
-                                DisplayResultItem(resultNode);
-                            }
-
-                            // Do not list individual "failed" tests after a one-time setup failure
-                            if (site == "SetUp")
-                                return;
-                        }
-                    }
-
-                    resultNode.ForEachChildNode(WriteErrorsFailuresAndWarnings);
-
-                    break;
-            }
-        }
-
-        public void WriteNotRunReport(XmlNode resultNode)
-        {
-            ReportIndex = 0;
-            OutWriter.WriteLine(ColorStyle.SectionHeader, "Tests Not Run");
-            OutWriter.WriteLine();
-            WriteNotRunResults(resultNode);
-        }
-
-        private void WriteNotRunResults(XmlNode resultNode)
-        {
-            switch (resultNode.Name)
-            {
-                case "test-case":
-                    string? status = resultNode.GetAttribute("result");
-
-                    if (status == "Skipped")
-                        DisplayResultItem(resultNode);
-
-                    break;
-
-                case "test-suite":
-                case "test-run":
-                    resultNode.ForEachChildNode(WriteNotRunResults);
-
-                    break;
-            }
-        }
-
-        private static readonly char[] EOL_CHARS = new char[] { '\r', '\n' };
-
-        private void DisplayResultItem(XmlNode resultNode)
-        {
-            string? resultState = resultNode.GetAttribute("result");
-            string? fullName = resultNode.GetAttribute("fullname");
-            string? message = (resultNode.SelectSingleNode("failure/message") ?? resultNode.SelectSingleNode("reason/message"))?.InnerText.Trim(EOL_CHARS);
-
-            OutWriter.WriteLine(GetColorStyle(),
-                string.Format($"{++ReportIndex}) {resultState} : {fullName}"));
-            if (!string.IsNullOrEmpty(message))
-                OutWriter.WriteLine(ColorStyle.Output, message);
-            var stackTrace = resultNode.SelectSingleNode("failure/stack-trace")?.InnerText;
-            if (!string.IsNullOrEmpty(stackTrace))
-                OutWriter.WriteLine(stackTrace);
-            OutWriter.WriteLine();
-
-            ColorStyle GetColorStyle()
-            {
-                return resultState == "Failed"
-                    ? ColorStyle.Failure
-                    : resultState == "Warning"
-                        ? ColorStyle.Warning
-                        : ColorStyle.Output;
-            }
-        }
-
-        private void WriteSummaryCount(string label, int count)
-        {
-            OutWriter.WriteLabel(label, count.ToString(CultureInfo.CurrentUICulture));
-        }
-
-        private void WriteSummaryCount(string label, int count, ColorStyle color)
-        {
-            OutWriter.WriteLabel(label, count.ToString(CultureInfo.CurrentUICulture), count > 0 ? color : ColorStyle.Value);
-        }
-
         public static void WriteResultFile(XmlNode resultNode, string outputPath)
         {
             using (var stream = new FileStream(outputPath, FileMode.Create, FileAccess.Write))

src/NUnitCommon/nunit.common.tests/TextDisplay/ResultReporterTests.cs

@@ -103,9 +103,9 @@ public void SummaryReportTest()
             {
                 "Test Run Summary",
                 "    Overall result: Failed",
-               $"    Test Count: {MockAssembly.Tests}, Pass: {MockAssembly.Passed}, Fail: 11, Warn: 1, Inconclusive: 1, Skip: 7",
-                "        Failed Tests - Failures: 1, Errors: 7, Invalid: 3",
-                "        Skipped Tests - Ignored: 4, Explicit: 3, Other: 0",
+               $"    Test Count: {MockAssembly.Tests}, Pass: {MockAssembly.Passed}, Fail: {MockAssembly.Failed}, Warn: {MockAssembly.Warnings}, Inconclusive: {MockAssembly.Inconclusive}, Skip: {MockAssembly.Skipped}",
+               $"        Failed Tests - Failures: {MockAssembly.Failed_Other}, Errors: {MockAssembly.Failed_Error}, Invalid: {MockAssembly.Failed_NotRunnable}",
+               $"        Skipped Tests - Ignored: 4, Explicit: 3, Other: 0",
                 "    Start time: 2015-10-19 02:12:28Z",
                 "    End time: 2015-10-19 02:12:29Z",
                 "    Duration: 0.349 seconds",
@@ -178,11 +178,11 @@ public void TestsNotRunTest()
             Assert.That(ReportLines, Is.EqualTo(expected));
         }
 
-        [Test, Explicit("Displays failure behavior")]
-        public void WarningsOnlyDisplayOnce()
-        {
-            Assert.Warn("Just a warning");
-        }
+        //[Test, Explicit("Displays failure behavior")]
+        //public void WarningsOnlyDisplayOnce()
+        //{
+        //    Assert.Warn("Just a warning");
+        //}
 
         [Test]
         public void TestParameterSettingsWrittenCorrectly()

src/NUnitCommon/nunit.common/ResultSummary.cs

@@ -12,8 +12,8 @@ public class ResultSummary
     {
         public ResultSummary(XmlNode resultNode)
         {
-            if (resultNode.Name != "test-run")
-                throw new InvalidOperationException("Expected <test-run> as top-level element but was <" + resultNode.Name + ">");
+            if (resultNode.Name != "test-run" && resultNode.Name != "test-suite")
+                throw new InvalidOperationException("Expected <test-run> or <test-suite> as top-level element but was <" + resultNode.Name + ">");
 
             InitializeCounters();
 
@@ -216,10 +216,11 @@ private void Summarize(XmlNode node, bool failedInFixtureTearDown)
                         InvalidAssemblies++;
                         UnexpectedError = true;
                     }
-                    if ((type == "SetUpFixture" || type == "TestFixture") && status == "Failed" && label == "Error" && site == "TearDown")
-                    {
-                        failedInFixtureTearDown = true;
-                    }
+                    // TODO: This seems wrong and causes failures. Leaving it commented while awaiting feedback.
+                    //if ((type == "SetUpFixture" || type == "TestFixture") && status == "Failed" && label == "Error" && site == "TearDown")
+                    //{
+                    //    failedInFixtureTearDown = true;
+                    //}
 
                     Summarize(node.ChildNodes, failedInFixtureTearDown);
                     break;