Parent test result support for data driven tests (#417)

* Parent test result support for data driven tests

* smoke tests fix

* review comments

* review comments and UTs

* UTs

* review comments

src/Adapter/MSTest.CoreAdapter/Constants.cs

@@ -44,6 +44,12 @@ internal static class Constants
 
         internal static readonly TestProperty DoNotParallelizeProperty = TestProperty.Register("MSTestDiscoverer.DoNotParallelize", DoNotParallelizeLabel, typeof(bool), TestPropertyAttributes.Hidden, typeof(TestCase));
 
+        internal static readonly TestProperty ExecutionIdProperty = TestProperty.Register("ExecutionId", ExecutionIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult));
+
+        internal static readonly TestProperty ParentExecIdProperty = TestProperty.Register("ParentExecId", ParentExecIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult));
+
+        internal static readonly TestProperty InnerResultsCountProperty = TestProperty.Register("InnerResultsCount", InnerResultsCountLabel, typeof(int), TestPropertyAttributes.Hidden, typeof(TestResult));
+
         #endregion
 
         #region Private Constants
@@ -59,6 +65,9 @@ internal static class Constants
         private const string PriorityLabel = "Priority";
         private const string DeploymentItemsLabel = "DeploymentItems";
         private const string DoNotParallelizeLabel = "DoNotParallelize";
+        private const string ExecutionIdLabel = "ExecutionId";
+        private const string ParentExecIdLabel = "ParentExecId";
+        private const string InnerResultsCountLabel = "InnerResultsCount";
 
         #endregion
     }

src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs

@@ -215,12 +215,23 @@ internal UnitTestResult[] RunTestMethod()
             Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null.");
 
             List<UTF.TestResult> results = new List<UTF.TestResult>();
+            var isDataDriven = false;
+
+            // Parent result. Added in properties bag only when results are greater than 1.
+            var parentResultWatch = new Stopwatch();
+            parentResultWatch.Start();
+            var parentResult = new UTF.TestResult
+            {
+                Outcome = UTF.UnitTestOutcome.InProgress,
+                ExecutionId = Guid.NewGuid()
+            };
 
             if (this.testMethodInfo.TestMethodOptions.Executor != null)
             {
                 UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes<UTF.DataSourceAttribute>(false);
                 if (dataSourceAttribute != null && dataSourceAttribute.Length == 1)
                 {
+                    isDataDriven = true;
                     Stopwatch watch = new Stopwatch();
                     watch.Start();
 
@@ -296,6 +307,7 @@ internal UnitTestResult[] RunTestMethod()
 
                     if (testDataSources != null && testDataSources.Length > 0)
                     {
+                        isDataDriven = true;
                         foreach (var testDataSource in testDataSources)
                         {
                             foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
@@ -345,35 +357,81 @@ internal UnitTestResult[] RunTestMethod()
                 this.testMethodInfo.TestMethodName);
             }
 
-            if (results != null && results.Count > 0)
-            {
-                // aggregate for data driven tests
-                UTF.UnitTestOutcome aggregateOutcome = UTF.UnitTestOutcome.Passed;
+            parentResultWatch.Stop();
+            parentResult.Duration = parentResultWatch.Elapsed;
 
-                foreach (var result in results)
-                {
-                    if (result.Outcome != UTF.UnitTestOutcome.Passed)
-                    {
-                        if (aggregateOutcome != UTF.UnitTestOutcome.Failed)
-                        {
-                            if (result.Outcome == UTF.UnitTestOutcome.Failed
-                                || aggregateOutcome != UTF.UnitTestOutcome.Timeout)
-                            {
-                                aggregateOutcome = result.Outcome;
-                            }
-                        }
-                    }
-                }
+            // Get aggregate outcome.
+            var aggregateOutcome = this.GetAggregateOutcome(results);
+            this.testContext.SetOutcome(aggregateOutcome);
 
-                this.testContext.SetOutcome(aggregateOutcome);
+            // Set a result in case no result is present.
+            if (!results.Any())
+            {
+                results.Add(new UTF.TestResult() { Outcome = aggregateOutcome, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) });
             }
-            else
+
+            // In case of data driven, set parent info in results.
+            if (isDataDriven)
             {
-                this.testContext.SetOutcome(UTF.UnitTestOutcome.Unknown);
-                results.Add(new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Unknown, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) });
+                parentResult.Outcome = aggregateOutcome;
+                results = this.UpdateResultsWithParentInfo(results, parentResult);
             }
 
             return results.ToArray().ToUnitTestResults();
         }
+
+        /// <summary>
+        /// Gets aggregate outcome.
+        /// </summary>
+        /// <param name="results">Results.</param>
+        /// <returns>Aggregate outcome.</returns>
+        private UTF.UnitTestOutcome GetAggregateOutcome(List<UTF.TestResult> results)
+        {
+            // In case results are not present, set outcome as unknown.
+            if (!results.Any())
+            {
+                return UTF.UnitTestOutcome.Unknown;
+            }
+
+            // Get aggregate outcome.
+            var aggregateOutcome = results[0].Outcome;
+            foreach (var result in results)
+            {
+                aggregateOutcome = UnitTestOutcomeExtensions.GetMoreImportantOutcome(aggregateOutcome, result.Outcome);
+            }
+
+            return aggregateOutcome;
+        }
+
+        /// <summary>
+        /// Updates given resutls with parent info if results are greater than 1.
+        /// Add parent results as first result in updated result.
+        /// </summary>
+        /// <param name="results">Results.</param>
+        /// <param name="parentResult">Parent results.</param>
+        /// <returns>Updated results which contains parent result as first result. All other results contains parent result info.</returns>
+        private List<UTF.TestResult> UpdateResultsWithParentInfo(List<UTF.TestResult> results, UTF.TestResult parentResult)
+        {
+            // Return results in case there are no results.
+            if (!results.Any())
+            {
+                return results;
+            }
+
+            // UpdatedResults contain parent result at first position and remaining results has parent info updated.
+            var updatedResults = new List<UTF.TestResult>();
+            updatedResults.Add(parentResult);
+
+            foreach (var result in results)
+            {
+                result.ExecutionId = Guid.NewGuid();
+                result.ParentExecId = parentResult.ExecutionId;
+                parentResult.InnerResultsCount++;
+
+                updatedResults.Add(result);
+            }
+
+            return updatedResults;
+        }
     }
 }

src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs

@@ -47,6 +47,9 @@ public static UnitTestResult[] ToUnitTestResults(this UTF.TestResult[] testResul
                 unitTestResult.DisplayName = testResults[i].DisplayName;
                 unitTestResult.DatarowIndex = testResults[i].DatarowIndex;
                 unitTestResult.ResultFiles = testResults[i].ResultFiles;
+                unitTestResult.ExecutionId = testResults[i].ExecutionId;
+                unitTestResult.ParentExecId = testResults[i].ParentExecId;
+                unitTestResult.InnerResultsCount = testResults[i].InnerResultsCount;
                 unitTestResults[i] = unitTestResult;
             }
 

src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs

@@ -61,7 +61,9 @@ public static UnitTestOutcome ToUnitTestOutcome(this UTF.UnitTestOutcome framewo
         /// <returns> Outcome which has higher importance.</returns>
         internal static UTF.UnitTestOutcome GetMoreImportantOutcome(this UTF.UnitTestOutcome outcome1, UTF.UnitTestOutcome outcome2)
         {
-            return outcome1 < outcome2 ? outcome1 : outcome2;
+            var unitTestOutcome1 = outcome1.ToUnitTestOutcome();
+            var unitTestOutcome2 = outcome2.ToUnitTestOutcome();
+            return unitTestOutcome1 < unitTestOutcome2 ? outcome1 : outcome2;
         }
     }
 }

src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs

@@ -75,6 +75,21 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage)
         /// </summary>
         public string ErrorStackTrace { get; internal set; }
 
+        /// <summary>
+        /// Gets the execution id of the result
+        /// </summary>
+        public Guid ExecutionId { get; internal set; }
+
+        /// <summary>
+        /// Gets the parent execution id of the result
+        /// </summary>
+        public Guid ParentExecId { get; internal set; }
+
+        /// <summary>
+        /// Gets the inner results count of the result
+        /// </summary>
+        public int InnerResultsCount { get; internal set; }
+
         /// <summary>
         /// Gets the duration of the result
         /// </summary>
@@ -149,6 +164,10 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da
                                             EndTime = endTime
             };
 
+            testResult.SetPropertyValue<Guid>(Constants.ExecutionIdProperty, this.ExecutionId);
+            testResult.SetPropertyValue<Guid>(Constants.ParentExecIdProperty, this.ParentExecId);
+            testResult.SetPropertyValue<int>(Constants.InnerResultsCountProperty, this.InnerResultsCount);
+
             if (!string.IsNullOrEmpty(this.StandardOut))
             {
                 TestResultMessage message = new TestResultMessage(TestResultMessage.StandardOutCategory, this.StandardOut);

src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs

@@ -411,6 +411,21 @@ public TestResult()
         /// </summary>
         public string TestContextMessages { get; set; }
 
+        /// <summary>
+        /// Gets or sets the execution id of the result.
+        /// </summary>
+        public Guid ExecutionId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the parent execution id of the result.
+        /// </summary>
+        public Guid ParentExecId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the inner results count of the result.
+        /// </summary>
+        public int InnerResultsCount { get; set; }
+
         /// <summary>
         /// Gets or sets the duration of test execution.
         /// </summary>

test/E2ETests/Automation.CLI/CLITestBase.cs

@@ -134,12 +134,20 @@ public void ValidatePassedTests(params string[] passedTests)
         /// </remarks>
         public void ValidateFailedTests(string source, params string[] failedTests)
         {
-            // Make sure only expected number of tests failed and not more.
-            Assert.AreEqual(failedTests.Length, this.runEventsHandler.FailedTests.Count);
-
+            this.ValidateFailedTestsCount(failedTests.Length);
             this.ValidateFailedTestsContain(source, failedTests);
         }
 
+        /// <summary>
+        /// Validates the count of failed tests.
+        /// </summary>
+        /// <param name="expectedFailedTestsCount">Expected failed tests count.</param>
+        public void ValidateFailedTestsCount(int expectedFailedTestsCount)
+        {
+            // Make sure only expected number of tests failed and not more.
+            Assert.AreEqual(expectedFailedTestsCount, this.runEventsHandler.FailedTests.Count);
+        }
+
         /// <summary>
         /// Validates if the test results have the specified set of skipped tests.
         /// </summary>

test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs

@@ -38,7 +38,10 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests()
                 "CustomTestMethod2 (B)",
                 "CustomTestMethod2 (B)",
                 "CustomTestMethod2 (B)");
-            this.ValidateFailedTests(
+
+            // Parent results should fail and thus failed count should be 7.
+            this.ValidateFailedTestsCount(7);
+            this.ValidateFailedTestsContain(
                 TestAssembly,
                 "CustomTestMethod2 (A)",
                 "CustomTestMethod2 (A)",