From e13ca085694aa9a9908afb3e9a90a58841b5fe03 Mon Sep 17 00:00:00 2001 From: Abhishek Kumawat Date: Tue, 5 Jun 2018 17:45:06 +0530 Subject: [PATCH] Parent test result support for data driven tests (#417) * Parent test result support for data driven tests * smoke tests fix * review comments * review comments and UTs * UTs * review comments --- src/Adapter/MSTest.CoreAdapter/Constants.cs | 9 + .../Execution/TestMethodRunner.cs | 102 ++++++-- .../Extensions/TestResultExtensions.cs | 3 + .../Extensions/UnitTestOutcomeExtensions.cs | 4 +- .../ObjectModel/UnitTestResult.cs | 19 ++ .../Attributes/VSTestAttributes.cs | 15 ++ test/E2ETests/Automation.CLI/CLITestBase.cs | 14 +- .../CustomTestExecutionExtensibilityTests.cs | 5 +- .../Execution/TestMethodRunnerTests.cs | 236 +++++++++++++++++- .../Extensions/TestResultExtensionsTests.cs | 24 ++ .../ObjectModel/UnitTestResultTests.cs | 21 ++ 11 files changed, 416 insertions(+), 36 deletions(-) diff --git a/src/Adapter/MSTest.CoreAdapter/Constants.cs b/src/Adapter/MSTest.CoreAdapter/Constants.cs index 5f561f1519..2e91aa4f81 100644 --- a/src/Adapter/MSTest.CoreAdapter/Constants.cs +++ b/src/Adapter/MSTest.CoreAdapter/Constants.cs @@ -44,6 +44,12 @@ internal static class Constants internal static readonly TestProperty DoNotParallelizeProperty = TestProperty.Register("MSTestDiscoverer.DoNotParallelize", DoNotParallelizeLabel, typeof(bool), TestPropertyAttributes.Hidden, typeof(TestCase)); + internal static readonly TestProperty ExecutionIdProperty = TestProperty.Register("ExecutionId", ExecutionIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult)); + + internal static readonly TestProperty ParentExecIdProperty = TestProperty.Register("ParentExecId", ParentExecIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult)); + + internal static readonly TestProperty InnerResultsCountProperty = TestProperty.Register("InnerResultsCount", InnerResultsCountLabel, typeof(int), TestPropertyAttributes.Hidden, typeof(TestResult)); + #endregion #region Private Constants @@ -59,6 +65,9 @@ internal static class Constants private const string PriorityLabel = "Priority"; private const string DeploymentItemsLabel = "DeploymentItems"; private const string DoNotParallelizeLabel = "DoNotParallelize"; + private const string ExecutionIdLabel = "ExecutionId"; + private const string ParentExecIdLabel = "ParentExecId"; + private const string InnerResultsCountLabel = "InnerResultsCount"; #endregion } diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 2ece924fcc..4ac11553a2 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -215,12 +215,23 @@ internal UnitTestResult[] RunTestMethod() Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null."); List results = new List(); + var isDataDriven = false; + + // Parent result. Added in properties bag only when results are greater than 1. + var parentResultWatch = new Stopwatch(); + parentResultWatch.Start(); + var parentResult = new UTF.TestResult + { + Outcome = UTF.UnitTestOutcome.InProgress, + ExecutionId = Guid.NewGuid() + }; if (this.testMethodInfo.TestMethodOptions.Executor != null) { UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes(false); if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) { + isDataDriven = true; Stopwatch watch = new Stopwatch(); watch.Start(); @@ -296,6 +307,7 @@ internal UnitTestResult[] RunTestMethod() if (testDataSources != null && testDataSources.Length > 0) { + isDataDriven = true; foreach (var testDataSource in testDataSources) { foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) @@ -345,35 +357,81 @@ internal UnitTestResult[] RunTestMethod() this.testMethodInfo.TestMethodName); } - if (results != null && results.Count > 0) - { - // aggregate for data driven tests - UTF.UnitTestOutcome aggregateOutcome = UTF.UnitTestOutcome.Passed; + parentResultWatch.Stop(); + parentResult.Duration = parentResultWatch.Elapsed; - foreach (var result in results) - { - if (result.Outcome != UTF.UnitTestOutcome.Passed) - { - if (aggregateOutcome != UTF.UnitTestOutcome.Failed) - { - if (result.Outcome == UTF.UnitTestOutcome.Failed - || aggregateOutcome != UTF.UnitTestOutcome.Timeout) - { - aggregateOutcome = result.Outcome; - } - } - } - } + // Get aggregate outcome. + var aggregateOutcome = this.GetAggregateOutcome(results); + this.testContext.SetOutcome(aggregateOutcome); - this.testContext.SetOutcome(aggregateOutcome); + // Set a result in case no result is present. + if (!results.Any()) + { + results.Add(new UTF.TestResult() { Outcome = aggregateOutcome, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) }); } - else + + // In case of data driven, set parent info in results. + if (isDataDriven) { - this.testContext.SetOutcome(UTF.UnitTestOutcome.Unknown); - results.Add(new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Unknown, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) }); + parentResult.Outcome = aggregateOutcome; + results = this.UpdateResultsWithParentInfo(results, parentResult); } return results.ToArray().ToUnitTestResults(); } + + /// + /// Gets aggregate outcome. + /// + /// Results. + /// Aggregate outcome. + private UTF.UnitTestOutcome GetAggregateOutcome(List results) + { + // In case results are not present, set outcome as unknown. + if (!results.Any()) + { + return UTF.UnitTestOutcome.Unknown; + } + + // Get aggregate outcome. + var aggregateOutcome = results[0].Outcome; + foreach (var result in results) + { + aggregateOutcome = UnitTestOutcomeExtensions.GetMoreImportantOutcome(aggregateOutcome, result.Outcome); + } + + return aggregateOutcome; + } + + /// + /// Updates given resutls with parent info if results are greater than 1. + /// Add parent results as first result in updated result. + /// + /// Results. + /// Parent results. + /// Updated results which contains parent result as first result. All other results contains parent result info. + private List UpdateResultsWithParentInfo(List results, UTF.TestResult parentResult) + { + // Return results in case there are no results. + if (!results.Any()) + { + return results; + } + + // UpdatedResults contain parent result at first position and remaining results has parent info updated. + var updatedResults = new List(); + updatedResults.Add(parentResult); + + foreach (var result in results) + { + result.ExecutionId = Guid.NewGuid(); + result.ParentExecId = parentResult.ExecutionId; + parentResult.InnerResultsCount++; + + updatedResults.Add(result); + } + + return updatedResults; + } } } diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs index 2edc05d058..60f3794969 100644 --- a/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs +++ b/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs @@ -47,6 +47,9 @@ public static UnitTestResult[] ToUnitTestResults(this UTF.TestResult[] testResul unitTestResult.DisplayName = testResults[i].DisplayName; unitTestResult.DatarowIndex = testResults[i].DatarowIndex; unitTestResult.ResultFiles = testResults[i].ResultFiles; + unitTestResult.ExecutionId = testResults[i].ExecutionId; + unitTestResult.ParentExecId = testResults[i].ParentExecId; + unitTestResult.InnerResultsCount = testResults[i].InnerResultsCount; unitTestResults[i] = unitTestResult; } diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs index d123b9a661..2df6c87710 100644 --- a/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs +++ b/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs @@ -61,7 +61,9 @@ public static UnitTestOutcome ToUnitTestOutcome(this UTF.UnitTestOutcome framewo /// Outcome which has higher importance. internal static UTF.UnitTestOutcome GetMoreImportantOutcome(this UTF.UnitTestOutcome outcome1, UTF.UnitTestOutcome outcome2) { - return outcome1 < outcome2 ? outcome1 : outcome2; + var unitTestOutcome1 = outcome1.ToUnitTestOutcome(); + var unitTestOutcome2 = outcome2.ToUnitTestOutcome(); + return unitTestOutcome1 < unitTestOutcome2 ? outcome1 : outcome2; } } } diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs index b04e8d9404..c8aae7fd21 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs @@ -75,6 +75,21 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage) /// public string ErrorStackTrace { get; internal set; } + /// + /// Gets the execution id of the result + /// + public Guid ExecutionId { get; internal set; } + + /// + /// Gets the parent execution id of the result + /// + public Guid ParentExecId { get; internal set; } + + /// + /// Gets the inner results count of the result + /// + public int InnerResultsCount { get; internal set; } + /// /// Gets the duration of the result /// @@ -149,6 +164,10 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da EndTime = endTime }; + testResult.SetPropertyValue(Constants.ExecutionIdProperty, this.ExecutionId); + testResult.SetPropertyValue(Constants.ParentExecIdProperty, this.ParentExecId); + testResult.SetPropertyValue(Constants.InnerResultsCountProperty, this.InnerResultsCount); + if (!string.IsNullOrEmpty(this.StandardOut)) { TestResultMessage message = new TestResultMessage(TestResultMessage.StandardOutCategory, this.StandardOut); diff --git a/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs b/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs index 78977e1d80..474499fbbc 100644 --- a/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs +++ b/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs @@ -411,6 +411,21 @@ public TestResult() /// public string TestContextMessages { get; set; } + /// + /// Gets or sets the execution id of the result. + /// + public Guid ExecutionId { get; set; } + + /// + /// Gets or sets the parent execution id of the result. + /// + public Guid ParentExecId { get; set; } + + /// + /// Gets or sets the inner results count of the result. + /// + public int InnerResultsCount { get; set; } + /// /// Gets or sets the duration of test execution. /// diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index f846b3e69a..2dc761445b 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -134,12 +134,20 @@ public void ValidatePassedTests(params string[] passedTests) /// public void ValidateFailedTests(string source, params string[] failedTests) { - // Make sure only expected number of tests failed and not more. - Assert.AreEqual(failedTests.Length, this.runEventsHandler.FailedTests.Count); - + this.ValidateFailedTestsCount(failedTests.Length); this.ValidateFailedTestsContain(source, failedTests); } + /// + /// Validates the count of failed tests. + /// + /// Expected failed tests count. + public void ValidateFailedTestsCount(int expectedFailedTestsCount) + { + // Make sure only expected number of tests failed and not more. + Assert.AreEqual(expectedFailedTestsCount, this.runEventsHandler.FailedTests.Count); + } + /// /// Validates if the test results have the specified set of skipped tests. /// diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs index 23f7cd81c9..eea8e4b3d1 100644 --- a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -38,7 +38,10 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() "CustomTestMethod2 (B)", "CustomTestMethod2 (B)", "CustomTestMethod2 (B)"); - this.ValidateFailedTests( + + // Parent results should fail and thus failed count should be 7. + this.ValidateFailedTestsCount(7); + this.ValidateFailedTestsContain( TestAssembly, "CustomTestMethod2 (A)", "CustomTestMethod2 (A)", diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs index 3dbd30d23e..9419502f2d 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs @@ -430,6 +430,7 @@ public void RunTestMethodForMultipleResultsReturnMultipleResults() var results = testMethodRunner.Execute(); Assert.AreEqual(2, results.Length); + Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); } @@ -543,9 +544,11 @@ public void RunTestMethodShouldSetDataRowIndexForDataDrivenTestsWhenDataIsProvid var results = testMethodRunner.RunTestMethod(); // check for datarowIndex - Assert.AreEqual(results[0].DatarowIndex, 0); - Assert.AreEqual(results[1].DatarowIndex, 1); - Assert.AreEqual(results[2].DatarowIndex, 2); + // 1st is parent result. + Assert.AreEqual(results[0].DatarowIndex, -1); + Assert.AreEqual(results[1].DatarowIndex, 0); + Assert.AreEqual(results[2].DatarowIndex, 1); + Assert.AreEqual(results[3].DatarowIndex, 2); } [TestMethodV1] @@ -570,9 +573,11 @@ public void RunTestMethodShoudlRunOnlyDataSourceTestsWhenBothDataSourceAndDataRo var results = testMethodRunner.RunTestMethod(); // check for datarowIndex as only DataSource Tests are Run - Assert.AreEqual(results[0].DatarowIndex, 0); - Assert.AreEqual(results[1].DatarowIndex, 1); - Assert.AreEqual(results[2].DatarowIndex, 2); + // 1st is parent result. + Assert.AreEqual(results[0].DatarowIndex, -1); + Assert.AreEqual(results[1].DatarowIndex, 0); + Assert.AreEqual(results[2].DatarowIndex, 1); + Assert.AreEqual(results[3].DatarowIndex, 2); } [TestMethodV1] @@ -595,7 +600,10 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowDisplayNameIfProvided this.testablePlatformServiceProvider.MockReflectionOperations.Setup(ro => ro.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); - Assert.AreEqual(results[0].DisplayName, "DataRowTestDisplayName"); + + // 1st results should be parent result. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[1].DisplayName, "DataRowTestDisplayName"); } [TestMethodV1] @@ -617,7 +625,10 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowArgumentsIfNoDisplayN this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); - Assert.AreEqual(results[0].DisplayName, "DummyTestMethod (2,DummyString)"); + + // 1st results should be parent result. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[1].DisplayName, "DummyTestMethod (2,DummyString)"); } [TestMethodV1] @@ -640,8 +651,215 @@ public void RunTestMethodShouldSetResultFilesIfPresentForDataDrivenTests() this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); - CollectionAssert.Contains(results[0].ResultFiles.ToList(), "C:\\temp.txt"); CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); + CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); + } + + [TestMethodV1] + public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTests() + { + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); + + var attribs = new Attribute[] { dataSourceAttribute }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); + + var results = testMethodRunner.RunTestMethod(); + + // check for parent result + Assert.AreEqual(4, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTestsContainingSingleTest() + { + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); + + var attribs = new Attribute[] { dataSourceAttribute }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1 }); + + var results = testMethodRunner.RunTestMethod(); + + // Parent result should exist. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTests() + { + UTF.TestResult testResult = new UTF.TestResult(); + testResult.ResultFiles = new List() { "C:\\temp.txt" }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); + + int dummyIntData1 = 1; + int dummyIntData2 = 2; + UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); + UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); + + var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + + var results = testMethodRunner.RunTestMethod(); + CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); + CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); + + // Parent result should exist. + Assert.AreEqual(3, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTestsContainingSingleTest() + { + UTF.TestResult testResult = new UTF.TestResult(); + testResult.ResultFiles = new List() { "C:\\temp.txt" }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); + + int dummyIntData1 = 1; + UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); + + var attribs = new Attribute[] { dataRowAttribute1 }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + + var results = testMethodRunner.RunTestMethod(); + CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); + + // Parent result should exist. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldNotReturnParentResultForNonDataDrivenTests() + { + var testMethodAttributeMock = new Mock(); + testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[] + { + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }, + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } + }); + + var localTestMethodOptions = new TestMethodOptions + { + Timeout = 200, + Executor = testMethodAttributeMock.Object, + TestContext = this.testContextImplementation, + ExpectedException = null + }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + var results = testMethodRunner.Execute(); + Assert.AreEqual(2, results.Length); + + // Parent result should not exists as its not data driven test. + Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + } + + [TestMethodV1] + public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataSourceDataDrivenTests() + { + var testExecutedCount = 0; + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => + { + return (testExecutedCount++ == 0) ? + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; + }); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); + + var attribs = new Attribute[] { dataSourceAttribute }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); + + var results = testMethodRunner.RunTestMethod(); + + // check for parent result + Assert.AreEqual(4, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + + // Check for aggregate outcome. + Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[3].Outcome); + } + + [TestMethodV1] + public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataRowDataDrivenTests() + { + var testExecutedCount = 0; + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => + { + return (testExecutedCount++ == 0) ? + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; + }); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); + + int dummyIntData1 = 1; + int dummyIntData2 = 2; + UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); + UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); + + var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + + var results = testMethodRunner.RunTestMethod(); + + // Parent result should exist. + Assert.AreEqual(3, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); + + // Check for aggregate outcome. + Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); } #region Test data diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs index 518b7b2157..ad839a1d9b 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs @@ -171,6 +171,30 @@ public void ToUnitTestResultsForTestResultShouldSetDataRowIndex() Assert.AreEqual(1, convertedResults[0].DatarowIndex); } + [TestMethod] + public void ToUnitTestResultsForTestResultShouldSetParentInfo() + { + var executionId = Guid.NewGuid(); + var parentExecId = Guid.NewGuid(); + var innerResultsCount = 5; + + var results = new[] + { + new UTF.TestResult() + { + ExecutionId = executionId, + ParentExecId = parentExecId, + InnerResultsCount = innerResultsCount + } + }; + + var convertedResults = results.ToUnitTestResults(); + + Assert.AreEqual(executionId, convertedResults[0].ExecutionId); + Assert.AreEqual(parentExecId, convertedResults[0].ParentExecId); + Assert.AreEqual(innerResultsCount, convertedResults[0].InnerResultsCount); + } + [TestMethod] public void ToUnitTestResultsShouldHaveResultsFileProvidedToTestResult() { diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs index ce9b50329f..f6ce578efd 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs @@ -154,6 +154,27 @@ public void ToTestResultForUniTestResultWithNoResultFilesShouldReturnTestResultW Assert.AreEqual(testresult.Attachments.Count, 0); } + [TestMethod] + public void ToTestResultForUniTestResultWithParentInfoShouldReturnTestResultWithParentInfo() + { + var executionId = Guid.NewGuid(); + var parentExecId = Guid.NewGuid(); + var innerResultsCount = 5; + + UnitTestResult result = new UnitTestResult() + { + ExecutionId = executionId, + ParentExecId = parentExecId, + InnerResultsCount = innerResultsCount + }; + TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + Assert.AreEqual(executionId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ExecutionIdProperty)); + Assert.AreEqual(parentExecId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ParentExecIdProperty)); + Assert.AreEqual(innerResultsCount, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.InnerResultsCountProperty)); + } + [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomePassedShouldReturnTestOutcomePassed() {