Skip to content

Commit e13ca08

Browse files
abhishkkabhishkk
authored
Parent test result support for data driven tests (#417)
* Parent test result support for data driven tests * smoke tests fix * review comments * review comments and UTs * UTs * review comments
1 parent 78676d7 commit e13ca08

File tree

11 files changed

+416
-36
lines changed

11 files changed

+416
-36
lines changed

‎src/Adapter/MSTest.CoreAdapter/Constants.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ internal static class Constants
4444

4545
internal static readonly TestProperty DoNotParallelizeProperty = TestProperty.Register("MSTestDiscoverer.DoNotParallelize", DoNotParallelizeLabel, typeof(bool), TestPropertyAttributes.Hidden, typeof(TestCase));
4646

47+
internal static readonly TestProperty ExecutionIdProperty = TestProperty.Register("ExecutionId", ExecutionIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult));
48+
49+
internal static readonly TestProperty ParentExecIdProperty = TestProperty.Register("ParentExecId", ParentExecIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult));
50+
51+
internal static readonly TestProperty InnerResultsCountProperty = TestProperty.Register("InnerResultsCount", InnerResultsCountLabel, typeof(int), TestPropertyAttributes.Hidden, typeof(TestResult));
52+
4753
#endregion
4854

4955
#region Private Constants
@@ -59,6 +65,9 @@ internal static class Constants
5965
private const string PriorityLabel = "Priority";
6066
private const string DeploymentItemsLabel = "DeploymentItems";
6167
private const string DoNotParallelizeLabel = "DoNotParallelize";
68+
private const string ExecutionIdLabel = "ExecutionId";
69+
private const string ParentExecIdLabel = "ParentExecId";
70+
private const string InnerResultsCountLabel = "InnerResultsCount";
6271

6372
#endregion
6473
}

‎src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs

Lines changed: 80 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,23 @@ internal UnitTestResult[] RunTestMethod()
215215
Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null.");
216216

217217
List<UTF.TestResult> results = new List<UTF.TestResult>();
218+
var isDataDriven = false;
219+
220+
// Parent result. Added in properties bag only when results are greater than 1.
221+
var parentResultWatch = new Stopwatch();
222+
parentResultWatch.Start();
223+
var parentResult = new UTF.TestResult
224+
{
225+
Outcome = UTF.UnitTestOutcome.InProgress,
226+
ExecutionId = Guid.NewGuid()
227+
};
218228

219229
if (this.testMethodInfo.TestMethodOptions.Executor != null)
220230
{
221231
UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes<UTF.DataSourceAttribute>(false);
222232
if (dataSourceAttribute != null && dataSourceAttribute.Length == 1)
223233
{
234+
isDataDriven = true;
224235
Stopwatch watch = new Stopwatch();
225236
watch.Start();
226237

@@ -296,6 +307,7 @@ internal UnitTestResult[] RunTestMethod()
296307

297308
if (testDataSources != null && testDataSources.Length > 0)
298309
{
310+
isDataDriven = true;
299311
foreach (var testDataSource in testDataSources)
300312
{
301313
foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
@@ -345,35 +357,81 @@ internal UnitTestResult[] RunTestMethod()
345357
this.testMethodInfo.TestMethodName);
346358
}
347359

348-
if (results != null && results.Count > 0)
349-
{
350-
// aggregate for data driven tests
351-
UTF.UnitTestOutcome aggregateOutcome = UTF.UnitTestOutcome.Passed;
360+
parentResultWatch.Stop();
361+
parentResult.Duration = parentResultWatch.Elapsed;
352362

353-
foreach (var result in results)
354-
{
355-
if (result.Outcome != UTF.UnitTestOutcome.Passed)
356-
{
357-
if (aggregateOutcome != UTF.UnitTestOutcome.Failed)
358-
{
359-
if (result.Outcome == UTF.UnitTestOutcome.Failed
360-
|| aggregateOutcome != UTF.UnitTestOutcome.Timeout)
361-
{
362-
aggregateOutcome = result.Outcome;
363-
}
364-
}
365-
}
366-
}
363+
// Get aggregate outcome.
364+
var aggregateOutcome = this.GetAggregateOutcome(results);
365+
this.testContext.SetOutcome(aggregateOutcome);
367366

368-
this.testContext.SetOutcome(aggregateOutcome);
367+
// Set a result in case no result is present.
368+
if (!results.Any())
369+
{
370+
results.Add(new UTF.TestResult() { Outcome = aggregateOutcome, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) });
369371
}
370-
else
372+
373+
// In case of data driven, set parent info in results.
374+
if (isDataDriven)
371375
{
372-
this.testContext.SetOutcome(UTF.UnitTestOutcome.Unknown);
373-
results.Add(new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Unknown, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) });
376+
parentResult.Outcome = aggregateOutcome;
377+
results = this.UpdateResultsWithParentInfo(results, parentResult);
374378
}
375379

376380
return results.ToArray().ToUnitTestResults();
377381
}
382+
383+
/// <summary>
384+
/// Gets aggregate outcome.
385+
/// </summary>
386+
/// <param name="results">Results.</param>
387+
/// <returns>Aggregate outcome.</returns>
388+
private UTF.UnitTestOutcome GetAggregateOutcome(List<UTF.TestResult> results)
389+
{
390+
// In case results are not present, set outcome as unknown.
391+
if (!results.Any())
392+
{
393+
return UTF.UnitTestOutcome.Unknown;
394+
}
395+
396+
// Get aggregate outcome.
397+
var aggregateOutcome = results[0].Outcome;
398+
foreach (var result in results)
399+
{
400+
aggregateOutcome = UnitTestOutcomeExtensions.GetMoreImportantOutcome(aggregateOutcome, result.Outcome);
401+
}
402+
403+
return aggregateOutcome;
404+
}
405+
406+
/// <summary>
407+
/// Updates given resutls with parent info if results are greater than 1.
408+
/// Add parent results as first result in updated result.
409+
/// </summary>
410+
/// <param name="results">Results.</param>
411+
/// <param name="parentResult">Parent results.</param>
412+
/// <returns>Updated results which contains parent result as first result. All other results contains parent result info.</returns>
413+
private List<UTF.TestResult> UpdateResultsWithParentInfo(List<UTF.TestResult> results, UTF.TestResult parentResult)
414+
{
415+
// Return results in case there are no results.
416+
if (!results.Any())
417+
{
418+
return results;
419+
}
420+
421+
// UpdatedResults contain parent result at first position and remaining results has parent info updated.
422+
var updatedResults = new List<UTF.TestResult>();
423+
updatedResults.Add(parentResult);
424+
425+
foreach (var result in results)
426+
{
427+
result.ExecutionId = Guid.NewGuid();
428+
result.ParentExecId = parentResult.ExecutionId;
429+
parentResult.InnerResultsCount++;
430+
431+
updatedResults.Add(result);
432+
}
433+
434+
return updatedResults;
435+
}
378436
}
379437
}

‎src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ public static UnitTestResult[] ToUnitTestResults(this UTF.TestResult[] testResul
4747
unitTestResult.DisplayName = testResults[i].DisplayName;
4848
unitTestResult.DatarowIndex = testResults[i].DatarowIndex;
4949
unitTestResult.ResultFiles = testResults[i].ResultFiles;
50+
unitTestResult.ExecutionId = testResults[i].ExecutionId;
51+
unitTestResult.ParentExecId = testResults[i].ParentExecId;
52+
unitTestResult.InnerResultsCount = testResults[i].InnerResultsCount;
5053
unitTestResults[i] = unitTestResult;
5154
}
5255

‎src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,9 @@ public static UnitTestOutcome ToUnitTestOutcome(this UTF.UnitTestOutcome framewo
6161
/// <returns> Outcome which has higher importance.</returns>
6262
internal static UTF.UnitTestOutcome GetMoreImportantOutcome(this UTF.UnitTestOutcome outcome1, UTF.UnitTestOutcome outcome2)
6363
{
64-
return outcome1 < outcome2 ? outcome1 : outcome2;
64+
var unitTestOutcome1 = outcome1.ToUnitTestOutcome();
65+
var unitTestOutcome2 = outcome2.ToUnitTestOutcome();
66+
return unitTestOutcome1 < unitTestOutcome2 ? outcome1 : outcome2;
6567
}
6668
}
6769
}

‎src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,21 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage)
7575
/// </summary>
7676
public string ErrorStackTrace { get; internal set; }
7777

78+
/// <summary>
79+
/// Gets the execution id of the result
80+
/// </summary>
81+
public Guid ExecutionId { get; internal set; }
82+
83+
/// <summary>
84+
/// Gets the parent execution id of the result
85+
/// </summary>
86+
public Guid ParentExecId { get; internal set; }
87+
88+
/// <summary>
89+
/// Gets the inner results count of the result
90+
/// </summary>
91+
public int InnerResultsCount { get; internal set; }
92+
7893
/// <summary>
7994
/// Gets the duration of the result
8095
/// </summary>
@@ -149,6 +164,10 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da
149164
EndTime = endTime
150165
};
151166

167+
testResult.SetPropertyValue<Guid>(Constants.ExecutionIdProperty, this.ExecutionId);
168+
testResult.SetPropertyValue<Guid>(Constants.ParentExecIdProperty, this.ParentExecId);
169+
testResult.SetPropertyValue<int>(Constants.InnerResultsCountProperty, this.InnerResultsCount);
170+
152171
if (!string.IsNullOrEmpty(this.StandardOut))
153172
{
154173
TestResultMessage message = new TestResultMessage(TestResultMessage.StandardOutCategory, this.StandardOut);

‎src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,21 @@ public TestResult()
411411
/// </summary>
412412
public string TestContextMessages { get; set; }
413413

414+
/// <summary>
415+
/// Gets or sets the execution id of the result.
416+
/// </summary>
417+
public Guid ExecutionId { get; set; }
418+
419+
/// <summary>
420+
/// Gets or sets the parent execution id of the result.
421+
/// </summary>
422+
public Guid ParentExecId { get; set; }
423+
424+
/// <summary>
425+
/// Gets or sets the inner results count of the result.
426+
/// </summary>
427+
public int InnerResultsCount { get; set; }
428+
414429
/// <summary>
415430
/// Gets or sets the duration of test execution.
416431
/// </summary>

‎test/E2ETests/Automation.CLI/CLITestBase.cs

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,12 +134,20 @@ public void ValidatePassedTests(params string[] passedTests)
134134
/// </remarks>
135135
public void ValidateFailedTests(string source, params string[] failedTests)
136136
{
137-
// Make sure only expected number of tests failed and not more.
138-
Assert.AreEqual(failedTests.Length, this.runEventsHandler.FailedTests.Count);
139-
137+
this.ValidateFailedTestsCount(failedTests.Length);
140138
this.ValidateFailedTestsContain(source, failedTests);
141139
}
142140

141+
/// <summary>
142+
/// Validates the count of failed tests.
143+
/// </summary>
144+
/// <param name="expectedFailedTestsCount">Expected failed tests count.</param>
145+
public void ValidateFailedTestsCount(int expectedFailedTestsCount)
146+
{
147+
// Make sure only expected number of tests failed and not more.
148+
Assert.AreEqual(expectedFailedTestsCount, this.runEventsHandler.FailedTests.Count);
149+
}
150+
143151
/// <summary>
144152
/// Validates if the test results have the specified set of skipped tests.
145153
/// </summary>

‎test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests()
3838
"CustomTestMethod2 (B)",
3939
"CustomTestMethod2 (B)",
4040
"CustomTestMethod2 (B)");
41-
this.ValidateFailedTests(
41+
42+
// Parent results should fail and thus failed count should be 7.
43+
this.ValidateFailedTestsCount(7);
44+
this.ValidateFailedTestsContain(
4245
TestAssembly,
4346
"CustomTestMethod2 (A)",
4447
"CustomTestMethod2 (A)",

0 commit comments

Comments
 (0)