Skip to content

Commit cfee8db

Browse files
mikaelweavemikaelweave
authored
Use Microsoft Testing Platform & Auto-Retry Tests (#5365)
* Enhance test project configurations and improve test execution reliability by adding retry logic for failed tests * Add retry logic for failed tests and update test configurations * Update test configurations to enhance code coverage reporting and improve test execution reliability * Fix indentation for testRunTitle and publishTestResults in Cosmos and SQL integration test configurations * Enhance test configurations by adding output capture settings and refining arguments for unit tests * Add TrxReport package and update test scripts for improved reporting and reliability * Disable failTaskOnFailedTests temporarily due to known issue with TestFX * Refactor test arguments in Cosmos and SQL integration test configurations for consistency * Add CodeCoverage settings and update test arguments for consistency across build jobs * Remove CollectFromChildProcesses setting from CodeCoverage configuration * Add YTest.MTP.XUnit2 component to third party notices * Update coverage report paths to include subdirectories in build jobs * Refactor code coverage settings and add validation for aggregated coverage report * Enhance code coverage settings and reporting configuration * Update code coverage settings to exclude missing assemblies and refine source exclusions * Update coverage reporting paths to use 'coverage-aggregated' directory * Refine code coverage settings by removing specific exclusions and generalizing configuration paths * PR review comments
1 parent b0bfd3f commit cfee8db

File tree

10 files changed

+234
-42
lines changed

10 files changed

+234
-42
lines changed

‎CodeCoverage.Mtp.settings.xml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Configuration>
3+
<IncludeTestAssembly>false</IncludeTestAssembly>
4+
<DeterministicReport>true</DeterministicReport>
5+
<ExcludeAssembliesWithoutSources>None</ExcludeAssembliesWithoutSources>
6+
<CodeCoverage>
7+
<ModulePaths>
8+
<Include>
9+
<ModulePath>(^|.*[\\/])Microsoft\.Health\.Fhir\.[^\\/]*\.dll$</ModulePath>
10+
</Include>
11+
<Exclude>
12+
<ModulePath>(^|.*[\\/])Microsoft\.Health\.Fhir\.[^\\/]*Tests[^\\/]*\.dll$</ModulePath>
13+
<ModulePath>(^|.*[\\/])Microsoft\.Health\.Fhir\..*\.Web(\..*)?\.dll$</ModulePath>
14+
<ModulePath>(^|.*[\\/])Microsoft\.Health\.Fhir\..*\.ValueSets(\..*)?\.dll$</ModulePath>
15+
</Exclude>
16+
</ModulePaths>
17+
<Attributes>
18+
<Exclude>
19+
<Attribute>^System\.Diagnostics\.CodeAnalysis\.ExcludeFromCodeCoverageAttribute$</Attribute>
20+
</Exclude>
21+
</Attributes>
22+
<Sources>
23+
<Exclude>
24+
<Source>.*[\\/].*\.g\.cs$</Source>
25+
<Source>.*[\\/]test[\\/].*</Source>
26+
<Source>.*\.Designer\.cs$</Source>
27+
<Source>.*RegistrationExtensions\.cs$</Source>
28+
<Source>.*[\\/]Registration[\\/][^\\/]*Extensions\.cs$</Source>
29+
<Source>.*[\\/]Modules[\\/][^\\/]*Module\.cs$</Source>
30+
<Source>.*[\\/]Extensions[\\/]ServiceCollectionExtensions\.cs$</Source>
31+
<Source>.*[\\/]((Microsoft\.Health\.Fhir\.)?Api\.OpenIddict)[\\/]Configuration[\\/][^\\/]*\.cs$</Source>
32+
<Source>.*[\\/]Microsoft\.Health\.Fhir\.SqlServer[\\/]Features[\\/]Schema[\\/]Model[\\/].*\.cs$</Source>
33+
<Source>.*[\\/]SchemaManager[\\/]Command[^\\/]*\.cs$</Source>
34+
<Source>.*[\\/]SchemaManager[\\/][^\\/]*ParserBuilder\.cs$</Source>
35+
<Source>.*[\\/]Modules[\\/]KnownAssemblies\.cs$</Source>
36+
<Source>.*[\\/]Modules[\\/]FeatureFlags[\\/][^\\/]+[\\/]ValidatePostConfigureOptions\.cs$</Source>
37+
<Source>.*[\\/]XmlFormatterConfiguration\.cs$</Source>
38+
<Source>.*[\\/]Core[\\/]Configs[\\/][^\\/]*Configuration\.cs$</Source>
39+
<Source>.*[\\/]CosmosDb\.Core[\\/]Configs[\\/][^\\/]*Configuration\.cs$</Source>
40+
</Exclude>
41+
</Sources>
42+
</CodeCoverage>
43+
</Configuration>

‎Directory.Build.props‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
<GenerateDocumentationFile>true</GenerateDocumentationFile>
1010
<DebugType>embedded</DebugType>
1111
<Deterministic>true</Deterministic>
12+
<DeterministicSourcePaths Condition="'$(ContinuousIntegrationBuild)' == 'true'">true</DeterministicSourcePaths>
1213
<EmbedUntrackedSources>true</EmbedUntrackedSources>
1314
<HighEntropyVA>true</HighEntropyVA>
1415
<LangVersion>latest</LangVersion>
@@ -37,15 +38,28 @@
3738
</Choose>
3839

3940
<Choose>
40-
<When Condition="$(MSBuildProjectName.Contains('Test'))">
41+
<When Condition="$(MSBuildProjectName.Contains('Test')) AND '$(IsTestProject)' != 'false'">
4142
<PropertyGroup>
4243
<IsPackable>false</IsPackable>
4344
<CodeAnalysisRuleSet>$(MSBuildThisFileDirectory)/CustomAnalysisRules.Test.ruleset</CodeAnalysisRuleSet>
45+
<TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
46+
<TestingPlatformCaptureOutput>false</TestingPlatformCaptureOutput>
47+
<OutputType>Exe</OutputType>
4448
</PropertyGroup>
4549
<ItemGroup>
46-
<PackageReference Include="coverlet.collector" PrivateAssets="all" IncludeAssets="Runtime;Build;Native;contentFiles;Analyzers" />
50+
<PackageReference Include="YTest.MTP.XUnit2" />
51+
<PackageReference Include="Microsoft.Testing.Extensions.Retry" />
52+
<PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
53+
<PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
54+
<PackageReference Remove="xunit.runner.visualstudio" />
4755
</ItemGroup>
4856
</When>
57+
<When Condition="$(MSBuildProjectName.Contains('Test')) AND '$(IsTestProject)' == 'false'">
58+
<PropertyGroup>
59+
<IsPackable>false</IsPackable>
60+
<CodeAnalysisRuleSet>$(MSBuildThisFileDirectory)/CustomAnalysisRules.Test.ruleset</CodeAnalysisRuleSet>
61+
</PropertyGroup>
62+
</When>
4963
<Otherwise>
5064
<PropertyGroup>
5165
<AnalysisMode>AllEnabledByDefault</AnalysisMode>

‎Directory.Packages.props‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@
3939
<PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.4.0" />
4040
<PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version="1.2.0" />
4141
<PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
42-
<PackageVersion Include="coverlet.collector" Version="6.0.4" />
42+
<PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.1.0" />
43+
<PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.0.2" />
4344
<PackageVersion Include="DotNet.ReproducibleBuilds" Version="1.2.25" />
4445
<PackageVersion Include="Ensure.That" Version="10.1.0" />
4546
<PackageVersion Include="FluentValidation" Version="11.11.0" />
@@ -131,11 +132,13 @@
131132
<PackageVersion Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
132133
<PackageVersion Include="System.Net.Http" Version="4.3.4" />
133134
<PackageVersion Include="System.Private.ServiceModel" Version="4.10.3" />
135+
<PackageVersion Include="xunit.runner.visualstudio" Version="3.0.2" />
136+
<PackageVersion Include="YTest.MTP.XUnit2" Version="1.0.3" />
137+
<PackageVersion Include="Microsoft.Testing.Extensions.Retry" Version="2.0.2" />
134138
<PackageVersion Include="xunit.extensibility.core" Version="2.9.3" />
135139
<PackageVersion Include="Xunit.SkippableFact" Version="1.5.23" />
136140
<PackageVersion Include="xunit" Version="2.9.3" />
137141
<PackageVersion Include="xunit.assert" Version="2.9.3" />
138-
<PackageVersion Include="xunit.runner.visualstudio" Version="3.0.2" />
139142
<PackageVersion Include="System.Drawing.Common" Version="8.0.8" />
140143
<PackageVersion Include="Moq" Version="4.20.69" />
141144
<PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />

‎THIRDPARTYNOTICES.md‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -970,6 +970,33 @@ This file is based on or incorporates material from the projects listed below (T
970970
> See the License for the specific language governing permissions and
971971
> limitations under the License.
972972
973+
## YTest.MTP.XUnit2 1.0.3
974+
* Component Source: https://github.com/Youssef1313/YTest.MTP.XUnit2
975+
* Component Copyright and License:
976+
> The MIT License (MIT)
977+
>
978+
> Copyright (c) Youssef Victor Gerges Fahmy
979+
>
980+
> All rights reserved.
981+
>
982+
> Permission is hereby granted, free of charge, to any person obtaining a copy
983+
> of this software and associated documentation files (the "Software"), to deal
984+
> in the Software without restriction, including without limitation the rights
985+
> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
986+
> copies of the Software, and to permit persons to whom the Software is
987+
> furnished to do so, subject to the following conditions:
988+
>
989+
> The above copyright notice and this permission notice shall be included in all
990+
> copies or substantial portions of the Software.
991+
>
992+
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
993+
> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
994+
> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
995+
> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
996+
> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
997+
> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
998+
> SOFTWARE.
999+
9731000
## Xunit.SkippableFact 1.4.13
9741001
* Component Source: https://github.com/AArnott/Xunit.SkippableFact
9751002
* Component Copyright and License:
@@ -996,4 +1023,4 @@ This file is based on or incorporates material from the projects listed below (T
9961023
> (C) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software.
9971024
> (D) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license.
9981025
> (E) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement.
999-
1026+

‎build/jobs/build.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,35 +32,35 @@ steps:
3232
- ${{ if eq(parameters.unitTest, 'true') }}:
3333
- task: DotNetCoreCLI@2
3434
displayName: 'dotnet test'
35-
retryCountOnTaskFailure: 1
3635
inputs:
3736
command: test
3837
projects: '**/*UnitTests/*.csproj'
39-
arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}}'
38+
arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3'
4039
testRunTitle: 'Unit Tests'
4140

4241
- ${{ if eq(parameters.codeCoverage, 'true') }}:
4342
- task: DotNetCoreCLI@2
4443
displayName: 'dotnet test with coverage'
45-
retryCountOnTaskFailure: 1
4644
inputs:
4745
command: test
4846
projects: '**/*UnitTests/*.csproj'
49-
arguments: '--configuration $(buildConfiguration) --no-build --collect "XPlat Code Coverage" -s "$(build.sourcesDirectory)/CodeCoverage.runsettings" -v normal -f ${{parameters.targetBuildFramework}}'
47+
arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"'
5048
testRunTitle: 'Unit Tests'
49+
env:
50+
platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage'
5151
- task: reportgenerator@5
5252
displayName: 'aggregate unit test coverage'
5353
condition: succeededOrFailed()
5454
inputs:
55-
reports: '$(Agent.TempDirectory)/*/coverage.cobertura.xml'
55+
reports: '$(Agent.TempDirectory)/coverage/**/*.cobertura.xml'
5656
reporttypes: 'Cobertura'
57-
targetdir: '$(Agent.TempDirectory)/coverage'
57+
targetdir: '$(Agent.TempDirectory)/coverage-aggregated'
5858

5959
- task: PublishBuildArtifacts@1
6060
displayName: 'Publish unit test coverage artifact'
6161
condition: succeededOrFailed()
6262
inputs:
63-
pathToPublish: '$(Agent.TempDirectory)/coverage'
63+
pathToPublish: '$(Agent.TempDirectory)/coverage-aggregated'
6464
artifactName: 'Coverage_UnitTests'
6565
artifactType: 'container'
6666

‎build/jobs/e2e-tests.yml‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,29 @@ steps:
2121
parameters:
2222
appServiceName: ${{ parameters.appServiceName }}
2323

24-
- task: DotNetCoreCLI@2
24+
- task: PowerShell@2
2525
displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}'
26-
retryCountOnTaskFailure: 1
2726
inputs:
28-
command: test
29-
arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 15m --filter "FullyQualifiedName~${{parameters.appServiceType}}&${{ parameters.categoryFilter }}"'
30-
workingDirectory: "$(System.ArtifactsDirectory)"
31-
testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}'
27+
targetType: inline
28+
pwsh: true
29+
script: |
30+
$ErrorActionPreference = 'Stop'
31+
$testRoot = "$(Agent.TempDirectory)/E2ETests"
32+
$testName = "Microsoft.Health.Fhir.${{ parameters.version }}.Tests.E2E"
33+
$dll = Get-ChildItem -Path $testRoot -Recurse -Filter "$testName.dll" | Select-Object -First 1
34+
if (-not $dll) {
35+
throw "Could not find $testName.dll under $testRoot"
36+
}
37+
38+
$filter = "FullyQualifiedName~${{ parameters.appServiceType }}&${{ parameters.categoryFilter }}"
39+
$args = @('--filter', $filter, '--retry-failed-tests', '3', '--report-trx')
40+
41+
Write-Host "Running dotnet $($dll.FullName)"
42+
& dotnet $dll.FullName @args
43+
44+
if ($LASTEXITCODE -ne 0) {
45+
throw "E2E tests failed with exit code $LASTEXITCODE"
46+
}
3247
env:
3348
'TestEnvironmentUrl': $(TestEnvironmentUrl)
3449
'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
@@ -67,3 +82,15 @@ steps:
6782
'AZURESUBSCRIPTION_TENANT_ID': $(AZURESUBSCRIPTION_TENANT_ID)
6883
'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': $(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)
6984
'SYSTEM_ACCESSTOKEN': $(System.AccessToken)
85+
platformOptions__resultDirectory: '$(Agent.TempDirectory)/testresults'
86+
87+
- task: PublishTestResults@2
88+
displayName: 'Publish test results'
89+
inputs:
90+
testResultsFormat: 'VSTest'
91+
testResultsFiles: '$(Agent.TempDirectory)/testresults/**/*.trx'
92+
mergeTestResults: true
93+
testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}'
94+
# TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed
95+
failTaskOnFailedTests: false
96+
condition: succeededOrFailed()

‎build/jobs/run-cosmos-tests.yml‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,34 +57,44 @@ jobs:
5757

5858
- task: DotNetCoreCLI@2
5959
displayName: 'Run Cosmos Integration Tests with coverage'
60-
retryCountOnTaskFailure: 1
6160
inputs:
6261
command: test
6362
projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj'
64-
arguments: '--filter DisplayName!~SqlServer --configuration $(buildConfiguration) --collect "XPlat Code Coverage" -s "$(Pipeline.Workspace)/source/CodeCoverage.runsettings" -v normal --no-build -f $(defaultBuildFramework)'
63+
arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx'
6564
testRunTitle: '${{ parameters.version }} Cosmos Integration Tests'
66-
publishTestResults: true
6765
env:
6866
'CosmosDb__Host': $(CosmosDb--Host)
6967
'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)'
7068
'CosmosDb__UseManagedIdentity': true
69+
platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage'
7170
'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)'
7271
'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)'
7372
'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)'
7473
'SYSTEM_ACCESSTOKEN': $(System.AccessToken)
7574

75+
- task: PublishTestResults@2
76+
displayName: 'Publish integration test results'
77+
inputs:
78+
testResultsFormat: 'VSTest'
79+
testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx'
80+
mergeTestResults: true
81+
testRunTitle: '${{ parameters.version }} Cosmos Integration Tests'
82+
# TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed
83+
failTaskOnFailedTests: false
84+
condition: succeededOrFailed()
85+
7686
- task: reportgenerator@5
7787
displayName: 'Aggregate Cosmos integration test coverage'
7888
condition: succeededOrFailed()
7989
inputs:
80-
reports: '$(Agent.TempDirectory)/*/coverage.cobertura.xml'
90+
reports: '$(Agent.TempDirectory)/coverage/**/*.cobertura.xml'
8191
reporttypes: 'Cobertura'
82-
targetdir: '$(Agent.TempDirectory)/coverage'
92+
targetdir: '$(Agent.TempDirectory)/coverage-aggregated'
8393

8494
- task: PublishBuildArtifacts@1
8595
displayName: 'Publish Cosmos integration test coverage'
8696
inputs:
87-
pathToPublish: '$(Agent.TempDirectory)/coverage'
97+
pathToPublish: '$(Agent.TempDirectory)/coverage-aggregated'
8898
artifactName: 'Coverage_IntegrationTests_Cosmos_${{ parameters.version }}'
8999
artifactType: 'container'
90100

0 commit comments

Comments
 (0)