Skip to content

Commit 4da13a9

Browse files
paulmedynskipaulmedynski
authored
Cherry-pick to 6.1 of: Fix code coverage job disk full (#3798) (#3813)
1 parent b21442d commit 4da13a9

2 files changed

Lines changed: 182 additions & 158 deletions

File tree

‎eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml‎

Lines changed: 178 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -3,160 +3,194 @@
33
# The .NET Foundation licenses this file to you under the MIT license. #
44
# See the LICENSE file in the project root for more information. #
55
#################################################################################
6+
7+
# This job processes code coverage reports generated during test runs,
8+
# merges them, generates code coverage reports, publishes them to the
9+
# pipeline, and uploads them to CodeCov.
10+
611
parameters:
12+
13+
# True to include debug steps.
714
- name: debug
815
type: boolean
916
default: false
1017

11-
- name: upload
18+
# The pool image to use.
19+
- name: image
1220
type: string
13-
default: $(ci_var_uploadTestResult)
1421

15-
- name: poolName
22+
# The agent pool name.
23+
- name: pool
1624
type: string
17-
default: $(defaultHostedPoolName)
1825

19-
- name: image
20-
type: string
21-
default: 'windows-2022'
22-
23-
- name: downloadArtifactsSteps
24-
type: stepList
25-
default: []
26+
# Array of target frameworks to process code coverage for:
27+
#
28+
# e.g. [net462, net8.0, net9.0]
29+
#
30+
- name: targetFrameworks
31+
type: object
32+
33+
# True to upload code coverage results to CodeCov.
34+
- name: upload
35+
type: boolean
36+
default: true
2637

2738
jobs:
28-
- job: CodeCoverage
29-
displayName: 'Merge Code Coverage'
30-
31-
variables:
32-
uploadTestResult: ${{ parameters.upload }}
33-
34-
pool:
35-
name: '${{ parameters.poolName }}'
36-
vmImage: ${{ parameters.image }}
37-
38-
steps:
39-
- ${{if eq(parameters.debug, true)}}:
40-
- powershell: |
41-
Get-ChildItem env: | Sort-Object Name
42-
displayName: 'List Environment Variables [debug]'
43-
44-
- task: NuGetAuthenticate@1
45-
displayName: 'NuGet Authenticate'
46-
47-
- template: ../steps/ensure-dotnet-version.yml@self
48-
parameters:
49-
packageType: 'sdk'
50-
version: '8.0'
51-
52-
- ${{ parameters.downloadArtifactsSteps }}
53-
54-
- ${{ if eq(parameters.debug, true)}}:
55-
- powershell: |
56-
Get-ChildItem $(Build.SourcesDirectory)\coverageNetFx\ -Recurse -File -Filter *.coverage
57-
displayName: 'List coverageNetFx files [debug]'
58-
59-
- powershell: |
60-
Get-ChildItem $(Build.SourcesDirectory)\coverageNetCore\ -Recurse -File -Filter *.coverage
61-
displayName: 'List coverageNetCore files [debug]'
62-
63-
- pwsh: |
64-
dotnet tool install --global dotnet-coverage
65-
66-
function MergeFiles {
67-
param(
68-
[string]$InputDirectoryPath,
69-
[string]$OutputDirectoryName
70-
)
71-
72-
$files = Get-ChildItem $InputDirectoryPath -Recurse -File -Filter *.coverage
73-
74-
# echo $files
75-
mkdir $OutputDirectoryName
76-
$counter=0
77-
78-
$toProcess = @()
79-
80-
foreach ($file in $files) {
81-
$toProcess += @{
82-
File = $file.FullName
83-
OutputFile = "$OutputDirectoryName\$counter.coveragexml"
39+
- job: CodeCoverage
40+
displayName: Publish Code Coverage
41+
42+
pool:
43+
name: ${{ parameters.pool }}
44+
${{ if eq(parameters.pool, 'Azure Pipelines') }}:
45+
vmImage: ${{ parameters.image }}
46+
${{ else }}:
47+
demands:
48+
- imageOverride -equals ${{ parameters.image }}
49+
50+
variables:
51+
netFxDir: $(Build.SourcesDirectory)\coverageNetFx
52+
netCoreDir: $(Build.SourcesDirectory)\coverageNetCore
53+
54+
steps:
55+
- ${{if eq(parameters.debug, true)}}:
56+
- pwsh: |
57+
Get-ChildItem env: | Sort-Object Name
58+
displayName: 'List Environment Variables [debug]'
59+
60+
- pwsh: Get-Volume
61+
displayName: '[Debug] Show Disk Usage'
62+
63+
- pwsh: 'Get-ChildItem env: | Sort-Object Name'
64+
displayName: '[Debug] List Environment Variables'
65+
66+
- template: ../steps/ensure-dotnet-version.yml@self
67+
parameters:
68+
packageType: sdk
69+
version: '9.0'
70+
71+
- pwsh: |
72+
dotnet tool install --global dotnet-coverage
73+
dotnet tool install --global dotnet-reportgenerator-globaltool
74+
displayName: Install dotnet tools
75+
76+
- ${{ each targetFramework in parameters.targetFrameworks }}:
77+
- task: DownloadPipelineArtifact@2
78+
displayName: 'Download Coverage Reports [${{ targetFramework }}]'
79+
inputs:
80+
itemPattern: '**\${{ targetFramework }}*'
81+
${{ if startsWith(targetFramework, 'net4') }}:
82+
targetPath: $(netFxDir)
83+
${{ else }}:
84+
targetPath: $(netCoreDir)
85+
86+
- ${{if eq(parameters.debug, true)}}:
87+
- pwsh: Get-Volume
88+
displayName: '[Debug] Show Disk Usage'
89+
90+
- pwsh: Get-ChildItem $(netFxDir) -Recurse -File -Filter *.coverage
91+
displayName: '[Debug] List coverageNetFx files'
92+
93+
- pwsh: Get-ChildItem $(netCoreDir) -Recurse -File -Filter *.coverage
94+
displayName: '[Debug] List coverageNetCore files'
95+
96+
- pwsh: |
97+
function MergeFiles {
98+
param(
99+
[string]$InputDirectoryPath,
100+
[string]$OutputDirectoryName
101+
)
102+
103+
$files = Get-ChildItem $InputDirectoryPath -Recurse -File -Filter *.coverage
104+
105+
# echo $files
106+
mkdir $OutputDirectoryName
107+
$counter=0
108+
$toProcess = @()
109+
110+
foreach ($file in $files) {
111+
$toProcess += @{
112+
File = $file.FullName
113+
OutputFile = "$OutputDirectoryName\$counter.coveragexml"
114+
}
115+
116+
$counter++
84117
}
85118
86-
$counter++
87-
}
88-
89-
$jobs = @()
90-
foreach ($file in $toProcess){
91-
$jobs += Start-ThreadJob -ScriptBlock {
92-
$params = $using:file
93-
& dotnet-coverage merge $($params.File) --output $($params.OutputFile) --output-format xml
94-
}
95-
}
96-
97-
Write-Host "Merging started..."
98-
Wait-Job -Job $jobs
99-
100-
foreach ($job in $jobs) {
101-
Receive-Job -Job $job -Wait -AutoRemoveJob
102-
}
103-
}
104-
105-
MergeFiles -InputDirectoryPath "$(Build.SourcesDirectory)\coverageNetFx\" -OutputDirectoryName "coverageNetFxXml"
106-
MergeFiles -InputDirectoryPath "$(Build.SourcesDirectory)\coverageNetCore\" -OutputDirectoryName "coverageNetCoreXml"
107-
108-
# dir coverageNetFxXml\
109-
# dir coverageNetCoreXml\
110-
111-
Write-Host "Clean up disk ... [removing coverageNetFx & coverageNetCore]"
112-
113-
Remove-Item $(Build.SourcesDirectory)\coverageNetFx -Recurse -Force
114-
Remove-Item $(Build.SourcesDirectory)\coverageNetCore -Recurse -Force
115-
116-
displayName: 'Convert coverage files to xml'
117-
118-
- ${{ if eq(parameters.debug, true)}}:
119-
- powershell: |
120-
dir coverageNetFxXml\
121-
dir coverageNetCoreXml\
122-
displayName: 'List converted files [debug]'
123-
124-
- pwsh: |
125-
dotnet tool install dotnet-reportgenerator-globaltool --tool-path tools
126-
127-
$jobs = @()
128-
$jobs += Start-ThreadJob -ScriptBlock {
129-
& tools\reportgenerator "-reports:coverageNetFxXml\*.coveragexml" "-targetdir:coveragereportNetFx" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
130-
}
131-
132-
$jobs += Start-ThreadJob -ScriptBlock {
133-
& tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportAddOns" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider" "-classfilters:+Microsoft.Data.*"
134-
}
135-
136-
$jobs += Start-ThreadJob -ScriptBlock {
137-
& tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportNetCore" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
138-
}
139-
140-
Write-Host "Running ReportGenerator..."
141-
Wait-Job -Job $jobs
142-
143-
foreach ($job in $jobs) {
144-
Receive-Job -Job $job -Wait -AutoRemoveJob
145-
}
146-
displayName: 'Run ReportGenerator'
147-
148-
- task: PublishCodeCoverageResults@2
149-
displayName: 'Publish code coverage from netcore'
150-
inputs:
151-
summaryFileLocation: '*\Cobertura.xml'
152-
153-
- powershell: |
154-
#download Codecov CLI
155-
$ProgressPreference = 'SilentlyContinue'
156-
Invoke-WebRequest -Uri https://cli.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
157-
158-
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetFx\Cobertura.xml" -F netfx
159-
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetCore\Cobertura.xml" -F netcore
160-
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportAddOns\Cobertura.xml" -F addons
161-
displayName: 'Upload to CodeCov'
162-
condition: and(succeeded(), eq(variables['uploadTestResult'], 'true'))
119+
$jobs = @()
120+
foreach ($file in $toProcess){
121+
$jobs += Start-ThreadJob -ScriptBlock {
122+
$params = $using:file
123+
& dotnet-coverage merge $($params.File) --output $($params.OutputFile) --output-format xml
124+
}
125+
}
126+
127+
Write-Host "Merging started..."
128+
Wait-Job -Job $jobs
129+
130+
foreach ($job in $jobs) {
131+
Receive-Job -Job $job -Wait -AutoRemoveJob
132+
}
133+
}
134+
135+
MergeFiles -InputDirectoryPath "$(netFxDir)" -OutputDirectoryName "coverageNetFxXml"
136+
MergeFiles -InputDirectoryPath "$(netCoreDir)" -OutputDirectoryName "coverageNetCoreXml"
137+
138+
Write-Host "Removing original coverage files..."
139+
Remove-Item $(netFxDir) -Recurse -Force -ErrorAction SilentlyContinue
140+
Remove-Item $(netCoreDir) -Recurse -Force -ErrorAction SilentlyContinue
141+
displayName: Convert coverage files to xml
142+
143+
- ${{if eq(parameters.debug, true)}}:
144+
- pwsh: Get-Volume
145+
displayName: '[Debug] Show Disk Usage'
146+
147+
- pwsh: |
148+
dir coverageNetFxXml\
149+
dir coverageNetCoreXml\
150+
displayName: '[Debug] List converted files'
151+
152+
- pwsh: |
153+
$jobs = @()
154+
$jobs += Start-ThreadJob -ScriptBlock {
155+
& reportgenerator "-reports:coverageNetFxXml\*.coveragexml" "-targetdir:coveragereportNetFx" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
156+
}
157+
158+
$jobs += Start-ThreadJob -ScriptBlock {
159+
& reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportNetCore" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
160+
}
161+
162+
$jobs += Start-ThreadJob -ScriptBlock {
163+
& reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportAddOns" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider" "-classfilters:+Microsoft.Data.*"
164+
}
165+
166+
Write-Host "Running ReportGenerator..."
167+
Wait-Job -Job $jobs
168+
169+
foreach ($job in $jobs) {
170+
Receive-Job -Job $job -Wait -AutoRemoveJob
171+
}
172+
173+
Write-Host "Removing merged XML files..."
174+
Remove-Item coverageNetFxXml -Recurse -Force -ErrorAction SilentlyContinue
175+
Remove-Item coverageNetCoreXml -Recurse -Force -ErrorAction SilentlyContinue
176+
displayName: Run ReportGenerator
177+
178+
- ${{if eq(parameters.debug, true)}}:
179+
- pwsh: Get-Volume
180+
displayName: '[Debug] Show Disk Usage'
181+
182+
- task: PublishCodeCoverageResults@2
183+
displayName: Publish code coverage results
184+
inputs:
185+
summaryFileLocation: '*\Cobertura.xml'
186+
187+
- ${{if eq(parameters.upload, true)}}:
188+
- pwsh: |
189+
#download Codecov CLI
190+
$ProgressPreference = 'SilentlyContinue'
191+
Invoke-WebRequest -Uri https://cli.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
192+
193+
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetFx\Cobertura.xml" -F netfx
194+
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetCore\Cobertura.xml" -F netcore
195+
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportAddOns\Cobertura.xml" -F addons
196+
displayName: Upload to CodeCov

‎eng/pipelines/dotnet-sqlclient-ci-core.yml‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,6 @@ variables:
9292
- name: artifactName
9393
value: Artifacts
9494

95-
- name: defaultHostedPoolName
96-
value: 'Azure Pipelines'
97-
9895
stages:
9996
- stage: build_nugets
10097
displayName: 'Build NuGet Packages'
@@ -152,16 +149,9 @@ stages:
152149
- template: common/templates/jobs/ci-code-coverage-job.yml@self
153150
parameters:
154151
debug: ${{ parameters.debug }}
155-
downloadArtifactsSteps:
156-
- ${{ each targetFramework in parameters.codeCovTargetFrameworks }}:
157-
- task: DownloadPipelineArtifact@2
158-
displayName: 'Download Coverage Reports [${{ targetFramework }}]'
159-
inputs:
160-
itemPattern: '**\${{ targetFramework }}*'
161-
${{ if contains(targetFramework, 'net4') }}:
162-
targetPath: '$(Build.SourcesDirectory)\coverageNetFx'
163-
${{ else }}:
164-
targetPath: '$(Build.SourcesDirectory)\coverageNetCore'
152+
image: ADO-MMS22-CodeCov
153+
pool: ${{ parameters.defaultPoolName }}
154+
targetFrameworks: ${{ parameters.codeCovTargetFrameworks }}
165155

166156
# test stages configurations
167157
# self hosted SQL Server on Windows
@@ -510,7 +500,7 @@ stages:
510500

511501
# Self hosted SQL Server on Mac
512502
mac_sql_22:
513-
pool: $(defaultHostedPoolName)
503+
pool: Azure Pipelines
514504
hostedPool: true
515505
images:
516506
MacOSLatest_Sql22: macos-latest

0 commit comments

Comments
 (0)