Skip to content

Commit d3aef7c

Browse files
paulirwinpaulirwin
committed
Run docfx in parallel
1 parent 9cebb20 commit d3aef7c

2 files changed

Lines changed: 117 additions & 38 deletions

File tree

‎.github/workflows/Lucene-Net-Documentation.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ jobs:
178178

179179
- name: Build docs
180180
run: ./main-repo/websites/apidocs/docs.ps1 -Clean -LuceneNetVersion ${{ env.RELEASE_VERSION }}
181-
shell: powershell
181+
shell: pwsh
182182

183183
- name: Upload apidocs as build artifact
184184
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

‎websites/apidocs/docs.ps1‎

Lines changed: 116 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,23 @@ param (
3232
[Parameter(Mandatory = $false)]
3333
[string] $BaseUrl = 'https://lucenenet.apache.org/docs/',
3434
[Parameter(Mandatory = $false)]
35-
[int] $StagingPort = 8080
35+
[int] $StagingPort = 8080,
36+
# Number of docfx projects to build concurrently. Set to 1 to build everything serially.
37+
# Named to match the $maximumParallelJobs property used by the psake build in .build/runbuild.ps1.
38+
[Parameter(Mandatory = $false)]
39+
[int] $maximumParallelJobs = [Math]::Max(1, [Math]::Min(8, [Environment]::ProcessorCount - 1))
3640
)
3741
$MinimumSdkVersion = "8.0.100" # Minimum Required .NET SDK (must not be a pre-release)
3842

3943
$ErrorActionPreference = "Stop"
4044

45+
# This script builds the docfx projects concurrently using ForEach-Object -Parallel, which was
46+
# introduced in PowerShell 7. Windows PowerShell 5.1 fails with a confusing parameter binding
47+
# error, so check for it up front and explain what is needed.
48+
if ($PSVersionTable.PSVersion.Major -lt 7) {
49+
throw "PowerShell 7 or higher is required to build the API docs (found $($PSVersionTable.PSVersion)). Run this script with 'pwsh' instead of 'powershell'."
50+
}
51+
4152
# if the base URL is the lucene live site default value we also need to include the version
4253
if ($BaseUrl -eq 'https://lucenenet.apache.org/docs/') {
4354
$BaseUrl += $LuceneNetVersion
@@ -166,20 +177,35 @@ if ($? -and $DisableMetaData -eq $false) {
166177
Set-Location $PreviousLocation
167178
}
168179

169-
foreach ($proj in $DocFxJsonMeta) {
170-
$projFile = Join-Path -Path $ApiDocsFolder $proj
180+
# Metadata generation has no cross-project dependencies (unlike the build step below, which
181+
# consumes other projects' xref maps), and each config writes to its own obj/docfx/api/<name>
182+
# folder. That makes it safe to run these concurrently. The duplicates in $DocFxJsonMeta only
183+
# exist to work around circular xref maps during the build, so we de-duplicate here.
184+
$MetaProjects = $DocFxJsonMeta | Select-Object -Unique
185+
186+
Write-Host "Building api metadata for $($MetaProjects.Count) projects (up to $maximumParallelJobs at a time)..."
171187

172-
$DocFxLog = Join-Path -Path $ApiDocsFolder "obj\${proj}.meta.log"
188+
$MetaResults = $MetaProjects | ForEach-Object -ThrottleLimit $maximumParallelJobs -Parallel {
189+
$proj = $_
190+
$projFile = Join-Path -Path $using:ApiDocsFolder $proj
191+
$DocFxLog = Join-Path -Path $using:ApiDocsFolder "obj\${proj}.meta.log"
192+
193+
# Each runspace has its own current directory, so this does not race with the other jobs.
194+
Set-Location $using:RepoRoot
173195

174-
# build the output
175196
Write-Host "Building api metadata for $projFile..."
176-
$PreviousLocation = Get-Location
177-
Set-Location $RepoRoot
178-
try {
179-
& dotnet tool run docfx metadata $projFile --log "$DocFxLog" --logLevel $LogLevel --noRestore
180-
} finally {
181-
Set-Location $PreviousLocation
182-
}
197+
# Capture docfx's console output rather than letting it flow into this block's output
198+
# stream, which is reserved for the result object below, then echo it via Write-Host.
199+
$output = & dotnet tool run docfx metadata $projFile --log "$DocFxLog" --logLevel $using:LogLevel --noRestore 2>&1
200+
$exitCode = $LASTEXITCODE
201+
Write-Host ($output | Out-String)
202+
203+
[pscustomobject]@{ Project = $proj; ExitCode = $exitCode }
204+
}
205+
206+
$FailedMeta = @($MetaResults | Where-Object { $_.ExitCode -ne 0 })
207+
if ($FailedMeta.Count -gt 0) {
208+
throw "Failed to build api metadata for: $(($FailedMeta | ForEach-Object { $_.Project }) -join ', ')"
183209
}
184210
}
185211

@@ -201,33 +227,86 @@ if ($? -and $DisableBuild -eq $false) {
201227
# Update the CLI link to the latest LuceneNetVersion
202228
(Get-Content -Path $BreadcrumbPath -Raw) -Replace '(?<="_navCliHref":\s*?"https?\:\/\/lucenenet\.apache\.org\/docs\/)\d+?\.\d+?\.\d+?(?:\.\d+?)?(?:-\w+)?', $LuceneNetVersion | Set-Content -Path $BreadcrumbPath
203229

204-
foreach ($proj in $DocFxJsonMeta) {
205-
$projFile = Join-Path -Path $ApiDocsFolder $proj
206-
207-
$DocFxLog = Join-Path -Path $ApiDocsFolder "obj\${proj}.build.log"
208-
209-
Start-Sleep -Seconds 1
210-
211-
# build the output
212-
Write-Host "Building site output for $projFile..."
213-
$PreviousLocation = Get-Location
214-
Set-Location $RepoRoot
215-
try {
216-
& dotnet tool run docfx build $projFile --log "$DocFxLog" --logLevel $LogLevel --debug --maxParallelism 1
217-
} finally {
218-
Set-Location $PreviousLocation
230+
# Unlike metadata generation, the build step DOES have cross-project dependencies: each config
231+
# consumes the xrefmap.yml produced by the projects listed in its "xref" section. So the configs
232+
# are grouped into waves that are run one after another, while the members of a single wave -
233+
# which do not depend on each other - are built concurrently.
234+
#
235+
# The waves below preserve the ordering the serial build relied on:
236+
# 1. codecs/core/analysis-common have circular xref maps between them, so they are built in the
237+
# original order, one at a time, and core/codecs are rebuilt in the last wave to pick up the
238+
# xref maps that did not exist yet on their first pass (the "intentional duplicates").
239+
# 2. icu must precede highlighter, and queryparser/analysis-common must precede demo.
240+
$DocFxBuildWaves = @(
241+
, @("docfx.codecs.json")
242+
, @("docfx.core.json")
243+
, @("docfx.analysis-common.json")
244+
, @(
245+
"docfx.analysis-kuromoji.json",
246+
"docfx.analysis-morfologik.json",
247+
"docfx.analysis-opennlp.json",
248+
"docfx.analysis-phonetic.json",
249+
"docfx.analysis-smartcn.json",
250+
"docfx.analysis-stempel.json",
251+
"docfx.benchmark.json",
252+
"docfx.classification.json",
253+
"docfx.expressions.json",
254+
"docfx.facet.json",
255+
"docfx.grouping.json",
256+
"docfx.icu.json",
257+
"docfx.join.json",
258+
"docfx.memory.json",
259+
"docfx.misc.json",
260+
"docfx.queries.json",
261+
"docfx.queryparser.json",
262+
"docfx.replicator.json",
263+
"docfx.sandbox.json",
264+
"docfx.spatial.json",
265+
"docfx.suggest.json",
266+
"docfx.test-framework.json"
267+
)
268+
, @("docfx.highlighter.json", "docfx.demo.json")
269+
# intentional duplicates - see note above
270+
, @("docfx.codecs.json")
271+
, @("docfx.core.json")
272+
)
273+
274+
foreach ($wave in $DocFxBuildWaves) {
275+
$BuildResults = $wave | ForEach-Object -ThrottleLimit $maximumParallelJobs -Parallel {
276+
$proj = $_
277+
$projFile = Join-Path -Path $using:ApiDocsFolder $proj
278+
$DocFxLog = Join-Path -Path $using:ApiDocsFolder "obj\${proj}.build.log"
279+
280+
# Each runspace has its own current directory, so this does not race with the other jobs.
281+
Set-Location $using:RepoRoot
282+
283+
Write-Host "Building site output for $projFile..."
284+
# Capture docfx's console output rather than letting it flow into this block's output
285+
# stream, which is reserved for the result object below, then echo it via Write-Host.
286+
$output = & dotnet tool run docfx build $projFile --log "$DocFxLog" --logLevel $using:LogLevel --debug --maxParallelism 1 2>&1
287+
$exitCode = $LASTEXITCODE
288+
Write-Host ($output | Out-String)
289+
290+
if ($exitCode -eq 0) {
291+
# Add the baseUrl to the output xrefmap, see https://github.com/dotnet/docfx/issues/2346#issuecomment-356054027
292+
$projFileJson = Get-Content $projFile | ConvertFrom-Json
293+
$projBuildDest = $projFileJson.build.dest
294+
$buildOutputFolder = Join-Path -Path ((Get-Item $projFile).DirectoryName) $projBuildDest
295+
$xrefFile = Join-Path $buildOutputFolder "xrefmap.yml"
296+
$xrefMap = Get-Content $xrefFile -Raw
297+
$xrefMap = $xrefMap.Replace("### YamlMime:XRefMap", "").Trim()
298+
$projBaseUrl = $using:BaseUrl + $projBuildDest.Substring(5, $projBuildDest.Length - 5) # trim the _site part of the string
299+
$xrefMap = "### YamlMime:XRefMap" + [Environment]::NewLine + "baseUrl: " + $projBaseUrl + "/" + [Environment]::NewLine + $xrefMap
300+
Set-Content -Path $xrefFile -Value $xrefMap
301+
}
302+
303+
[pscustomobject]@{ Project = $proj; ExitCode = $exitCode }
219304
}
220305

221-
# Add the baseUrl to the output xrefmap, see https://github.com/dotnet/docfx/issues/2346#issuecomment-356054027
222-
$projFileJson = Get-Content $projFile | ConvertFrom-Json
223-
$projBuildDest = $projFileJson.build.dest
224-
$buildOutputFolder = Join-Path -Path ((Get-Item $projFile).DirectoryName) $projBuildDest
225-
$xrefFile = Join-Path $buildOutputFolder "xrefmap.yml"
226-
$xrefMap = Get-Content $xrefFile -Raw
227-
$xrefMap = $xrefMap.Replace("### YamlMime:XRefMap", "").Trim()
228-
$projBaseUrl = $BaseUrl + $projBuildDest.Substring(5, $projBuildDest.Length - 5) # trim the _site part of the string
229-
$xrefMap = "### YamlMime:XRefMap" + [Environment]::NewLine + "baseUrl: " + $projBaseUrl + "/" + [Environment]::NewLine + $xrefMap
230-
Set-Content -Path $xrefFile -Value $xrefMap
306+
$FailedBuild = @($BuildResults | Where-Object { $_.ExitCode -ne 0 })
307+
if ($FailedBuild.Count -gt 0) {
308+
throw "Failed to build site output for: $(($FailedBuild | ForEach-Object { $_.Project }) -join ', ')"
309+
}
231310
}
232311
}
233312

0 commit comments

Comments
 (0)