run-tests-on-os.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Downloads test binaries and executes tests using dotnet test,
# runs the tests for each project on a background job in parallel,
# then uploads the results to Azure DevOps pipelines

# NOTE: Depends on environment variables $(IsNightly), $(IsWeekly), and $(DisplayFullName)

parameters:
  osName: 'Windows' # The name of the operating system for display purposes.
  framework: '' # The target framework indicating which framework tests will be run on. See: https://docs.microsoft.com/en-us/dotnet/standard/frameworks.
  binaryArtifactName: 'testbinaries' # The name of the Azure DevOps build artifact where the test assemblies will be downloaded from. Default 'testbinaries'.
  nugetArtifactName: 'nuget' # The name of the Azure DevOps build artifact where the NuGet packages will be downloaded from. Default 'nuget'.
  testResultsArtifactName: 'testresults' # The name of the Azure DevOps build artifact where the test results will be published. Default 'testresults'.
  vsTestPlatform: 'x64' # Target platform architecture used for test execution. Valid values are x86, x64, and ARM.
  testBinaryFilesPattern: '\.*\.Tests\.?[^\\/]*?\.?[^\\/]*?.dll$' # The regex pattern (within $(System.DefaultWorkingDirectory)/**/<TargetFramework>/) where to look for test .dll files, so they can be distinguished from other .dll file types.
  testResultsFileName: 'TestResults.trx' # The name of the file (not path) of the test results. Default 'TestResults.trx'.
  maximumParallelJobs: 8
  maximumAllowedFailures: 0
  where: '' # A test filter expression, as defined by dotnet test
  dotNetSdkVersion: '' # The .NET SDK version to install

steps:
- checkout: none # self represents the repo where the initial Pipelines YAML file was found

- pwsh: |
    function EnsureNotNullOrEmpty([string]$param, [string]$nameOfParam) {
        if ([string]::IsNullOrEmpty($param)) {
            Write-Host "##vso[task.logissue type=error;]Missing template parameter \"$nameOfParam\""
            Write-Host "##vso[task.complete result=Failed;]"
        }
    }
    EnsureNotNullOrEmpty('${{ parameters.osName }}', 'osName')
    EnsureNotNullOrEmpty('${{ parameters.framework }}', 'framework')
    EnsureNotNullOrEmpty('${{ parameters.binaryArtifactName }}', 'binaryArtifactName')
    EnsureNotNullOrEmpty('${{ parameters.nugetArtifactName }}', 'nugetArtifactName')
    EnsureNotNullOrEmpty('${{ parameters.testResultsArtifactName }}', 'testResultsArtifactName')
    EnsureNotNullOrEmpty('${{ parameters.vsTestPlatform }}', 'vsTestPlatform')
    EnsureNotNullOrEmpty('${{ parameters.testBinaryFilesPattern }}', 'testBinaryFilesPattern')
    EnsureNotNullOrEmpty('${{ parameters.testResultsFileName }}', 'testResultsFileName')
    EnsureNotNullOrEmpty('${{ parameters.maximumParallelJobs }}', 'maximumParallelJobs')
    EnsureNotNullOrEmpty('${{ parameters.maximumAllowedFailures }}', 'maximumAllowedFailures')
    EnsureNotNullOrEmpty('${{ parameters.dotNetSdkVersion }}', 'dotNetSdkVersion')
  displayName: 'Validate Template Parameters'


- task:  DownloadPipelineArtifact@2
  displayName: 'Download Build Artifacts: ${{ parameters.binaryArtifactName }} to $(System.DefaultWorkingDirectory)/$(parameters.framework)'
  inputs:
    artifactName: '${{ parameters.binaryArtifactName }}_${{ parameters.framework }}'
    targetPath: '$(System.DefaultWorkingDirectory)/${{ parameters.framework }}'

# Tests for lucene-cli depend on the NuGet package for testing, so we do an extra download for that file
- task:  DownloadPipelineArtifact@2
  displayName: 'Download Build Artifacts: ${{ parameters.nugetArtifactName }} to $(System.DefaultWorkingDirectory)'
  inputs:
    artifactName: '${{ parameters.nugetArtifactName }}'
    targetPath: '$(System.DefaultWorkingDirectory)'
    patterns: 'lucene-cli.*'

#- pwsh: Get-ChildItem -Path $(System.DefaultWorkingDirectory) # Uncomment for debugging

- pwsh: |
    $testPlatform = '${{ parameters.vsTestPlatform }}'
    if ($IsWindows -eq $null) {
        $IsWindows = $env:OS.StartsWith('Win')
    }
    $performMulitLevelLookup = if ($IsWindows -and $testPlatform.Equals('x86')) { 'true' } else { 'false' }
    Write-Host "##vso[task.setvariable variable=PerformMultiLevelLookup;]$performMulitLevelLookup"

#- template: 'show-all-environment-variables.yml' # Uncomment for debugging

- template: 'install-dotnet-sdk.yml'
  parameters:
    sdkVersion: '${{ parameters.dotNetSdkVersion }}'
    performMultiLevelLookup: '${{ variables.PerformMultiLevelLookup }}'

    # Hack: .NET 8+ no longer installs the x86 bits and they must be installed separately. However, it is not
    # trivial to get it into the path and to get it to pass the minimum SDK version check in runbuild.ps1.
    # So, we install it afterward and set the environment variable so the above SDK can delegate to it.
    # This code only works on Windows.
- pwsh: |
    $sdkVersion = '${{ parameters.dotNetSdkVersion }}'
    $architecture = '${{ parameters.vsTestPlatform }}'
    $installScriptPath = "${env:AGENT_TEMPDIRECTORY}/dotnet-install.ps1"
    $installScriptUrl = "https://raw.githubusercontent.com/dotnet/install-scripts/main/src/dotnet-install.ps1"
    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    Invoke-WebRequest $installScriptUrl -OutFile $installScriptPath -TimeoutSec 60
    $installPath = "${env:ProgramFiles(x86)}/dotnet"
    & $installScriptPath -Version $sdkVersion -Architecture $architecture -InstallDir $installPath
    Write-Host "##vso[task.setvariable variable=DOTNET_ROOT_X86;]$installPath"
  displayName: 'Use .NET SDK ${{ parameters.dotNetSdkVersion }} (x86)'
  condition: and(succeeded(), contains('${{ parameters.framework }}', 'net10.'), eq('${{ parameters.vsTestPlatform }}', 'x86'))

- task: UseDotNet@2
  displayName: 'Use .NET SDK 8.0.404'
  inputs:
    packageType: 'sdk'
    version: '8.0.404'
    performMultiLevelLookup: '${{ variables.PerformMultiLevelLookup }}'
  condition: and(succeeded(), contains('${{ parameters.framework }}', 'net8.'))

    # Hack: .NET 8+ no longer installs the x86 bits and they must be installed separately. However, it is not
    # trivial to get it into the path and to get it to pass the minimum SDK version check in runbuild.ps1.
    # So, we install it afterward and set the environment variable so the above SDK can delegate to it.
    # This code only works on Windows.
- pwsh: |
    $sdkVersion = '8.0.404'
    $architecture = '${{ parameters.vsTestPlatform }}'
    $installScriptPath = "${env:AGENT_TEMPDIRECTORY}/dotnet-install.ps1"
    $installScriptUrl = "https://raw.githubusercontent.com/dotnet/install-scripts/main/src/dotnet-install.ps1"
    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    Invoke-WebRequest $installScriptUrl -OutFile $installScriptPath -TimeoutSec 60
    $installPath = "${env:ProgramFiles(x86)}/dotnet"
    & $installScriptPath -Version $sdkVersion -Architecture $architecture -InstallDir $installPath
    Write-Host "##vso[task.setvariable variable=DOTNET_ROOT_X86;]$installPath"
  displayName: 'Use .NET SDK 8.0.404 (x86)'
  condition: and(succeeded(), contains('${{ parameters.framework }}', 'net8.'), eq('${{ parameters.vsTestPlatform }}', 'x86'))

- task: UseDotNet@2
  displayName: 'Use .NET SDK 9.0.308'
  inputs:
    packageType: 'sdk'
    version: '9.0.308'
    performMultiLevelLookup: '${{ variables.PerformMultiLevelLookup }}'
  condition: and(succeeded(), contains('${{ parameters.framework }}', 'net9.'))

    # Hack: .NET 8+ no longer installs the x86 bits and they must be installed separately. However, it is not
    # trivial to get it into the path and to get it to pass the minimum SDK version check in runbuild.ps1.
    # So, we install it afterward and set the environment variable so the above SDK can delegate to it.
    # This code only works on Windows.
- pwsh: |
    $sdkVersion = '9.0.308'
    $architecture = '${{ parameters.vsTestPlatform }}'
    $installScriptPath = "${env:AGENT_TEMPDIRECTORY}/dotnet-install.ps1"
    $installScriptUrl = "https://raw.githubusercontent.com/dotnet/install-scripts/main/src/dotnet-install.ps1"
    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    Invoke-WebRequest $installScriptUrl -OutFile $installScriptPath -TimeoutSec 60
    $installPath = "${env:ProgramFiles(x86)}/dotnet"
    & $installScriptPath -Version $sdkVersion -Architecture $architecture -InstallDir $installPath
    Write-Host "##vso[task.setvariable variable=DOTNET_ROOT_X86;]$installPath"
  displayName: 'Use .NET SDK 9.0.308 (x86)'
  condition: and(succeeded(), contains('${{ parameters.framework }}', 'net9.'), eq('${{ parameters.vsTestPlatform }}', 'x86'))


#- template: 'show-all-files.yml' # Uncomment for debugging
- pwsh: |
    $framework = '${{ parameters.framework }}'
    $testBinaryRootDirectory = "$(System.DefaultWorkingDirectory)"
    $testResultsArtifactDirectory = "${{ format('$(Build.ArtifactStagingDirectory)/{0}',parameters.testResultsArtifactName) }}"
    $testPlatform = '${{ parameters.vsTestPlatform }}'
    $testOSName = '${{ parameters.osName }}'
    $testBinaryFilesPattern = '${{ parameters.testBinaryFilesPattern }}'
    $testResultsFileName = '${{ parameters.testResultsFileName }}'
    $maximumParalellJobs = '${{ parameters.maximumParallelJobs }}'
    $where = '${{ parameters.where }}'
    $tempDirectory = "$(Agent.TempDirectory)"
    $isNightly = if ($env:ISNIGHTLY -eq 'true') { 'true' } else { 'false' }
    $isWeekly = if ($env:ISWEEKLY -eq 'true') { 'true' } else { 'false' }
    $displayFullName = if ($env:DISPLAYFULLNAME -eq 'false') { 'false' } else { 'true' }

    function IsSupportedFramework([string]$framework) {
        if ($IsWindows -eq $null) {
            $IsWindows = $env:OS.StartsWith('Win')
        }
        if (!$IsWindows -and $framework.StartsWith('net4')) {
            return $false
        }
        return $true
    }

    function RunTests([string]$framework, [string]$fileRegexPattern) {
        if (!(IsSupportedFramework($framework))) { continue }

        $testBinaries = Get-ChildItem -Path "$testBinaryRootDirectory" -File -Recurse | Where-Object {$_.FullName -match "$framework" -and $_.FullName -match "$fileRegexPattern" -and !$_.Name.EndsWith('.resources.dll') } | Sort-Object -Property FullName
        Write-Host $testBinaries
        foreach ($testBinary in $testBinaries) {
            $testName = [System.IO.Path]::GetFileNameWithoutExtension($testBinary.FullName)

            if ($maximumParalellJobs -gt 1) {
                # Pause if we have queued too many parallel jobs
                $running = @(Get-Job | Where-Object { $_.State -eq 'Running' })
                if ($running.Count -ge $maximumParalellJobs) {
                    Write-Host ""
                    Write-Host "  Running tests in parallel on $($running.Count) projects." -ForegroundColor Cyan
                    Write-Host "  Next in queue is $testName on $framework. This will take a bit, please wait..." -ForegroundColor Cyan
                    $running | Wait-Job -Any | Out-Null
                }
            }

            $testResultDirectory = "$testResultsArtifactDirectory/$testOSName/$framework/$testPlatform/$testName"
            if (!(Test-Path "$testResultDirectory")) {
                New-Item "$testResultDirectory" -ItemType Directory -Force | Out-Null
            }

            if ($isNightly -ne 'true' -and $isWeekly -ne 'true') {
                $blameHangTimeout = "--blame-hang-timeout 15minutes"
            } else {
                $blameHangTimeout = "--blame-hang-timeout 40minutes"
            }

            Write-Host "Running with $blameHangTimeout"

            $testExpression = "dotnet test ""$($testBinary.FullName)"" --framework ""$framework"" --blame --no-build --no-restore" + `
                " --logger:""console;verbosity=normal"" --logger:""trx;LogFileName=$testResultsFileName""" + `
                " --results-directory:""$testResultDirectory""" + `
                " --blame-hang --blame-hang-dump-type mini $blameHangTimeout"

            if (![string]::IsNullOrEmpty($where)) {
                $testExpression = "$testExpression --filter ""$where"""
            }

            $testExpression = "$testExpression -- RunConfiguration.TargetPlatform=$testPlatform"

            if ($displayFullName -eq 'true') {
                $testExpression = "$testExpression NUnit.DisplayName=FullName"
            }

            Write-Host "Testing '$($testBinary.FullName)' on framework '$framework' and outputting test results to '$testResultDirectory/$testResultsFileName'..."
            Write-Host $testExpression -ForegroundColor Magenta
            if ($maximumParalellJobs -le 1) {
                Invoke-Expression $testExpression # For running in the foreground
            } else {

                $testExpression += " > ""$testResultDirectory/dotnet-test.log"" 2> ""$testResultDirectory/dotnet-test-error.log"""
                $scriptBlock = {
                    param([string]$testExpression)
                    Invoke-Expression $testExpression
                }

                # Avoid dotnet test collisions by delaying for 500ms
                Start-Sleep -Milliseconds 500

                # Execute the jobs in parallel
                Start-Job -Name "$testName,$framework,$testPlatform" -ScriptBlock $scriptBlock -ArgumentList $testExpression | Out-Null
            }
        }
    }

    RunTests -Framework "$framework" -FileRegexPattern "$testBinaryFilesPattern"

    if ($maximumParalellJobs -gt 1) {
        # Wait for it all to complete
        do {
            $running = @(Get-Job | Where-Object { $_.State -eq 'Running' })
            if ($running.Count -gt 0) {
                Write-Host ""
                Write-Host "  Almost finished, only $($running.Count) projects left..." -ForegroundColor Cyan
                [int]$number = 0
                foreach ($runningJob in $running) {
                    $number++
                    $jobName = $runningJob | Select -ExpandProperty Name
                    Write-Host "$number. $jobName"
                }
                $running | Wait-Job -Any
            }
        } until ($running.Count -eq 0)
    }

    $global:LASTEXITCODE = 0 # Force the script to continue on error
  displayName: 'dotnet test ${{ parameters.framework }},${{ parameters.vsTestPlatform }}'
  ignoreLASTEXITCODE: true

#- template: 'show-all-files.yml' # Uncomment for debugging

- task: PublishPipelineArtifact@1
  displayName: 'Publish Artifact: ${{ parameters.testResultsArtifactName }}'
  inputs:
    targetPath: '$(Build.ArtifactStagingDirectory)/${{ parameters.testResultsArtifactName }}'
    ArtifactName: '${{ parameters.testResultsArtifactName }}_${{ parameters.osName }}_${{ parameters.framework }}_${{ parameters.vsTestPlatform }}__JobID_$(System.JobId)'
  condition: succeededOrFailed()

# Due to the fact that it is not possible to loop a task and
# it would be a ton of work to make a replacement for the
# Publish Test Results task or the (deprecated) TfsPublisher
# our only other option is to make a task for every supported
# platform and project and update it whenever a new platform
# is targeted or test project is created in Lucene.Net.

- template: 'publish-test-results-for-test-projects.yml'
  parameters:
    osName: '${{ parameters.osName }}'
    framework: '${{ parameters.framework }}'
    vsTestPlatform: '${{ parameters.vsTestPlatform }}'

- pwsh: |
    $failed = $false
    if ($env:HOSTCRASHED -eq 'true') {
        $runsExpanded = "$($env:CRASHEDRUNS)" -replace ',',"`r`n"
        Write-Host "##vso[task.logissue type=error;]Test host process(es) crashed, hung, or failed to load assemblies. Review the testresults artifacts for details. (click here to view the projects that failed)`r`n$runsExpanded"
        $failed = $true
    }
    $maximumAllowedFailures = '${{ parameters.maximumAllowedFailures }}'
    if ([int]$env:TOTALFAILURES -gt [int]$maximumAllowedFailures) {
        Write-Host "##vso[task.logissue type=error;]Test run failed due to too many failed tests. Maximum failures allowed: $maximumAllowedFailures, total failures: $($env:TOTALFAILURES)."
        $failed = $true
    }
    if ([int]$maximumAllowedFailures -eq 0 -and $env:STDERRFAILURE -eq 'true') {
        $runsExpanded = "$($env:STDERRFAILURERUNS)" -replace ',',"`r`n"
        Write-Host "##vso[task.logissue type=error;]StdErr file(s) indicate test failures. Review the testresults artifacts for details. (click here to view the projects that failed)`r`n$runsExpanded"
        $failed = $true
    }
    if ([int]$maximumAllowedFailures -eq 0 -and $env:STDOUTFAILURE -eq 'true') {
        $runsExpanded = "$($env:STDOUTFAILURERUNS)" -replace ',',"`r`n"
        Write-Host "##vso[task.logissue type=error;]StdOut file(s) indicate test failures. Review the testresults artifacts for details. (click here to view the projects that failed)`r`n$runsExpanded"
        $failed = $true
    }
    if ($failed) {
        Write-Host "##vso[task.complete result=Failed;]"
    }