ci: enable parallel integration test execution

.github/workflows/all_solutions.yml

@@ -16,6 +16,11 @@ on:
         required: false
         type: boolean
         default: false
+      parallelize_integration_tests:
+        description: 'Enable parallel integration test execution (requires pre-built test apps)'
+        required: false
+        type: boolean
+        default: true
 
   schedule:
     - cron: "0 9 * * 1-5"
@@ -221,6 +226,64 @@ jobs:
           MSBuild.exe -restore -m -p:Configuration=Release -p:DeployOnBuild=true -p:PublishProfile=LocalDeploy ${{ env.integration_solution_path }}
         shell: powershell
 
+      - name: Pre-publish .NET Core test apps
+        run: |
+          # Pre-publish .NET Core test apps so test fixtures can copy pre-built output
+          # instead of invoking dotnet publish at runtime. This eliminates file-lock
+          # collisions on shared dependency obj/ directories during parallel test execution.
+          # We skip class libraries and Azure Function apps (func.exe requires the original
+          # build output structure, not RID-specific publish output).
+          $runtime = "win-x64"
+          $searchPaths = @("Applications", "SharedApplications")
+          $basePath = "${{ github.workspace }}\tests\Agent\IntegrationTests"
+          # Azure Function apps are incompatible with pre-publish: func.exe start --no-build
+          # expects framework-dependent build output, not RID-specific publish output
+          $skipProjects = @("AzureFunctionApplication", "AzureFunctionInProcApplication")
+          $published = 0; $skipped = 0; $failed = 0
+          foreach ($searchPath in $searchPaths) {
+            $fullPath = Join-Path $basePath $searchPath
+            if (-not (Test-Path $fullPath)) { continue }
+            $projects = Get-ChildItem -Path $fullPath -Filter "*.csproj" -Recurse
+            foreach ($project in $projects) {
+              $content = Get-Content $project.FullName -Raw
+              # Skip explicit class libraries and excluded projects
+              if ($content -match '<OutputType>\s*Library\s*</OutputType>') { $skipped++; continue }
+              if ($skipProjects -contains $project.BaseName) { $skipped++; continue }
+              if ($content -match '<TargetFrameworks?>(.*?)</TargetFrameworks?>') {
+                $tfms = $Matches[1] -split ';'
+                foreach ($tfm in $tfms) {
+                  if ($tfm -match '^net\d+\.\d+$') {
+                    Write-Host "Pre-publishing $($project.Name) for $tfm/$runtime"
+                    $output = dotnet publish --configuration Release --runtime $runtime --framework $tfm $project.FullName 2>&1
+                    if ($LASTEXITCODE -ne 0) {
+                      Write-Warning "Failed to pre-publish $($project.Name) for $tfm/$runtime"
+                      $output | Where-Object { $_ -match "error" } | ForEach-Object { Write-Host "  $_" }
+                      $failed++
+                    } else {
+                      $published++
+                    }
+                  }
+                }
+              }
+            }
+          }
+          Write-Host "Pre-publish complete: $published succeeded, $skipped skipped (libraries), $failed failed"
+
+          # Clean up intermediate RID-specific build artifacts to reduce artifact size.
+          # dotnet publish --runtime creates large intermediate files alongside the publish/
+          # subdirectory (including self-contained runtime copies). Only publish/ is needed.
+          $cleaned = 0
+          foreach ($searchPath in $searchPaths) {
+            $fullPath = Join-Path $basePath $searchPath
+            if (-not (Test-Path $fullPath)) { continue }
+            Get-ChildItem -Path $fullPath -Directory -Recurse -Filter $runtime | ForEach-Object {
+              Get-ChildItem -Path $_.FullName -Exclude "publish" | Remove-Item -Recurse -Force
+              $cleaned++
+            }
+          }
+          Write-Host "Cleaned intermediate build artifacts from $cleaned RID directories"
+        shell: powershell
+
       - name: Archive Artifacts
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
@@ -274,6 +337,57 @@ jobs:
           MSBuild.exe -restore -m -p:Configuration=Release -p:DeployOnBuild=true -p:PublishProfile=LocalDeploy ${{ env.unbounded_solution_path }}
         shell: powershell
 
+      - name: Pre-publish .NET Core test apps
+        run: |
+          # Pre-publish .NET Core test apps so test fixtures can copy pre-built output
+          # instead of invoking dotnet publish at runtime. This eliminates file-lock
+          # collisions on shared dependency obj/ directories during parallel test execution.
+          # We skip class libraries (OutputType=Library or netstandard-only targets).
+          $runtime = "win-x64"
+          $searchPaths = @("UnboundedApplications", "SharedApplications")
+          $basePath = "${{ github.workspace }}\tests\Agent\IntegrationTests"
+          $published = 0; $skipped = 0; $failed = 0
+          foreach ($searchPath in $searchPaths) {
+            $fullPath = Join-Path $basePath $searchPath
+            if (-not (Test-Path $fullPath)) { continue }
+            $projects = Get-ChildItem -Path $fullPath -Filter "*.csproj" -Recurse
+            foreach ($project in $projects) {
+              $content = Get-Content $project.FullName -Raw
+              # Skip explicit class libraries
+              if ($content -match '<OutputType>\s*Library\s*</OutputType>') { $skipped++; continue }
+              if ($content -match '<TargetFrameworks?>(.*?)</TargetFrameworks?>') {
+                $tfms = $Matches[1] -split ';'
+                foreach ($tfm in $tfms) {
+                  if ($tfm -match '^net\d+\.\d+$') {
+                    Write-Host "Pre-publishing $($project.Name) for $tfm/$runtime"
+                    $output = dotnet publish --configuration Release --runtime $runtime --framework $tfm $project.FullName 2>&1
+                    if ($LASTEXITCODE -ne 0) {
+                      Write-Warning "Failed to pre-publish $($project.Name) for $tfm/$runtime"
+                      $output | Where-Object { $_ -match "error" } | ForEach-Object { Write-Host "  $_" }
+                      $failed++
+                    } else {
+                      $published++
+                    }
+                  }
+                }
+              }
+            }
+          }
+          Write-Host "Pre-publish complete: $published succeeded, $skipped skipped (libraries), $failed failed"
+
+          # Clean up intermediate RID-specific build artifacts to reduce artifact size.
+          $cleaned = 0
+          foreach ($searchPath in $searchPaths) {
+            $fullPath = Join-Path $basePath $searchPath
+            if (-not (Test-Path $fullPath)) { continue }
+            Get-ChildItem -Path $fullPath -Directory -Recurse -Filter $runtime | ForEach-Object {
+              Get-ChildItem -Path $_.FullName -Exclude "publish" | Remove-Item -Recurse -Force
+              $cleaned++
+            }
+          }
+          Write-Host "Cleaned intermediate build artifacts from $cleaned RID directories"
+        shell: powershell
+
       - name: Archive Artifacts
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
@@ -361,6 +475,7 @@ jobs:
       # Make this variable true to enable extra data-gathering and logging to help troubleshoot test failures, at the cost of additional time and resources
       enhanced_logging: false
       NR_DOTNET_TEST_SAVE_WORKING_DIRECTORY: 1
+      NR_DOTNET_TEST_PREBUILT_APPS: 1 # use pre-published test app output instead of dotnet publish at runtime
       azure_func_exe_path: C:\ProgramData\chocolatey\lib\azure-functions-core-tools\tools\func.exe
       NEW_RELIC_AZURE_FUNCTION_LOG_LEVEL_OVERRIDE: 1 # enables profiler debug logs when testing an azure function
       # Set an environment variable that the tests will use to set the application name.
@@ -452,14 +567,19 @@ jobs:
 
           Write-Host "Run tests"
 
-          # Test parallelization is disabled until we can solve concurrent dotnet publish issues with ConsoleMF usage
+          # Most integration tests can run in parallel with pre-built test apps.
+          # Some namespaces must remain serial because they use shared external tools
+          # that don't support concurrent instances (e.g., Azure Functions Core Tools
+          # func.exe crashes when multiple instances launch simultaneously due to shared
+          # state in storage emulator/lock files). To parallelize these, the shared tool
+          # contention would need to be resolved.
+          $noParallelNamespaces = @("AzureFunction")
+          $parallelize = "${{ inputs.parallelize_integration_tests }}"
+          if ($parallelize -eq "" -or $parallelize -eq "true") { $parallelize = $true } else { $parallelize = $false }
+          if ($noParallelNamespaces -contains "${{ matrix.namespace }}") { $parallelize = $false }
           $json = Get-Content "${{ env.integration_tests_path }}/xunit.runner.json" | ConvertFrom-Json
-          $json | Add-Member -Name "parallelizeAssembly" -Value $false -MemberType NoteProperty
-          $json | Add-Member -Name "parallelizeTestCollections" -Value $false -MemberType NoteProperty
-          # if ("${{ matrix.namespace }}" -like "Logging.*" ) {
-          #   $json.parallelizeAssembly = $true
-          #   $json.parallelizeTestCollections = $true
-          # }
+          $json | Add-Member -Name "parallelizeAssembly" -Value $parallelize -MemberType NoteProperty
+          $json | Add-Member -Name "parallelizeTestCollections" -Value $parallelize -MemberType NoteProperty
           $json | ConvertTo-Json | Out-File "${{ env.integration_tests_path }}/xunit.runner.json"
 
           ${{ env.integration_tests_path }}/NewRelic.Agent.IntegrationTests.exe -namespace NewRelic.Agent.IntegrationTests.${{ matrix.namespace }} -trx "C:\IntegrationTestWorkingDirectory\TestResults\${{ matrix.namespace }}_testResults.trx"
@@ -565,6 +685,7 @@ jobs:
       integration_tests_shared_project: ${{ github.workspace }}/tests/Agent/IntegrationTests/Shared
       unbounded_tests_path: ${{ github.workspace }}/tests/Agent/IntegrationTests/UnboundedIntegrationTests/bin/Release/net10.0
       NR_DOTNET_TEST_SAVE_WORKING_DIRECTORY: 1
+      NR_DOTNET_TEST_PREBUILT_APPS: 1 # use pre-published test app output instead of dotnet publish at runtime
       # Make this variable true to enable extra data-gathering and logging to help troubleshoot test failures, at the cost of additional time and resources
       enhanced_logging: false
       # Set an environment variable that the tests will use to set the application name.
@@ -652,10 +773,16 @@ jobs:
             netstat -no
           }
 
-          # Test parallelization is disabled until we can solve concurrent dotnet publish issues with ConsoleMF usage
+          # Most unbounded tests can run in parallel with pre-built test apps.
+          # Some namespaces must remain serial because they use shared external infrastructure
+          # with hard-coded object names (e.g., MsSql tests all query NewRelic.dbo.TeamMembers
+          # with constant values, NServiceBus5 tests share MSMQ queues). To parallelize these,
+          # each test fixture would need isolated resources (per-test databases/tables/queues).
+          $noParallelNamespaces = @("MsSql", "NServiceBus", "NServiceBus5")
+          $parallelize = $noParallelNamespaces -notcontains "${{ matrix.namespace }}"
           $json = Get-Content "${{ env.unbounded_tests_path }}/xunit.runner.json" | ConvertFrom-Json
-          $json | Add-Member -Name "parallelizeAssembly" -Value $false -MemberType NoteProperty
-          $json | Add-Member -Name "parallelizeTestCollections" -Value $false -MemberType NoteProperty
+          $json | Add-Member -Name "parallelizeAssembly" -Value $parallelize -MemberType NoteProperty
+          $json | Add-Member -Name "parallelizeTestCollections" -Value $parallelize -MemberType NoteProperty
           $json | ConvertTo-Json | Out-File "${{ env.unbounded_tests_path }}/xunit.runner.json"
 
           ${{ env.unbounded_tests_path }}/NewRelic.Agent.UnboundedIntegrationTests.exe -namespace NewRelic.Agent.UnboundedIntegrationTests.${{ matrix.namespace }} -trx "C:\IntegrationTestWorkingDirectory\TestResults\${{ matrix.namespace }}_testResults.trx"

tests/Agent/IntegrationTests/IntegrationTestHelpers/RemoteServiceFixtures/DotnetTool.cs

@@ -2,11 +2,14 @@
 // SPDX-License-Identifier: Apache-2.0
 
 using System;
+using System.Collections.Concurrent;
 using System.Diagnostics;
 
 namespace NewRelic.Agent.IntegrationTestHelpers.RemoteServiceFixtures;
 public class DotnetTool : RemoteApplication
 {
+    private static readonly ConcurrentDictionary<string, object> ToolInstallLocks = new();
+
     private readonly string _packageName;
     private readonly string _toolName;
     private readonly string _workingDirectory;
@@ -56,44 +59,46 @@ private void PublishWithDotnetExe()
         TestLogger?.WriteLine($"[DotnetTool]: executing 'dotnet {startInfo.Arguments}'");
         process.StartInfo = startInfo;
 
-        process.Start();
+        // Serialize dotnet tool installs for the same package to avoid concurrent access
+        // to the shared NuGet package cache (e.g., ~/.nuget/packages/*.nupkg file locks).
+        var lockObj = ToolInstallLocks.GetOrAdd(_packageName, _ => new object());
+        lock (lockObj)
+        {
+            process.Start();
 
-        var processOutput = new ProcessOutput(TestLogger, process, true);
+            var processOutput = new ProcessOutput(TestLogger, process, true);
 
-        // Publishes take longer in CI currently, regularly taking longer than 3 minutes.
-        // 10 minutes may or may not be extreme but stabilizes these failures.
-        const int timeoutInMilliseconds = 10 * 60 * 1000;
-        if (!process.WaitForExit(timeoutInMilliseconds))
-        {
-            TestLogger?.WriteLine($"[DotetTool]: installing dotnet tool timed out while waiting for {_packageName} to install after {timeoutInMilliseconds} milliseconds.");
-            try
+            // Publishes take longer in CI currently, regularly taking longer than 3 minutes.
+            // 10 minutes may or may not be extreme but stabilizes these failures.
+            const int timeoutInMilliseconds = 10 * 60 * 1000;
+            if (!process.WaitForExit(timeoutInMilliseconds))
             {
-                //This usually happens because another publishing job has a lock on the file(s) being copied.
-                //We send a termination request because we no longer want dotnet tool install to continue to copy files
-                //when there's a good chance that at least some of the files are missing.
-                //We can only use "kill" to request termination here, because there isn't a "close" option for non-GUI apps.
-                process.Kill();
+                TestLogger?.WriteLine($"[DotnetTool]: installing dotnet tool timed out while waiting for {_packageName} to install after {timeoutInMilliseconds} milliseconds.");
+                try
+                {
+                    process.Kill();
+                }
+                catch (Exception e)
+                {
+                    TestLogger?.WriteLine($"======[DotnetTool]: installing dotnet tool failed to kill process that installs {_packageName} with exception =====");
+                    TestLogger?.WriteLine(e.ToString());
+                    TestLogger?.WriteLine($"-----[DotnetTool]: installing dotnet tool failed to kill process that installs {_packageName} end of exception -----");
+                }
             }
-            catch (Exception e)
+            else
             {
-                TestLogger?.WriteLine($"======[DotnetTool]: installing dotnet tool failed to kill process that installs {_packageName} with exception =====");
-                TestLogger?.WriteLine(e.ToString());
-                TestLogger?.WriteLine($"-----[DotnetTool]: installing dotnet tool failed to kill process that installs {_packageName} end of exception -----");
+                Console.WriteLine($"[DotnetTool]: [{DateTime.Now}] dotnet.exe exits with code {process.ExitCode}");
             }
-        }
-        else
-        {
-            Console.WriteLine($"[DotnetTool]: [{DateTime.Now}] dotnet.exe exits with code {process.ExitCode}");
-        }
 
-        processOutput.WriteProcessOutputToLog("[DotnetTool]: installing dotnet tool");
+            processOutput.WriteProcessOutputToLog("[DotnetTool]: installing dotnet tool");
 
-        if (!process.HasExited || process.ExitCode != 0)
-        {
-            var failedToPublishMessage = "Failed to install dotnet tool";
+            if (!process.HasExited || process.ExitCode != 0)
+            {
+                var failedToPublishMessage = "Failed to install dotnet tool";
 
-            TestLogger?.WriteLine($"[DotnetTool]: {failedToPublishMessage}");
-            throw new Exception(failedToPublishMessage);
+                TestLogger?.WriteLine($"[DotnetTool]: {failedToPublishMessage}");
+                throw new Exception(failedToPublishMessage);
+            }
         }
 
         sw.Stop();

tests/Agent/IntegrationTests/IntegrationTestHelpers/RemoteServiceFixtures/RemoteService.cs

@@ -104,16 +104,22 @@ public override void CopyToRemote()
 
     private void PublishWithDotnetExe(string framework)
     {
+        var deployPath = Path.Combine(DestinationRootDirectoryPath, ApplicationDirectoryName);
+        var runtime = Utilities.CurrentRuntime;
+
+        if (TryCopyPrebuiltPublishOutput(framework, runtime, deployPath))
+        {
+            return;
+        }
+
         var projectFile = Path.Combine(SourceApplicationsDirectoryPath, ApplicationDirectoryName,
             ApplicationDirectoryName + ".csproj");
-        var deployPath = Path.Combine(DestinationRootDirectoryPath, ApplicationDirectoryName);
 
         TestLogger?.WriteLine($"[RemoteService]: Publishing to {deployPath}.");
 
         var sw = new Stopwatch();
         sw.Start();
 
-        var runtime = Utilities.CurrentRuntime;
         var process = new Process();
         var startInfo = new ProcessStartInfo
         {
@@ -171,6 +177,39 @@ private void PublishWithDotnetExe(string framework)
         Console.WriteLine($"[{DateTime.Now}] Successfully published {projectFile} to {deployPath} in {sw.Elapsed}");
     }
 
+    /// <summary>
+    /// When NR_DOTNET_TEST_PREBUILT_APPS is set (e.g., in CI), copies pre-published output
+    /// instead of invoking dotnet publish. This avoids file-lock collisions on shared dependency
+    /// obj/ directories when multiple fixtures publish in parallel.
+    /// </summary>
+    private bool TryCopyPrebuiltPublishOutput(string framework, string runtime, string deployPath)
+    {
+        if (Environment.GetEnvironmentVariable("NR_DOTNET_TEST_PREBUILT_APPS") != "1")
+        {
+            return false;
+        }
+
+        var prebuiltPath = Path.Combine(SourceApplicationsDirectoryPath, ApplicationDirectoryName,
+            "bin", "Release", framework, runtime, "publish");
+
+        if (!Directory.Exists(prebuiltPath))
+        {
+            TestLogger?.WriteLine($"[RemoteService]: Pre-built publish output not found at {prebuiltPath}, falling back to dotnet publish.");
+            return false;
+        }
+
+        var sw = new Stopwatch();
+        sw.Start();
+
+        TestLogger?.WriteLine($"[RemoteService]: Copying pre-built publish output from {prebuiltPath} to {deployPath}.");
+        Directory.CreateDirectory(deployPath);
+        CommonUtils.CopyDirectory(prebuiltPath, deployPath);
+
+        sw.Stop();
+        Console.WriteLine($"[{DateTime.Now}] Copied pre-built publish output to {deployPath} in {sw.Elapsed}");
+        return true;
+    }
+
     private object GetPublishLockObjectForCoreApp()
     {
         return PublishCoreAppLocks.GetOrAdd(ApplicationDirectoryName, _ => new object());