diff --git a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Hosting.Tests/AddAWSLambdaBeforeSnapshotRequestTests.cs b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Hosting.Tests/AddAWSLambdaBeforeSnapshotRequestTests.cs index 3505d8bb3..f8de6ce9f 100644 --- a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Hosting.Tests/AddAWSLambdaBeforeSnapshotRequestTests.cs +++ b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Hosting.Tests/AddAWSLambdaBeforeSnapshotRequestTests.cs @@ -44,10 +44,16 @@ public async Task VerifyCallbackIsInvoked(LambdaEventSource hostingType) var serverTask = app.RunAsync(); - // let the server run for a max of 500 ms - await Task.WhenAny( - serverTask, - Task.Delay(TimeSpan.FromMilliseconds(500))); + // Poll for the before-snapshot callback to fire rather than racing a single fixed delay. + // A fixed 500 ms window is flaky under load (observed intermittently in CI): the + // before-snapshot request may not have completed yet, leaving the callback unset. Wait up + // to 10 seconds, checking frequently, and stop as soon as the callback has run. + var timeout = TimeSpan.FromSeconds(10); + var sw = System.Diagnostics.Stopwatch.StartNew(); + while (!callbackDidTheCallback && sw.Elapsed < timeout && !serverTask.IsCompleted) + { + await Task.Delay(TimeSpan.FromMilliseconds(50)); + } // shut down server await app.StopAsync(); diff --git a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Hosting.Tests/AssemblyInfo.cs b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Hosting.Tests/AssemblyInfo.cs new file mode 100644 index 000000000..5bc9f6b24 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Hosting.Tests/AssemblyInfo.cs @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Xunit; + +// Many tests in this assembly mutate the process-global AWS_LAMBDA_FUNCTION_NAME (and related) +// environment variables via EnvironmentVariableHelper to simulate running inside/outside Lambda. +// That state is shared across the whole process, so running test collections in parallel lets one +// test's env var leak into another (e.g. AddAWSLambdaHosting_NotInLambda_DoesNotRegisterHostingOptions +// intermittently sees a value set by a concurrently-running test and fails). Disable in-assembly +// parallelization so these env-var-dependent tests run one at a time. +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/AssemblyInfo.cs b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/AssemblyInfo.cs new file mode 100644 index 000000000..6b6d83099 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/AssemblyInfo.cs @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Xunit; + +// Tests in this assembly share process-global state: TestSnapStartInitialization sets the +// AWS_LAMBDA_FUNCTION_NAME / AWS_LAMBDA_INITIALIZATION_TYPE environment variables and drives a real +// LambdaBootstrap polling loop, and SnapStartController.Invoked is a static flag. Running test +// collections in parallel lets that background bootstrap and the env-var state interfere with other +// tests (observed in CI as the whole dotnet test process hanging until credentials expired). Disable +// in-assembly parallelization so these tests run one at a time. +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/BuildStreamingPreludeTests.cs b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/BuildStreamingPreludeTests.cs index c2971d0ab..b2c4a08ad 100644 --- a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/BuildStreamingPreludeTests.cs +++ b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/BuildStreamingPreludeTests.cs @@ -14,7 +14,10 @@ public class BuildStreamingPreludeTests { // Subclass that skips host startup entirely and // just exposes BuildStreamingPrelude directly without needing a running host. - private class StandalonePreludeBuilder : APIGatewayHttpApiV2ProxyFunction + // Derives from the REST (v1) APIGatewayProxyFunction because these tests assert on the + // multi-value MultiValueHeaders collection; the HTTP API v2 function instead collapses + // headers into the single-value Headers collection (see APIGatewayHttpApiV2ProxyFunction). + private class StandalonePreludeBuilder : APIGatewayProxyFunction { // Use the StartupMode.FirstRequest constructor so no host is started eagerly. public StandalonePreludeBuilder() diff --git a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/ResponseStreamingPropertyTests.cs b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/ResponseStreamingPropertyTests.cs index 05c6bed87..1e55ea5c7 100644 --- a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/ResponseStreamingPropertyTests.cs +++ b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/ResponseStreamingPropertyTests.cs @@ -71,7 +71,10 @@ protected override APIGatewayHttpApiV2ProxyResponse MarshallResponse( } } - private class StandalonePreludeBuilder : APIGatewayHttpApiV2ProxyFunction + // Derives from the REST (v1) APIGatewayProxyFunction because the prelude property tests + // (Property3/Property4) assert on the multi-value MultiValueHeaders collection; the HTTP + // API v2 function instead collapses headers into the single-value Headers collection. + private class StandalonePreludeBuilder : APIGatewayProxyFunction { public StandalonePreludeBuilder() : base(StartupMode.FirstRequest) { } diff --git a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestApiGatewayHttpApiV2Calls.cs b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestApiGatewayHttpApiV2Calls.cs index b28f82cc2..2c55f524e 100644 --- a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestApiGatewayHttpApiV2Calls.cs +++ b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestApiGatewayHttpApiV2Calls.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Http; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -299,15 +300,36 @@ public async Task TestSnapStartInitialization() new DefaultLambdaJsonSerializer()) .ConfigureOptions(opt => opt.RuntimeApiEndpoint = "localhost:123") .Build(); - - _ = bootstrap.RunAsync(cts.Token); + + // Keep the task so we can tear the bootstrap down before the test returns. Previously this + // was fire-and-forget (_ = bootstrap.RunAsync(...)); the polling loop (against a dead + // endpoint) then kept running after the assert, and under parallel CI load the leaked loop + // hung the whole test process until the credentials expired ~2h later. + var bootstrapTask = bootstrap.RunAsync(cts.Token); // allow some time for Bootstrap to initialize in background - await Task.Delay(100, cts.Token); + await Task.Delay(100); + + // Assert what the test is actually about (the SnapStart before-snapshot hook ran) before + // tearing down, so a teardown exception can't mask the real result. + Assert.True(SnapStartController.Invoked); await cts.CancelAsync(); - Assert.True(SnapStartController.Invoked); + // Wait for the polling loop to actually stop so no background work outlives the test. The + // bootstrap runs against a dead endpoint (localhost:123), so its loop either observes the + // cancellation (OperationCanceledException) or throws the connection failure it was + // retrying (HttpRequestException); both are expected teardown noise, not test failures. + try + { + await bootstrapTask; + } + catch (OperationCanceledException) + { + } + catch (HttpRequestException) + { + } } private async Task InvokeAPIGatewayRequest(string fileName, bool configureApiToReturnExceptionDetail = false) diff --git a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestMinimalAPI.cs b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestMinimalAPI.cs index 22709a1e3..e363a7d16 100644 --- a/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestMinimalAPI.cs +++ b/Libraries/test/Amazon.Lambda.AspNetCoreServer.Test/TestMinimalAPI.cs @@ -54,16 +54,32 @@ public T ExecuteRequest(string eventFilePath) processStartInfo.FileName = GetSystemShell(); processStartInfo.Arguments = $"{comamndArgument} dotnet run \"{requestFilePath}\" \"{responseFilePath}\""; processStartInfo.WorkingDirectory = GetTestAppDirectory(); + // Capture the child process output so a launch/build failure (e.g. the exit code 129 + // seen intermittently in CI) surfaces a diagnosable message instead of a bare exit code. + processStartInfo.RedirectStandardOutput = true; + processStartInfo.RedirectStandardError = true; + processStartInfo.UseShellExecute = false; lock (lock_process) { using var process = Process.Start(processStartInfo); - process.WaitForExit(15000); + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + + // WaitForExit(timeout) returns false when the process is still running at the + // timeout; kill it so it does not leak, then fail with the captured output. + if (!process.WaitForExit(60000)) + { + process.Kill(entireProcessTree: true); + throw new Exception( + $"Process timed out after 60s.{Environment.NewLine}STDOUT:{Environment.NewLine}{stdout.Result}{Environment.NewLine}STDERR:{Environment.NewLine}{stderr.Result}"); + } if (process.ExitCode != 0) { - throw new Exception("Process failed with exit code: " + process.ExitCode); + throw new Exception( + $"Process failed with exit code: {process.ExitCode}.{Environment.NewLine}STDOUT:{Environment.NewLine}{stdout.Result}{Environment.NewLine}STDERR:{Environment.NewLine}{stderr.Result}"); } if(!File.Exists(responseFilePath)) diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/AssemblyInfo.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/AssemblyInfo.cs new file mode 100644 index 000000000..4b6f50b5b --- /dev/null +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/AssemblyInfo.cs @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Xunit; + +// Tests in this assembly share process-global state: SetCustomerLoggerLogAction points a static +// field on the loaded Amazon.Lambda.Core assembly at a per-invocation StringWriter, and the handler +// tests drive LambdaBootstrap whose background RunAsync work writes through that static action. When +// test collections run in parallel, one test's logging can be captured by another's writer (observed +// intermittently under load as "Can't find method name in console text" in PositiveHandlerTestsAsync). +// Some classes already share a [Collection] to serialize relative to each other, but that does not +// cover the rest of the assembly, so disable in-assembly parallelization outright. +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/HandlerTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/HandlerTests.cs index dacd01f87..fe5c952ae 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/HandlerTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/HandlerTests.cs @@ -257,11 +257,15 @@ private async Task TestHandlerFailAsync(string handler, string expect using (var cancellationTokenSource = new CancellationTokenSource()) { - var exceptionWaiterTask = Task.Run(() => + var exceptionWaiterTask = Task.Run(async () => { _output.WriteLine($"Waiting for an exception."); + // Yield between checks instead of a tight spin. A busy-wait pins a CPU core and, + // under load (e.g. when several test projects run in parallel), starves the + // bootstrap thread that sets LastRecordedException, causing the test to flake. while (testRuntimeApiClient.LastRecordedException == null) { + await Task.Delay(1); } _output.WriteLine($"Exception available."); cancellationTokenSource.Cancel(); @@ -292,8 +296,12 @@ private async Task InvokeAsync(LambdaBootstrap bootstrap, string dataIn, var exceptionWaiterTask = Task.Run(async () => { _output.WriteLine($"Waiting for an output."); + // Yield between checks instead of a tight spin. A busy-wait pins a CPU core and, + // under load (e.g. when several test projects run in parallel), starves the + // bootstrap thread that sets LastOutputStream, causing the handler test to flake. while (testRuntimeApiClient.LastOutputStream == null) { + await Task.Delay(1); } _output.WriteLine($"Output available."); cancellationTokenSource.Cancel(); diff --git a/buildtools/build.proj b/buildtools/build.proj index e160116aa..54d6728d3 100644 --- a/buildtools/build.proj +++ b/buildtools/build.proj @@ -214,10 +214,10 @@ IgnoreStandardErrorWarningFormat="true"/> - - - - + +