test: resolve flaky integration test timing issues (#3503)

Author: tippmar-nr

tests/Agent/IntegrationTests/IntegrationTests/OpenTelemetry/OpenTelemetryMetricsTests.cs

@@ -35,8 +35,9 @@ protected OpenTelemetryMetricsTestsBase(TFixture fixture, ITestOutputHelper outp
             {
                 _fixture.AgentLog.WaitForLogLine(AgentLogFile.AgentConnectedLogLineRegex, TimeSpan.FromMinutes(1));
 
-                // otlp metrics export will be complete before the first analytics event harvest
-                _fixture.AgentLog.WaitForLogLine(AgentLogFile.AnalyticsEventDataLogLineRegex, TimeSpan.FromMinutes(1));
+                // Wait for actual OTLP metrics export (every 5s) rather than the analytics event
+                // harvest (every 60s), which can race with the WaitForLogLine timeout on slow CI.
+                _fixture.AgentLog.WaitForLogLine(AgentLogFile.OtlpMetricsExportedLogLineRegex, TimeSpan.FromMinutes(1));
 
                 _otlpSummaries = _fixture.GetCollectedOTLPMetrics();
             }

tests/Agent/IntegrationTests/IntegrationTests/OpenTelemetry/OpenTelemetryStressTests.cs

@@ -40,12 +40,10 @@ protected OpenTelemetryStressTestsBase(TFixture fixture, ITestOutputHelper outpu
                 _fixture.TestLogger.WriteLine($"Agent Log Path: {_fixture.AgentLog?.FilePath}");
                 _fixture.AgentLog.WaitForLogLine(AgentLogFile.AgentConnectedLogLineRegex, TimeSpan.FromMinutes(2));
 
-                // Wait for workload to complete - give more time for all metrics to be collected
-                _fixture.AgentLog.WaitForLogLine(AgentLogFile.AnalyticsEventDataLogLineRegex, TimeSpan.FromMinutes(5));
-
-                // Add extra delay to ensure all metrics are exported
-                // Increased from 30 to 45 seconds for slower CI environments
-                System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45));
+                // Wait for at least 3 OTLP exports (every 5s) to ensure sufficient metric data.
+                // This replaces a 60s analytic_event_data wait + 45s Thread.Sleep that added
+                // ~105s of unnecessary delay.
+                _fixture.AgentLog.WaitForLogLines(AgentLogFile.OtlpMetricsExportedLogLineRegex, TimeSpan.FromMinutes(2), 3);
 
                 // Don't specify exact count - just get what's available with a reasonable max
                 _otlpSummaries = _fixture.GetCollectedOTLPMetrics(count: 1000);

tests/Agent/IntegrationTests/SharedApplications/Common/MultiFunctionApplicationHelpers/NetStandardLibraries/PerformanceMetrics.cs

@@ -35,6 +35,15 @@ private static void StartAgent()
         // We need atleast one known GC invocation to verify our GC metrics.
         GC.Collect();
 
+        // GCSamplerV2 computes Gen0-only collections as: raw Gen0 count - raw Gen1 count
+        // (see ImmutableGCSample). The full GC.Collect() above increments both Gen0 and
+        // Gen1 raw counters by 1, so the Gen0-only count is 1 - 1 = 0. These two
+        // Gen0-only collections each increment the raw Gen0 counter without affecting
+        // Gen1, bringing it to (1+2) - 1 = 2 and ensuring the test sees a non-zero
+        // Gen0 metric.
+        GC.Collect(0);
+        GC.Collect(0);
+
         // Get everything started up and time for initial Sample().
         Thread.Sleep(TimeSpan.FromSeconds(10));
     }