Fix stress test harness isolation (#392)

Isolate process-wide unit test state and make selftest timeouts abort the host so timed-out fixture tasks cannot contaminate later TAP output.

Also make ContentDialog selftests wait for the dialog condition under stress load and preserve actionable selftest abort attribution.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: codemonkeychris

tests/Reactor.AppTests.Host/SelfTest/Fixtures/CoreCoverageFixtures.cs

@@ -334,9 +334,7 @@ public override async Task RunAsync()
                 ContentDialog("AutoOpen", TextBlock("dialog-body"), "OK") with { IsOpen = true }
             ));
 
-            await Harness.Render(100);
-
-            var dialog = FindOpenContentDialog(H, "AutoOpen");
+            var dialog = await WaitForOpenContentDialog(H, "AutoOpen");
             H.Check("ContentDialog_OpensAtMount_#246", dialog is not null);
 
             dialog?.Hide();
@@ -364,16 +362,30 @@ public override async Task RunAsync()
             H.Check("ContentDialog_NotOpenBeforeClick_#246", FindOpenContentDialog(H, "Toggled") is null);
 
             H.ClickButton("Show");
-            await Harness.Render(50);
 
-            var dialog = FindOpenContentDialog(H, "Toggled");
+            var dialog = await WaitForOpenContentDialog(H, "Toggled");
             H.Check("ContentDialog_OpensOnStateFlip_#246", dialog is not null);
 
             dialog?.Hide();
             await Harness.Render(50);
         }
     }
 
+    private static async Task<Microsoft.UI.Xaml.Controls.ContentDialog?> WaitForOpenContentDialog(
+        Harness h,
+        string title,
+        int timeoutMs = 2_000)
+    {
+        for (var elapsed = 0; elapsed <= timeoutMs; elapsed += 50)
+        {
+            await Harness.Render(elapsed == 0 ? 0 : 34);
+            var dialog = FindOpenContentDialog(h, title);
+            if (dialog is not null) return dialog;
+        }
+
+        return null;
+    }
+
     private static Microsoft.UI.Xaml.Controls.ContentDialog? FindOpenContentDialog(Harness h, string title)
     {
         var xamlRoot = (h.Window.Content as UIElement)?.XamlRoot

tests/Reactor.AppTests.Host/SelfTest/Fixtures/NativeDockingReliabilityFixture.cs

@@ -523,6 +523,8 @@ public override async Task RunAsync()
     /// </summary>
     internal class EventSubscriptionLeakBaseline(Harness h) : SelfTestFixtureBase(h)
     {
+        public override TimeSpan FixtureTimeout => TimeSpan.FromSeconds(30);
+
         public override async Task RunAsync()
         {
             var host = H.CreateHost();

tests/Reactor.AppTests.Host/SelfTest/Fixtures/NativeDockingSmokeFixture.cs

@@ -1048,7 +1048,7 @@ DockNode BuildInitial()
             host.Mount(_ => new DockManager { Layout = liveLayout });
             await Harness.Render();
             // Visual-demo "let the eye register" pauses kept short so the
-            // 20-step walk stays well inside the §SelfTestRunner.FixtureTimeout
+            // 20-step walk stays well inside the default fixture timeout
             // (15s) under CI load. Bump back up locally for slow-motion runs.
             await Task.Delay(50);
 
@@ -1233,7 +1233,7 @@ public override async Task RunAsync()
 
             host.Mount(_ => Build());
             await Harness.Render();
-            // Timing budget (must fit under SelfTestRunner.FixtureTimeout = 15s):
+            // Timing budget (must fit under the default fixture timeout):
             //   initial settle:      700ms
             //   5 loops × (5 nudges × 200ms + after-final 400ms): 7000ms
             //   total delay budget:  ~7.7s

tests/Reactor.AppTests.Host/SelfTest/SelfTestFixtureBase.cs

@@ -9,5 +9,7 @@ internal abstract class SelfTestFixtureBase
 
     protected SelfTestFixtureBase(Harness harness) => H = harness;
 
+    public virtual TimeSpan FixtureTimeout => TimeSpan.FromSeconds(15);
+
     public abstract Task RunAsync();
 }

tests/Reactor.AppTests.Host/SelfTest/SelfTestRunner.cs

@@ -25,18 +25,18 @@ internal static class SelfTestRunner
     public static bool SkipAotPatterns { get; set; } = true;
 
     // Per-fixture watchdog. A managed hang used to lock up the whole run; now
-    // we time out, mark it failed, and continue. (Note: native crashes under
-    // AOT terminate the process before this can fire — use AotSkip patterns
-    // to skip known-crashing fixtures.) Selftest fixtures normally complete
-    // in milliseconds — 15s is generous.
-    private static readonly TimeSpan FixtureTimeout = TimeSpan.FromSeconds(15);
-
-    // Off-dispatcher hang watchdog. The in-band FixtureTimeout above relies on
-    // the dispatcher processing a Task.Delay continuation, so it cannot fire
-    // when a fixture synchronously blocks the UI thread. This second watchdog
-    // runs on a background Thread (immune to dispatcher starvation) and
-    // declares a hang after HangTimeout of no progress in the fixture loop.
-    // Threshold is well past FixtureTimeout so it only catches the
+    // we time out, mark it failed, and abort the Host. Continuing in-process is
+    // unsafe because the timed-out fixture task can keep mutating UI and
+    // emitting TAP while later fixtures run. Selftest fixtures normally
+    // complete in milliseconds; long-running reliability fixtures can override
+    // SelfTestFixtureBase.FixtureTimeout explicitly.
+
+    // Off-dispatcher hang watchdog. The in-band fixture timeout relies on the
+    // dispatcher processing a Task.Delay continuation, so it cannot fire when a
+    // fixture synchronously blocks the UI thread. This second watchdog runs on
+    // a background Thread (immune to dispatcher starvation) and declares a hang
+    // after HangTimeout of no progress in the fixture loop.
+    // Threshold is well past the per-fixture timeout so it only catches the
     // dispatcher-starvation case. Override via REACTOR_SELFTEST_HANG_TIMEOUT_SECONDS;
     // set to 0 or a negative value to disable entirely (useful when attaching
     // a debugger). Also auto-disabled when Debugger.IsAttached.
@@ -258,14 +258,22 @@ public static void RunAll()
                                 // hang to this fixture by name even if the
                                 // child terminates abruptly afterward.
                                 Console.Out.Flush();
+                                var timeout = fixture.FixtureTimeout;
                                 var runTask = fixture.RunAsync();
-                                var timeoutTask = Task.Delay(FixtureTimeout);
+                                var timeoutTask = Task.Delay(timeout);
                                 var completed = await Task.WhenAny(runTask, timeoutTask);
-                                if (completed == timeoutTask)
+                                if (completed == timeoutTask && !runTask.IsCompleted)
+                                {
+                                    completed = await Task.WhenAny(runTask, Task.Delay(100));
+                                }
+
+                                if (completed != runTask)
                                 {
                                     crashed = true;
-                                    Console.WriteLine($"not ok {testIndex} {fixtureName}_TIMEOUT - exceeded {FixtureTimeout.TotalSeconds:0}s");
+                                    Console.WriteLine($"not ok {testIndex} {fixtureName}_TIMEOUT - exceeded {timeout.TotalSeconds:0}s");
+                                    Console.Out.Flush();
                                     harness.RecordFailure();
+                                    Environment.Exit(1);
                                 }
                                 else
                                 {

tests/Reactor.SelfTests/SelfTestBatch.cs

@@ -44,7 +44,7 @@ public static void RunSelfTests(TestContext context)
         if (!string.IsNullOrEmpty(stderr))
             _fullOutput += "\n--- stderr ---\n" + stderr;
 
-        ParseTap(stdout);
+        var tap = ParseTap(stdout);
 
         // Off-dispatcher watchdog in the Host emits a structured signal on
         // dispatcher-starvation hangs. Parse it from stdout *and* stderr (the
@@ -89,6 +89,8 @@ public static void RunSelfTests(TestContext context)
             _abortedReason = $"Run aborted by hang on fixture '{hangFixture}'";
         }
 
+        MarkEarlyAbortIfNeeded(exitCode, tap);
+
         _initialized = true;
 
         if (exitCode != 0 && _byFixture.IsEmpty)
@@ -131,7 +133,9 @@ private static string Tail(string s, int maxChars)
         return "..." + s[^maxChars..];
     }
 
-    private static void ParseTap(string stdout)
+    private sealed record TapParseResult(string? LastRunningFixture, bool SawTotalFailures);
+
+    private static TapParseResult ParseTap(string stdout)
     {
         // Two TAP emitter sources:
         //   Harness check:   "ok <checkName>"  /  "not ok <checkName> - <reason>"
@@ -145,10 +149,15 @@ private static void ParseTap(string stdout)
         string? current = null;
         var failuresForCurrent = new List<string>();
         var sawChecksForCurrent = false;
+        string? lastRunningFixture = null;
+        var sawTotalFailures = false;
 
         void Flush()
         {
             if (current is null) return;
+            if (_byFixture.TryGetValue(current, out var existing) && !existing.Passed && failuresForCurrent.Count == 0)
+                return;
+
             var passed = failuresForCurrent.Count == 0 && sawChecksForCurrent;
             var detail = failuresForCurrent.Count == 0
                 ? (sawChecksForCurrent ? "" : "fixture emitted no TAP checks")
@@ -163,9 +172,14 @@ void Flush()
             {
                 Flush();
                 current = line["# Running: ".Length..].Trim();
+                lastRunningFixture = current;
                 failuresForCurrent = new List<string>();
                 sawChecksForCurrent = false;
             }
+            else if (line.StartsWith("# Total failures:", StringComparison.Ordinal))
+            {
+                sawTotalFailures = true;
+            }
             else if (line.StartsWith("ok "))
             {
                 // Harness-level pass; ignore payload, just note that current saw checks.
@@ -176,9 +190,17 @@ void Flush()
                 var rest = line[7..].Trim();
                 if (TryParseRunnerLevelFailure(rest, out var fixtureName, out var detail))
                 {
-                    // Runner-level failure — attribute directly to the fixture name,
-                    // overriding any in-progress `current` bucket.
-                    _byFixture[fixtureName] = (false, detail);
+                    if (string.Equals(fixtureName, current, StringComparison.Ordinal))
+                    {
+                        sawChecksForCurrent = true;
+                        failuresForCurrent.Add(detail);
+                    }
+                    else
+                    {
+                        // Runner-level failure — attribute directly to the fixture name,
+                        // overriding any in-progress `current` bucket.
+                        _byFixture[fixtureName] = (false, detail);
+                    }
                 }
                 else
                 {
@@ -191,6 +213,7 @@ void Flush()
             }
         }
         Flush();
+        return new TapParseResult(lastRunningFixture, sawTotalFailures);
     }
 
     private static bool TryParseRunnerLevelFailure(string rest, out string fixtureName, out string detail)
@@ -218,12 +241,57 @@ private static bool TryParseRunnerLevelFailure(string rest, out string fixtureNa
         }
 
         if (namePart.Length == 0) return false;
-        fixtureName = namePart.EndsWith("_CRASH", StringComparison.Ordinal)
-            ? namePart[..^"_CRASH".Length]
-            : namePart;
+        fixtureName = StripRunnerFailureSuffix(namePart);
         return true;
     }
 
+    private static string StripRunnerFailureSuffix(string namePart)
+    {
+        string[] suffixes = ["_CRASH", "_TIMEOUT"];
+        foreach (var suffix in suffixes)
+        {
+            if (namePart.EndsWith(suffix, StringComparison.Ordinal))
+                return namePart[..^suffix.Length];
+        }
+
+        return namePart;
+    }
+
+    private static void MarkEarlyAbortIfNeeded(int exitCode, TapParseResult tap)
+    {
+        if (_abortedReason is not null || exitCode == 0 && tap.SawTotalFailures)
+            return;
+
+        var fixtureNames = FixtureNames.Value;
+        var firstMissingIndex = Array.FindIndex(fixtureNames, name => !_byFixture.ContainsKey(name));
+        if (firstMissingIndex < 0)
+            return;
+
+        var hasReportedAfterMissing = fixtureNames
+            .Skip(firstMissingIndex + 1)
+            .Any(name => _byFixture.ContainsKey(name));
+        if (hasReportedAfterMissing)
+            return;
+
+        var attributed = tap.LastRunningFixture;
+        if (attributed is not null)
+        {
+            if (!_byFixture.TryGetValue(attributed, out var existing) || existing.Passed)
+            {
+                _byFixture[attributed] = (false,
+                    $"Selftest Host exited before completing fixture '{attributed}'. " +
+                    $"Exit code: {exitCode}. Downstream fixtures were not executed.\n" +
+                    $"--- tail of full output ---\n{Tail(_fullOutput, 4000)}");
+            }
+
+            _abortedReason = $"Run aborted after fixture '{attributed}'";
+        }
+        else
+        {
+            _abortedReason = $"Run aborted before fixture '{fixtureNames[firstMissingIndex]}'";
+        }
+    }
+
     public static IEnumerable<object[]> AllFixtures => FixtureNames.Value.Select(n => new object[] { n });
 
     [DataTestMethod]

tests/Reactor.Tests/TestIsolationCollections.cs

@@ -0,0 +1,21 @@
+using Xunit;
+
+namespace Microsoft.UI.Reactor.Tests;
+
+/// <summary>
+/// xUnit collection marker for tests that replace process-wide console or trace
+/// state. These tests must not overlap any other collection because
+/// <see cref="Console.Out"/>, <see cref="Console.Error"/>, and
+/// <see cref="System.Diagnostics.Trace.Listeners"/> are global to the test process.
+/// </summary>
+[CollectionDefinition("ConsoleTests", DisableParallelization = true)]
+public sealed class ConsoleTestsCollection { }
+
+/// <summary>
+/// xUnit collection marker for tests that subscribe to
+/// <see cref="TaskScheduler.UnobservedTaskException"/> and force finalization. The
+/// event is process-wide, so these tests need exclusive execution to avoid counting
+/// faulted tasks owned by unrelated tests.
+/// </summary>
+[CollectionDefinition("UnobservedTaskException", DisableParallelization = true)]
+public sealed class UnobservedTaskExceptionCollection { }