Add dispatch-fairness benchmark scenario + bump default corpus scale

tests/LSP/Reqnroll.IdeSupport.LSP.Server.Tests/Performance/WorkspaceReloadContentionScenarioTests.cs

@@ -0,0 +1,45 @@
+#nullable enable
+
+using System.Linq;
+using System.Threading.Tasks;
+using Reqnroll.IdeSupport.LSP.Server.Benchmarks.Corpus;
+using Reqnroll.IdeSupport.LSP.Server.Benchmarks.Harness;
+using Reqnroll.IdeSupport.LSP.Server.Benchmarks.Scenarios;
+
+namespace Reqnroll.IdeSupport.LSP.Server.Tests.Performance;
+
+/// <summary>
+/// Self-test for <see cref="WorkspaceReloadContentionScenario"/> (issue #488): drives the real
+/// in-process server through a few "many tabs restored" reload storms and asserts it produces a
+/// populated baseline/under-load comparison. This assembly already disables xUnit's cross-class
+/// parallelization (see <c>AssemblyInfo.cs</c>, added for <c>ConcurrencyProbeTests</c>) -- this
+/// scenario is exactly the same "real wall-clock latency under induced contention" shape, so it
+/// needs the same isolation from unrelated concurrent test noise to produce meaningful numbers.
+/// </summary>
+public class WorkspaceReloadContentionScenarioTests
+{
+    [Fact]
+    public async Task Scenario_drives_the_real_server_and_produces_a_baseline_and_under_load_comparison()
+    {
+        var corpusRoot = CorpusLocator.FindCorpusRoot();
+
+        await using var harness = new BenchmarkLspHarness();
+        await harness.StartAsync(corpusRoot);
+
+        var features = await InteractiveScenarios.OpenFeaturesAsync(harness, corpusRoot, count: 5);
+        features.Should().HaveCountGreaterThanOrEqualTo(2);
+
+        var restoredFiles = features.Take(features.Count - 1).ToList();
+        var probe = features[^1];
+        var options = new WorkspaceReloadContentionOptions(Repetitions: 2, SettleDelayMs: 50);
+
+        var result = await new WorkspaceReloadContentionScenario(harness, restoredFiles, probe, options)
+            .RunAsync();
+
+        result.Baseline.SampleCount.Should().Be(2);
+        result.UnderLoad.SampleCount.Should().Be(2);
+        result.Baseline.P95Ms.Should().BeGreaterThanOrEqualTo(0);
+        result.UnderLoad.P95Ms.Should().BeGreaterThanOrEqualTo(0);
+        result.CeilingRatio.Should().Be(options.CeilingRatio);
+    }
+}

tests/Performance/Reqnroll.IdeSupport.LSP.Server.Benchmarks.Core/Latency/ContentionCheck.cs

@@ -0,0 +1,33 @@
+#nullable enable
+
+namespace Reqnroll.IdeSupport.LSP.Server.Benchmarks.Latency;
+
+/// <summary>
+/// A dispatch-fairness / head-of-line-blocking check (issue #488): does a cheap, unrelated request
+/// stay cheap while a workspace-wide storm of concurrent activity (e.g. many editor tabs restoring
+/// at once) is in flight? Unlike <see cref="PerfTarget"/>/<see cref="OperationResult"/>, which
+/// assert an absolute millisecond ceiling, this asserts a <b>ratio</b> of the under-load P95 to a
+/// same-run solo baseline P95.
+/// </summary>
+/// <remarks>
+/// A ratio, not an absolute-ms target, is deliberate: the #471/#477 investigation
+/// (<c>ConcurrencyProbeTests</c>'s own history) found the measured stall's absolute magnitude swings
+/// wildly with machine speed and concurrent CPU load (~15x-20x locally in isolation, ~1.3x-40x+
+/// under CI/parallel-test contention) — an absolute-ms ceiling on either the baseline or the
+/// under-load number would either never fire or fire constantly depending on the machine. The ratio
+/// to a baseline measured in the very same run cancels most of that variance out, leaving a
+/// generous <see cref="CeilingRatio"/> as a regression ceiling for "did dispatch fairness get
+/// dramatically worse," not a claim about a specific steady-state ratio.
+/// </remarks>
+public sealed record ContentionCheck(
+    string Operation,
+    LatencySummary Baseline,
+    LatencySummary UnderLoad,
+    double CeilingRatio)
+{
+    /// <summary>How much slower the cheap request got under the storm, at P95.</summary>
+    public double RatioAtP95 => UnderLoad.P95Ms / Baseline.P95Ms;
+
+    /// <summary>True when the P95 ratio stays within the regression ceiling.</summary>
+    public bool MeetsTarget => RatioAtP95 <= CeilingRatio;
+}

tests/Performance/Reqnroll.IdeSupport.LSP.Server.Benchmarks.Core/Reporting/BenchmarkReport.cs

@@ -24,10 +24,12 @@ public sealed record BenchmarkReport(
     IReadOnlyList<OperationResult> Results,
     IReadOnlyList<SkippedBatchScenario>? Skipped = null,
     SessionStats? Session = null,
-    string Transport = "in-process (in-memory pipe)")
+    string Transport = "in-process (in-memory pipe)",
+    IReadOnlyList<ContentionCheck>? ContentionChecks = null)
 {
     /// <summary>True when every asserted operation met its performance target.</summary>
-    public bool AllPassed => Results.All(r => r.MeetsTarget);
+    public bool AllPassed =>
+        Results.All(r => r.MeetsTarget) && (ContentionChecks?.All(c => c.MeetsTarget) ?? true);
 
     public string ToConsoleTable() => ConsoleReporter.Render(this);
 
@@ -86,6 +88,21 @@ public static string Render(BenchmarkReport report)
                 sb.AppendLine($"  {s.Target.Operation,-40} — {s.Reason}");
         }
 
+        if (report.ContentionChecks is { Count: > 0 })
+        {
+            sb.AppendLine();
+            sb.AppendLine("Dispatch-fairness / head-of-line blocking (cheap read latency vs. same-run solo");
+            sb.AppendLine("baseline; a ratio, not absolute ms, since both swing with machine speed -- see #488):");
+            foreach (var c in report.ContentionChecks)
+            {
+                var verdict = !report.AssertThresholds ? "—" : (c.MeetsTarget ? "PASS" : "FAIL");
+                sb.AppendLine(
+                    $"  {Trunc(c.Operation, 48),-48} baseline P95={c.Baseline.P95Ms,7:F1}ms  " +
+                    $"under-load P95={c.UnderLoad.P95Ms,8:F1}ms  ratio={c.RatioAtP95,6:F1}x  " +
+                    $"ceiling={c.CeilingRatio,4:F0}x  {verdict}");
+            }
+        }
+
         if (report.Session is { } session)
         {
             sb.AppendLine();

tests/Performance/Reqnroll.IdeSupport.LSP.Server.Benchmarks.Core/Scenarios/WorkspaceReloadContentionScenario.cs

@@ -0,0 +1,96 @@
+#nullable enable
+
+using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Reqnroll.IdeSupport.LSP.Server.Benchmarks.Harness;
+using Reqnroll.IdeSupport.LSP.Server.Benchmarks.Latency;
+
+namespace Reqnroll.IdeSupport.LSP.Server.Benchmarks.Scenarios;
+
+/// <summary>Knobs for <see cref="WorkspaceReloadContentionScenario"/>.</summary>
+public sealed record WorkspaceReloadContentionOptions(
+    int Repetitions = 5,
+    double CeilingRatio = 30.0,
+    int SettleDelayMs = 300);
+
+/// <summary>
+/// Dispatch-fairness / head-of-line-blocking scenario (issue #488, following up on #471/#477):
+/// models an editor restoring many already-open tabs at once, or a workspace-wide event (a branch
+/// switch, a <c>reqnroll.json</c> change, a solution reload) that fans a reaction out across every
+/// open feature file simultaneously — the exact "reparse-every-open-feature-file cascade" shape
+/// #477 fixed one contributor to — racing a cheap, unrelated <c>textDocument/foldingRange</c> read
+/// on a document the storm never touches.
+/// </summary>
+/// <remarks>
+/// Unlike <see cref="SessionScenario"/> (one simulated user, one active document, modest per-burst
+/// concurrency) this deliberately fires a large, workspace-wide burst of <c>didChange</c>
+/// notifications — issued together, not awaited individually, the same "pipelined on one
+/// connection" shape a real client uses — to reach the scale/shape that actually saturated the
+/// dispatch pipeline in the field, which the isolated per-operation scenarios and the one-user
+/// session scenario both undershoot.
+/// </remarks>
+public sealed class WorkspaceReloadContentionScenario
+{
+    public const string Operation = "workspace/tabs-restored#cheap-read-under-reload-storm";
+
+    private readonly BenchmarkLspHarness _harness;
+    private readonly IReadOnlyList<OpenFeature> _restoredFiles;
+    private readonly OpenFeature _probe;
+    private readonly WorkspaceReloadContentionOptions _options;
+
+    /// <param name="restoredFiles">
+    /// The "many tabs" that react together on each repetition's storm. Must already be open on
+    /// <paramref name="harness"/> (mirrors a real editor: the tabs were restored/opened before the
+    /// reload event fires).
+    /// </param>
+    /// <param name="probe">
+    /// A document the storm never edits — the "user is looking at something else" cheap-read target.
+    /// Must also already be open, and must not appear in <paramref name="restoredFiles"/>.
+    /// </param>
+    public WorkspaceReloadContentionScenario(
+        BenchmarkLspHarness harness, IReadOnlyList<OpenFeature> restoredFiles, OpenFeature probe,
+        WorkspaceReloadContentionOptions options)
+    {
+        _harness = harness;
+        _restoredFiles = restoredFiles;
+        _probe = probe;
+        _options = options;
+    }
+
+    public async Task<ContentionCheck> RunAsync()
+    {
+        var baseline = new LatencyRecorder(Operation + "-baseline");
+        var underLoad = new LatencyRecorder(Operation);
+        var version = 1000;
+
+        for (var rep = 0; rep < _options.Repetitions; rep++)
+        {
+            // Solo baseline: the cheap read with no concurrent storm in flight.
+            var baselineStart = Stopwatch.GetTimestamp();
+            await _harness.RequestFoldingRangeAsync(_probe.Uri).ConfigureAwait(false);
+            baseline.Add(Stopwatch.GetElapsedTime(baselineStart).TotalMilliseconds);
+
+            // The reload storm: every restored tab reacts at once. Notifications are fired back to
+            // back without awaiting a response (didChange has none) -- the same "issued together"
+            // shape a real client's file watcher/reload event produces on one connection, not a
+            // sequence of separately-awaited edits.
+            version++;
+            foreach (var f in _restoredFiles)
+                _harness.ChangeFeature(f.Uri, version, f.Text + $"\n  # reload-storm rep {rep} v{version}\n");
+
+            // Measure only the probe's own round-trip while the storm is in flight -- do not fold
+            // the storm's own settle time into this number.
+            var probeStart = Stopwatch.GetTimestamp();
+            await _harness.RequestFoldingRangeAsync(_probe.Uri).ConfigureAwait(false);
+            underLoad.Add(Stopwatch.GetElapsedTime(probeStart).TotalMilliseconds);
+
+            // Let this repetition's storm (reparse/diagnostics/debounced refreshes) settle before
+            // the next repetition's baseline sample, so a straggler doesn't bleed into it.
+            if (_options.SettleDelayMs > 0)
+                await Task.Delay(_options.SettleDelayMs).ConfigureAwait(false);
+        }
+
+        return new ContentionCheck(Operation, baseline.Summarize(), underLoad.Summarize(), _options.CeilingRatio);
+    }
+}

tests/Performance/Reqnroll.IdeSupport.LSP.Server.Benchmarks/BenchmarkRunner.cs

@@ -26,7 +26,11 @@ public static async Task<int> RunAsync(string[] args)
     {
         var warmup = IntArg(args, "--warmup", 10);
         var measured = IntArg(args, "--iterations", 50);
-        var fileCount = IntArg(args, "--files", 10);
+        // Default to the full committed corpus (50 files, ~1,350 steps): issue #488 found the
+        // previous default of 10 undershot the scale (~50 files / ~1,350 steps) that #471's
+        // investigation needed to reliably reproduce dispatch-pipeline contention, so routine runs
+        // never got close to it.
+        var fileCount = IntArg(args, "--files", 50);
         var outPath = StringArg(args, "--out");
         var corpusAssembly = StringArg(args, "--corpus-assembly") ?? CorpusAssemblyLocator.TryFind();
         var includeBatch = !args.Contains("--no-batch");
@@ -142,6 +146,21 @@ await BatchScenarios.ReflectionDiscoveryAsync(corpusRoot, corpusAssembly)
                 await BatchScenarios.FindUnusedStepDefinitionsAsync(harness).ConfigureAwait(false)));
         }
 
+        // Dispatch-fairness / head-of-line-blocking check (issue #488, following up on #471/#477):
+        // fire a workspace-wide didChange storm across most of the open files ("many tabs
+        // restored" / solution reload) and race it against a cheap read on a file the storm never
+        // touches. Needs at least two open files (storm set + probe); skipped below that.
+        ContentionCheck? contentionCheck = null;
+        if (features.Count >= 2)
+        {
+            Console.WriteLine("Running dispatch-fairness scenario (many tabs restored / solution reload storm)...");
+            var restoredFiles = features.Take(features.Count - 1).ToList();
+            var probe = features[^1];
+            contentionCheck = await new WorkspaceReloadContentionScenario(
+                harness, restoredFiles, probe, new WorkspaceReloadContentionOptions())
+                .RunAsync().ConfigureAwait(false);
+        }
+
         var results = summaries.Select(s => new OperationResult(s.Target, s.Summary)).ToList();
         var report = new BenchmarkReport(
             MachineName: Environment.MachineName,
@@ -152,7 +171,8 @@ await BatchScenarios.ReflectionDiscoveryAsync(corpusRoot, corpusAssembly)
                                $"{manifest.Fingerprint.StepCount} steps",
             Results: results,
             Skipped: skipped,
-            Transport: transport);
+            Transport: transport,
+            ContentionChecks: contentionCheck is not null ? new[] { contentionCheck } : null);
 
         Console.WriteLine();
         Console.WriteLine(report.ToConsoleTable());

tests/Performance/Reqnroll.IdeSupport.LSP.Server.Benchmarks/Program.cs

@@ -67,7 +67,8 @@ Drives the real LSP server and measures per-operation latency against the archit
             COMMANDS
               run                 Run the ISOLATED benchmark suite against the committed corpus —
                                   each operation measured on its own (the "contract check" against
-                                  the per-operation targets). Default when no command is given.
+                                  the per-operation targets), plus one dispatch-fairness check (see
+                                  below). Default when no command is given.
               session             Run the MIXED editing-session benchmark — interactive latency
                                   under realistic concurrent load (the "reality check"). See
                                   "'session' OPTIONS" below. Always report-only.
@@ -78,7 +79,7 @@ under realistic concurrent load (the "reality check"). See
             'run' OPTIONS
               --warmup <n>           Discarded warm-up iterations per interactive op.   (default 10)
               --iterations <n>       Measured iterations per interactive op.            (default 50)
-              --files <n>            Corpus feature files to open and drive.            (default 10)
+              --files <n>            Corpus feature files to open and drive.            (default 50)
               --out <path>           Write the results JSON (also the future regression-tracking
                                      baseline format) to <path>.
               --no-batch             Skip the batch scenarios (cold-start scan) for a quick run.
@@ -94,6 +95,16 @@ Use ONLY on a designated reference machine — shared/CI
                                      cold-start scan measure the real exe-launch cost.
               --server-exe <path>    Explicit server exe path (default: locate the built exe).
 
+            'run' also always includes a dispatch-fairness / head-of-line-blocking check (issue
+            #488, following up on #471/#477): with --files worth of feature files already open, it
+            fires a concurrent didChange storm across most of them (modelling many editor tabs
+            restoring at once, or a workspace-wide reload/config-change event) and races it against
+            a cheap textDocument/foldingRange read on the one file the storm never touches. Reported
+            as "Dispatch-fairness / head-of-line blocking" -- a ratio of the under-load read's P95 to
+            a same-run solo baseline, not an absolute-ms target (that ratio is noisy across
+            machines, but an absolute ms figure would be far noisier). Gated the same way as every
+            other target: report-only unless --assert / the reference-machine env var is set.
+
             'session' OPTIONS
               Models one user editing one active document: each edit fires a burst of requests
               (semantic tokens, outline, folding, completion) pipelined on the single connection
@@ -106,7 +117,7 @@ CodeLens are NOT part of that burst (neither fires on every keystroke in a real
               isolated-case references).
               --warmup <n>           Unrecorded warm-up bursts.                          (default 5)
               --bursts <n>           Measured edit bursts.                               (default 40)
-              --files <n>            Corpus feature files in rotation as the active doc. (default 10)
+              --files <n>            Corpus feature files in rotation as the active doc. (default 50)
               --supersede-rate <f>   Fraction of bursts cancelled mid-flight, 0..1.     (default 0.3)
               --typing-gap-ms <n>    Delay before the superseding "keystroke" cancels.   (default 2)
               --think-ms <n>         Pause between bursts (raise to model human pacing). (default 10)

tests/Performance/Reqnroll.IdeSupport.LSP.Server.Benchmarks/SessionRunner.cs

@@ -34,7 +34,9 @@ public static async Task<int> RunAsync(string[] args)
             NavigateEveryNthBurst = IntArg(args, "--navigate-every", defaults.NavigateEveryNthBurst),
             CodeLensEveryNthBurst = IntArg(args, "--codelens-every", defaults.CodeLensEveryNthBurst),
         };
-        var fileCount = IntArg(args, "--files", 10);
+        // See BenchmarkRunner's matching change (issue #488): default to the full committed corpus
+        // rather than a scale too small to reproduce realistic dispatch contention.
+        var fileCount = IntArg(args, "--files", 50);
         var outPath = StringArg(args, "--out");
         var outOfProcess = args.Contains("--out-of-process");
         var serverExe = StringArg(args, "--server-exe") ?? (outOfProcess ? ServerExeLocator.Find() : null);