ready to trial live

Author: dooly123

Basis Server/BasisBenchAgent/Program.cs

@@ -192,6 +192,8 @@ private static AgentResponse StartClient(AgentRequest request)
                 UseShellExecute = false,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true,
+                // Open only so the client can be asked to leave the server before it is killed.
+                RedirectStandardInput = true,
             };
 
             Process process = Process.Start(info) ?? throw new InvalidOperationException("could not start the load client");
@@ -265,7 +267,7 @@ private static void StopClient()
         if (process == null) return;
         try
         {
-            if (!process.HasExited)
+            if (!process.HasExited && !TryStopGracefully(process, TimeSpan.FromSeconds(10)))
             {
                 process.Kill(entireProcessTree: true);
                 process.WaitForExit(15000);
@@ -279,6 +281,35 @@ private static void StopClient()
         }
     }
 
+    /// <summary>
+    /// Asks the load client to leave the server before killing it, and returns whether it did.
+    ///
+    /// <para>Killing a process runs no managed code, so every client it was simulating vanishes
+    /// without a word and the server holds each one until it times out — which pollutes the next
+    /// run's population and its admission timings. Writing to stdin is the one graceful stop that
+    /// works on both platforms: there is no SIGTERM to send on Windows, and a console app cannot be
+    /// asked to close politely any other way.</para>
+    ///
+    /// <para>The kill still happens if it does not go quietly. This buys a clean departure when it
+    /// is available; it never trades away the guarantee that the process dies.</para>
+    /// </summary>
+    private static bool TryStopGracefully(Process process, TimeSpan timeout)
+    {
+        try
+        {
+            process.StandardInput.WriteLine("stop");
+            process.StandardInput.Flush();
+        }
+        catch
+        {
+            return false;
+        }
+
+        try { return process.WaitForExit((int)timeout.TotalMilliseconds); }
+        catch { return false; }
+    }
+
+
     private static string DiscoverClientDirectory()
     {
         // Beside the agent first - that is how it ships - then the development layout.

Basis Server/BasisNetworkClient/NetworkClient.cs

@@ -50,11 +50,29 @@ public void Update(float elapsedMilliseconds)
     }
     public void Disconnect()
     {
-        IsInUse = false;
         BNL.Log("Client Called Disconnect from server");
+        NotifyServerOfDeparture();
+        Shutdown();
+        BNL.Log("Worker thread stopped.");
+    }
+    /// <summary>
+    /// Tells the server this client is leaving, and does nothing else.
+    ///
+    /// <para>Split out from <see cref="Shutdown"/> because the two costs are nothing alike. This
+    /// writes one datagram straight to the socket and returns; shutting the transport down closes
+    /// the socket and joins its logic thread. Anything stopping a whole population has to get
+    /// every one of these out before it starts paying for the teardown, or the last clients are
+    /// still queued behind thread joins when the process is killed and the server is left to time
+    /// them out one by one.</para>
+    /// </summary>
+    public void NotifyServerOfDeparture()
+    {
+        IsInUse = false;
         peer?.Disconnect();
+    }
+    /// <summary>Closes the socket and joins the transport's threads.</summary>
+    public void Shutdown()
+    {
         client?.Stop();
-
-        BNL.Log("Worker thread stopped.");
     }
 }

Basis Server/BasisNetworkClientConsole/BasisNetworkClientConsole/Client/ClientManager.cs

@@ -252,10 +252,46 @@ public async Task ReconnectClientAsync(int index)
                 BNL.Log($"Reconnected: {name} ({identity.Did}) at index {index}");
             }
         }
+        /// <summary>
+        /// Leaves the server cleanly, then tears down.
+        ///
+        /// <para>The two passes are the point. Departure notices are one datagram each and go out
+        /// in a few milliseconds for the whole population; the teardown behind them closes a socket
+        /// and joins a thread per client and takes far longer than the shutdown budget a killed or
+        /// Ctrl-C'd process gets. Interleaving them meant client N was still waiting on client
+        /// N-1's thread join when the process died, so most of a run's clients never told the
+        /// server anything and it had to time each of them out instead.</para>
+        ///
+        /// <para>Nothing is logged per client here for the same reason: a log line costs a console
+        /// lock and a file write, and a few thousand of them is itself enough to run the budget
+        /// out.</para>
+        /// </summary>
         public Task StopClientsAsync()
         {
-            if (FinalClients != null)
-                foreach (var client in FinalClients) client?.Disconnect();
+            var clients = FinalClients;
+            if (clients == null) return Task.CompletedTask;
+
+            int announced = 0;
+            foreach (var client in clients)
+            {
+                try
+                {
+                    if (client == null) continue;
+                    client.NotifyServerOfDeparture();
+                    announced++;
+                }
+                catch
+                {
+                    // A client that is already gone cannot leave twice, and one failure must not
+                    // stop the rest of the population from announcing.
+                }
+            }
+            BNL.Log($"Told the server {announced} client(s) are leaving; tearing down.");
+
+            foreach (var client in clients)
+            {
+                try { client?.Shutdown(); } catch { }
+            }
             return Task.CompletedTask;
         }
         public Configuration CreateConfig()

Basis Server/BasisNetworkClientConsole/BasisNetworkClientConsole/Program.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics;
+using System.Runtime.InteropServices;
 using Basis.Logging;
 using Basis.Network;
 using Basis.Config;
@@ -13,6 +14,7 @@ partial class Program
         private const double MovementIntervalMs = 90.0;
         private const int MaxVoiceCatchUpFrames = 5;
         private static volatile bool _running = true;
+        private static int _shutdownStarted;
 
         /// <summary>Driver iterations that took longer than DriverTickMs — the harness falling behind.</summary>
         private static long DriverOverruns;
@@ -69,19 +71,31 @@ public static async Task Main(string[] args)
             var clientManager = new ClientManager();
             clientManager.Prepare();
 
-            AppDomain.CurrentDomain.ProcessExit += (_, __) =>
+            // Every way this process is asked to stop ends in the same place, and all of them are
+            // reachable: Ctrl-C interactively, SIGTERM from docker stop or systemd, a "stop" line
+            // or a closed stdin from a harness driving it, and ProcessExit as the backstop for a
+            // plain return. Before this, only ProcessExit was handled, and ProcessExit runs on a
+            // budget measured in seconds - so a population of a few thousand never finished
+            // announcing and the server timed most of them out instead of being told.
+            AppDomain.CurrentDomain.ProcessExit += (_, __) => Shutdown(clientManager);
+
+            Console.CancelKeyPress += (_, e) =>
             {
-                Console.WriteLine("Shutting down...");
-                _running = false;
-                MicrophoneCapture.Stop();
-                clientManager.StopClientsAsync().GetAwaiter().GetResult();
-                // Close the capture file here rather than relying on the finalizer: a run is
-                // normally ended with Ctrl-C, and a half-written last record would make the
-                // whole capture unreadable to the trainer.
-                string captureSummary = BundleCaptureSink.Finish();
-                if (captureSummary != null) Console.WriteLine(captureSummary);
+                // Cancel the default kill so shutdown runs to completion rather than racing it.
+                e.Cancel = true;
+                Shutdown(clientManager);
+                Environment.Exit(0);
             };
 
+            using var sigTerm = PosixSignalRegistration.Create(PosixSignal.SIGTERM, ctx =>
+            {
+                ctx.Cancel = true;
+                Shutdown(clientManager);
+                Environment.Exit(0);
+            });
+
+            StartStopRequestWatcher(clientManager);
+
             MovementSender.Initialize(clientManager.ClientCount);
             MovementSender.VoiceSender.Initialize(clientManager.ClientCount);
 
@@ -185,6 +199,81 @@ public static async Task Main(string[] args)
             await Task.Delay(-1); // keep main alive
         }
 
+
+        /// <summary>
+        /// Runs the shutdown once, whoever asks for it first.
+        ///
+        /// <para>Ctrl-C, SIGTERM and ProcessExit can all fire for one stop - Ctrl-C in particular
+        /// runs its handler and then ProcessExit - so this has to be idempotent or the population
+        /// is torn down twice and the second pass throws on already-disposed transports.</para>
+        /// </summary>
+        private static void Shutdown(ClientManager clientManager)
+        {
+            if (Interlocked.Exchange(ref _shutdownStarted, 1) != 0) return;
+
+            Console.WriteLine("Shutting down...");
+            _running = false;
+            MicrophoneCapture.Stop();
+            clientManager.StopClientsAsync().GetAwaiter().GetResult();
+            // Close the capture file here rather than relying on the finalizer: a run is
+            // normally ended with Ctrl-C, and a half-written last record would make the
+            // whole capture unreadable to the trainer.
+            string captureSummary = BundleCaptureSink.Finish();
+            if (captureSummary != null) Console.WriteLine(captureSummary);
+        }
+
+        /// <summary>
+        /// Lets whatever started this process ask it to leave cleanly.
+        ///
+        /// <para>A harness cannot send SIGTERM on Windows, and killing the process runs no managed
+        /// code at all - which is exactly the case that leaves a server holding several thousand
+        /// peers until they time out. Watching stdin gives every platform one graceful stop: a
+        /// "stop" or "quit" line, or simply closing the stream, both mean leave now.</para>
+        ///
+        /// <para>Harmless when nobody is driving it. An interactive run just blocks on a console
+        /// nobody types into, and this thread is a background one, so it never holds up exit.</para>
+        /// </summary>
+        private static void StartStopRequestWatcher(ClientManager clientManager)
+        {
+            var thread = new Thread(() =>
+            {
+                try
+                {
+                    while (true)
+                    {
+                        string line = Console.ReadLine();
+
+                        // End of stream is NOT a stop request. A process started with stdin closed
+                        // - nohup, systemd, a detached launch - reads EOF immediately, and treating
+                        // that as "leave now" would shut the run down the moment it started. Only an
+                        // explicit word means stop; EOF just means nobody is going to send one.
+                        if (line == null) return;
+
+                        line = line.Trim();
+                        if (line.Equals("stop", StringComparison.OrdinalIgnoreCase) ||
+                            line.Equals("quit", StringComparison.OrdinalIgnoreCase) ||
+                            line.Equals("exit", StringComparison.OrdinalIgnoreCase))
+                        {
+                            break;
+                        }
+                    }
+                }
+                catch
+                {
+                    // No console to read at all. Nothing to wait for, and nothing to stop.
+                    return;
+                }
+
+                Shutdown(clientManager);
+                Environment.Exit(0);
+            })
+            {
+                Name = "StopRequestWatcher",
+                IsBackground = true,
+            };
+            thread.Start();
+        }
+
         public static void StopClient(ClientManager manager, int index)
         {
             var peer = Volatile.Read(ref manager.FinalPeers[index]);

Basis Server/BasisNetworkCompute/GpuDistanceSolver.cs

@@ -71,7 +71,17 @@ private GpuDistanceSolver(Context context, Accelerator accelerator)
                 return null;
             }
 
-            accelerator = device.CreateAccelerator(context);
+            // ScheduleBlockingSync, not the default.
+            //
+            // CUDA's default waits for a kernel by spinning, which is right for a process whose
+            // only job is the device and exactly wrong here: the point of the offload is to hand
+            // cores back to the send phase and the transport's per-peer pass, and a spinning wait
+            // hands back nothing. Measured on one sweep at 1000 players: 1.04 ms of CPU burned
+            // waiting with the default, 0.00 ms blocking, for the same work. Costs a few hundred
+            // microseconds of wakeup latency on a pass that runs at most a few times a second.
+            accelerator = device is CudaDevice cuda
+                ? cuda.CreateCudaAccelerator(context, CudaAcceleratorFlags.ScheduleBlockingSync)
+                : device.CreateAccelerator(context);
             var solver = new GpuDistanceSolver(context, accelerator);
             solver.Backend = device.AcceleratorType == AcceleratorType.Cuda ? "cuda" : "opencl";
             return solver;

Basis Server/BasisServerBenchmark/Harness/LoadClientDriver.cs

@@ -63,6 +63,8 @@ public void Start(RunOptions options)
             UseShellExecute = false,
             RedirectStandardOutput = true,
             RedirectStandardError = true,
+            // Open only so the client can be asked to leave the server before it is killed.
+            RedirectStandardInput = true,
         };
 
         Process process = Process.Start(info) ?? throw new InvalidOperationException($"Could not start {exe}");
@@ -92,7 +94,7 @@ public void Stop()
 
         try
         {
-            if (!process.HasExited)
+            if (!process.HasExited && !TryStopGracefully(process, TimeSpan.FromSeconds(10)))
             {
                 process.Kill(entireProcessTree: true);
                 process.WaitForExit(15000);
@@ -102,6 +104,34 @@ public void Stop()
         finally { try { process.Dispose(); } catch { } }
     }
 
+    /// <summary>
+    /// Asks the load client to leave the server before killing it, and returns whether it did.
+    ///
+    /// <para>Killing a process runs no managed code, so every client it was simulating vanishes
+    /// without a word and the server holds each one until it times out — which pollutes the next
+    /// run's population and its admission timings. Writing to stdin is the one graceful stop that
+    /// works on both platforms: there is no SIGTERM to send on Windows, and a console app cannot be
+    /// asked to close politely any other way.</para>
+    ///
+    /// <para>The kill still happens if it does not go quietly. This buys a clean departure when it
+    /// is available; it never trades away the guarantee that the process dies.</para>
+    /// </summary>
+    private static bool TryStopGracefully(Process process, TimeSpan timeout)
+    {
+        try
+        {
+            process.StandardInput.WriteLine("stop");
+            process.StandardInput.Flush();
+        }
+        catch
+        {
+            return false;
+        }
+
+        try { return process.WaitForExit((int)timeout.TotalMilliseconds); }
+        catch { return false; }
+    }
+
     public void Dispose() => Stop();
 
     private static string Executable(string directory)