Skip to content

Commit e857159

Browse files
dooly123dooly123
committed
cleaned up a few issues from the test
1 parent 6909ecd commit e857159

119 files changed

Lines changed: 4547 additions & 249 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Basis Server/BasisNetworkClientConsole/BasisNetworkClientConsole/MessageHandler.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,12 @@ public static void OnReceive(ConsoleClientIdentity identity, int clientIndex, Ne
121121
SniffBundle(clientIndex, reader);
122122
}
123123
break;
124+
case BasisNetworkCommons.VoiceChannel:
125+
NoteVoiceDelivery(clientIndex, reader, largeId: false);
126+
break;
127+
case BasisNetworkCommons.VoiceLargeChannel:
128+
NoteVoiceDelivery(clientIndex, reader, largeId: true);
129+
break;
124130
case BasisNetworkCommons.AvatarChannel:
125131
// HVR's reliable/low-frequency path (handshake, variable definitions,
126132
// low-freq updates, high-frequency upgrades) — counting these splits
@@ -445,6 +451,28 @@ private static void NoteVoiceRange(int clientIndex, NetPacketReader reader, byte
445451
NoteSenderSeen(playerId);
446452
}
447453

454+
/// <summary>
455+
/// Books one relayed voice frame against its sender's sequence, which is what turns "the
456+
/// server says it sent voice" into "this receiver could actually have played it".
457+
///
458+
/// Wire, as written by BasisServerHandleEvents: [playerId:1|2][sequence:1][silence:1][opus].
459+
/// Read straight out of the buffer rather than through the reader, matching NoteVoiceRange —
460+
/// the reader is left untouched for anything downstream.
461+
/// </summary>
462+
private static void NoteVoiceDelivery(int clientIndex, NetPacketReader reader, bool largeId)
463+
{
464+
if (!VoiceDeliveryStats.Enabled) return;
465+
466+
int pos = reader.Position;
467+
byte[] raw = reader.RawData;
468+
int idBytes = largeId ? 2 : 1;
469+
if (raw == null || pos + idBytes + 1 > raw.Length) return;
470+
471+
int senderId = largeId ? (raw[pos] | (raw[pos + 1] << 8)) : raw[pos];
472+
byte sequence = raw[pos + idBytes];
473+
VoiceDeliveryStats.Note(clientIndex, senderId, sequence);
474+
}
475+
448476
// ── Per-sender delivery fairness ──────────────────────────────────────────────────────
449477
//
450478
// Counts inbound avatar frames per sender across the whole crowd. A server that is over

‎Basis Server/BasisNetworkClientConsole/BasisNetworkClientConsole/Program.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,23 @@ public static async Task Main(string[] args)
102102

103103
await clientManager.StartClientsAsync();
104104

105+
// Voice delivery accounting. On whenever voice is simulated: it is a per-frame dictionary
106+
// touch on the receive path, which is nothing against the avatar traffic beside it, and
107+
// without it a run can only report what the server chose to drop rather than what a
108+
// listener would actually have heard.
109+
if (Basis.Config.ConfigManager.SimulateVoice)
110+
{
111+
VoiceDeliveryStats.Enabled = true;
112+
_ = Task.Run(async () =>
113+
{
114+
while (_running)
115+
{
116+
await Task.Delay(5000);
117+
BNL.Log(VoiceDeliveryStats.Describe());
118+
}
119+
});
120+
}
121+
105122
// Periodic observer summary so a timed run ends with machine-readable totals.
106123
if (MovementSender.EmitFaceData || MessageHandler.ObserveOnly)
107124
{

‎Basis Server/BasisNetworkClientConsole/BasisNetworkClientConsole/VoiceDeliveryStats.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Threading;
4+
5+
namespace Basis.Network
6+
{
7+
/// <summary>
8+
/// Measures what a receiver actually HEARS, which is the only honest way to judge the voice path.
9+
///
10+
/// The server's own counters cannot answer this. They report what it chose to discard, not what
11+
/// arrived — a packet shed at the queue bound and a packet that never got sent look identical
12+
/// from the outside, and neither shows up as a gap until somebody tries to play the audio.
13+
///
14+
/// Every simulated voice frame already carries a per-sender sequence byte
15+
/// (MovementSender.VoiceSender.SendEncoded writes it as the first payload byte, and the server
16+
/// relays the body untouched behind a player id). Tracking that per (receiver, sender) pair turns
17+
/// the stream into a loss measurement: a jump of more than one is exactly the hole a listener
18+
/// hears.
19+
///
20+
/// Sequence is a single byte and voice runs at 50 frames/s, so it wraps every ~5.1 s. All
21+
/// arithmetic below is deliberately done in byte space for that reason.
22+
/// </summary>
23+
public static class VoiceDeliveryStats
24+
{
25+
/// <summary>
26+
/// A delta above this is read as reorder/duplicate rather than a very large gap. Server to
27+
/// client is unreliable, so late arrivals are expected and must not be counted as loss.
28+
/// Half the byte space is the only defensible split point without a wider sequence.
29+
/// </summary>
30+
private const int ReorderThreshold = 128;
31+
32+
private static long _received;
33+
private static long _lost;
34+
private static long _reordered;
35+
private static long _streams;
36+
37+
/// <summary>Last sequence seen per (receiver, sender). Guarded by <see cref="_gate"/>.</summary>
38+
private static readonly Dictionary<long, byte> _lastSeq = new Dictionary<long, byte>();
39+
private static readonly object _gate = new object();
40+
41+
public static bool Enabled;
42+
43+
public static long Received => Interlocked.Read(ref _received);
44+
public static long Lost => Interlocked.Read(ref _lost);
45+
public static long Reordered => Interlocked.Read(ref _reordered);
46+
public static long Streams => Interlocked.Read(ref _streams);
47+
48+
public static void Reset()
49+
{
50+
lock (_gate)
51+
{
52+
_lastSeq.Clear();
53+
Interlocked.Exchange(ref _received, 0);
54+
Interlocked.Exchange(ref _lost, 0);
55+
Interlocked.Exchange(ref _reordered, 0);
56+
Interlocked.Exchange(ref _streams, 0);
57+
}
58+
}
59+
60+
/// <summary>
61+
/// Records one received voice frame. <paramref name="senderId"/> and <paramref name="sequence"/>
62+
/// come straight off the wire; the caller has already stripped the player id.
63+
/// </summary>
64+
public static void Note(int receiverIndex, int senderId, byte sequence)
65+
{
66+
if (!Enabled) return;
67+
68+
Interlocked.Increment(ref _received);
69+
long key = ((long)receiverIndex << 32) | (uint)senderId;
70+
71+
lock (_gate)
72+
{
73+
if (!_lastSeq.TryGetValue(key, out byte last))
74+
{
75+
// First frame of a stream establishes the baseline. Counting the distance from
76+
// zero here would charge every talker's first packet as a burst of loss.
77+
_lastSeq[key] = sequence;
78+
Interlocked.Increment(ref _streams);
79+
return;
80+
}
81+
82+
int delta = (byte)(sequence - last);
83+
if (delta == 0 || delta > ReorderThreshold)
84+
{
85+
Interlocked.Increment(ref _reordered);
86+
return; // do not move the baseline backwards
87+
}
88+
89+
if (delta > 1)
90+
Interlocked.Add(ref _lost, delta - 1);
91+
92+
_lastSeq[key] = sequence;
93+
}
94+
}
95+
96+
/// <summary>
97+
/// Delivered share, 0..1. This is the number that answers "is voice breaking up" — at the
98+
/// bug it sat near 0.5 ("every second packet"), and a healthy path is ~1.
99+
/// </summary>
100+
public static double DeliveredFraction
101+
{
102+
get
103+
{
104+
long recv = Received, lost = Lost;
105+
long produced = recv + lost;
106+
return produced > 0 ? (double)recv / produced : 0;
107+
}
108+
}
109+
110+
public static string Describe()
111+
{
112+
long recv = Received, lost = Lost;
113+
return $"[VOICE] delivered {DeliveredFraction * 100:F2}% | received={recv} lost={lost} " +
114+
$"reordered={Reordered} streams={Streams}";
115+
}
116+
}
117+
}

‎Basis Server/BasisNetworkCore/BasisConfigXmlDocs.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ private static void RegisterServerConfig()
185185
t.Fields.Add(new FieldDoc("AvatarDeltaKeyframeMaxIntervalMs", " Ceiling for the adaptive keyframe stretch: while a sender's deltas stay tiny (idle avatar) the keyframe interval doubles up to this value; motion snaps it back to the base. Receivers that miss a keyframe request one on demand. 0 or <= base disables. int (ms); default 2000. "));
186186
t.Fields.Add(new FieldDoc("StripAdditionalDataAtLowQuality", " Drop AdditionalAvatarData (face blendshapes, custom behaviour params) from the Low and VeryLow avatar tiers — unreadable at those distances; the reliable low-frequency behaviour channel still reaches everyone. High/Medium keep it. true|false; default true. "));
187187
t.Fields.Add(new FieldDoc("EnableUplinkAvatarDelta", " Accept client-to-server avatar deltas and advertise support: clients upload a full keyframe every ~500 ms plus small deltas in between instead of full frames every packet (60-90% less avatar ingress). false = clients upload full keyframes only. true|false; default true. "));
188+
t.Fields.Add(new FieldDoc("ImageShareEgressMegabitsPerSecond", " Server egress one sharing player may spend on image replication, in megabits per second. A shared image is relayed once per recipient who is not on a direct P2P link, so this budget divided by the fan-out is the rate the sharer actually uploads at - at the old client-side assumption of 4 Mb/s a twenty-player instance moved a picture at about 25 KB/s. Sized per sharer, so the worst case is this times the number of people sharing at once; divide it down on a small pipe and raise it on a large one. int (Mb/s); 0 leaves the client on its own conservative default; default 200. "));
188189
t.Fields.Add(new FieldDoc("EnableBSRProfiling", " Emit Server Reduction System profiling output. true|false. "));
189190
t.Fields.Add(new FieldDoc("BSRMaxSliceCount", " Furthest the Server Reduction System may slice its roster under load. int; 0 = scale with player count, which is recommended. At slice N each tick serves only 1/N of the receivers, so everyone's update rate drops uniformly - it is the last-resort lever, used only after stretching the tick period and shedding distant players. This cap decides how far the server may degrade before it stops degrading and simply starts missing its tick instead. It used to be a fixed 32, chosen when 2000 was a large instance; at 8000 players a cap of 32 still leaves 250 receivers per tick, so a struggling server reaches the ceiling with nowhere left to go. Automatic keeps the per-tick fan-out roughly flat as population grows. Set a positive value only to pin it. "));
190191
t.Fields.Add(new FieldDoc("BSRMaxDegreeOfParallelism", " Worker cap for the Server Reduction System's parallel phases (send loop, message processing, distance sweep). int; 0 = automatic and recommended. Automatic scales the pool with the player count and caps it at the share of the machine the core allocator has granted this phase - a share whose ceiling is measured at runtime rather than assumed, so it already tracks the hardware. Setting a number here overrides all of that, including the measurement, and is clamped to the core count. The tick runs hundreds of times a second, so every worker costs dispatch and GC-poll traffic per tick; once the per-tick slice is large enough to keep them busy, extra workers cost more than they return. Set it only to hold the server down on a box shared with other services. Watch the 'send N/M workers' figures in the [CPU] log line to see what automatic is choosing. "));
@@ -244,7 +245,8 @@ private static void RegisterLnlConfig()
244245
t.Fields.Add(new FieldDoc("AllowPeerAddressChange", " Allow a peer's remote endpoint (IP/port) to change mid-session, e.g. mobile network roaming. true|false. "));
245246
t.Fields.Add(new FieldDoc("MergeHoldMs", " How long, in milliseconds, a partly-filled packet-merge buffer may wait for more data before being sent. float; 0 = send on every logic pass (legacy). The logic loop runs hundreds of times a second, so flushing every pass emits many half-empty datagrams and the server pays full per-packet cost for each; holding a partial buffer briefly lets consecutive passes coalesce. A buffer that fills the MTU is always sent immediately, so this caps added latency rather than adding it — only small sends ever wait, and never longer than this value. 2-5 is a reasonable range; raise it to cut packet rate further, lower it if voice latency matters more than CPU. "));
246247
t.Fields.Add(new FieldDoc("CompactMerged", " Frame merged unreliable traffic with the compact per-entry format. true|false; true is recommended. A merged message used to carry four bytes of framing (a two-byte nested length plus its own property and channel bytes); the compact form drops the property byte, since the datagram already says everything inside it is unreliable, and uses a one-byte length for payloads up to 255 - two bytes of framing, or three above 255. Different traffic still shares one MTU-sized datagram, so avatar updates and voice keep riding together. Measured at 500 players: 0.93% less total egress (~4.97 Mbit/s, about 2.24 GB/hour), 0.37% fewer UDP packets, no CPU change, no drops. This is a send-side setting and is safe to change on one end only, because both framings are always decoded; every client able to connect understands it, which the transport protocol id and the server version check together guarantee. Turn it off only to A/B the saving or to rule the framing out while diagnosing something else. "));
247-
t.Fields.Add(new FieldDoc("MaxUnreliableQueuePerPeer", " Maximum unreliable packets queued per peer before the oldest are dropped. int; 0 = size automatically from player count and available memory, which is recommended. This is the backstop that keeps an overloaded server alive: with no bound at all, a server that cannot drain its send queue grows the backlog instead of shedding, and at 2000 players that backlog reached ~40 GB before every peer timed out. Oldest are dropped first because a newer position update supersedes them. WARNING: this used to be a fixed 256, which is too small to be only a backstop - at 2000 players it discarded roughly half of every avatar update produced, and because discarding is cheaper than sending, the reduction system read the resulting fast ticks as spare capacity and produced even more. Raising it to 4096 on identical load measured zero drops, 22% more delivered bytes and 21% less CPU. Automatic sizes it per box; set a positive value only to pin it for a reproducible measurement. "));
248+
t.Fields.Add(new FieldDoc("MaxUnreliableQueuePerPeer", " Maximum unreliable packets queued per peer before the oldest are dropped. int; 0 = size automatically from player count and available memory, which is recommended. This is the backstop that keeps an overloaded server alive: with no bound at all, a server that cannot drain its send queue grows the backlog instead of shedding, and at 2000 players that backlog reached ~40 GB before every peer timed out. Oldest are dropped first because a newer position update supersedes them. WARNING: this used to be a fixed 256, which is too small to be only a backstop - at 2000 players it discarded roughly half of every avatar update produced, and because discarding is cheaper than sending, the reduction system read the resulting fast ticks as spare capacity and produced even more. Raising it to 4096 on identical load measured zero drops, 22% more delivered bytes and 21% less CPU. Automatic sizes it per box; set a positive value only to pin it for a reproducible measurement. Applies to bulk state traffic only - voice has its own queue and its own bound, see MaxPriorityUnreliableQueuePerPeer. "));
249+
t.Fields.Add(new FieldDoc("MaxPriorityUnreliableQueuePerPeer", " Maximum voice packets queued per peer before the oldest are dropped. int; 0 = size automatically from player count and available memory, which is recommended. Voice is queued separately from bulk avatar traffic and drained first, so a backlog of position updates can neither delay it nor shed it. That separation is the fix for a real bug: the bulk queue drops oldest-first because a newer avatar update supersedes the one behind it, which is not true of audio, so voice sharing that queue was being destroyed at the bulk stream's drop rate and whatever survived arrived behind the backlog, too late to play. This queue is allowed to be DEEPER than the bulk one, which sounds backwards and is the whole point: bulk depth buys avatar frames that the next frame replaces anyway, while voice depth buys audio that has no replacement. Measured at 1000 clients on a deliberately starved server, moving budget from bulk to voice improved both at once - voice delivered 85.7% to 93.6%, peak RSS 7.8 GB to 4.6 GB. A flat 256 here delivered only 32.8%, because a receiver in a crowd takes a voice packet from every audible talker every frame period and 256 covers single-digit milliseconds of that. Watch droppedVoice on /health, which should stay flat. "));
248250
t.Fields.Add(new FieldDoc("PeerUpdatePeersPerWorker", " Peers each worker in the transport's per-peer update pass is expected to service. int; 0 = 128. Lower means more workers for the same player count. This is the setting that decides how much of a large machine the server can actually use: the default was fitted to a 32-thread host, and it sizes the pool by population rather than by the machine, so at 4000 peers it picks 31 workers however many cores exist - a 128-core host then sits near a quarter utilisation. Halve it to double the workers. Tune it against the pass time in the [CPU] log line: above 25 ms with cores to spare means this is too high. Machines with many slower cores want a lower value than few fast ones, because each worker gets through fewer peers per pass. "));
249251
t.Fields.Add(new FieldDoc("PeerUpdateParallelism"," Worker cap for the transport's per-peer update pass. int; 0 = automatic and recommended. Automatic sizes the pool from the peer count and holds it inside the share the core allocator has granted this pass, which moves with load - the pass widens while it is behind and gives the cores back when it recovers. Setting a number here pins the pool and opts out of that, and is clamped to the core count. The pass runs hundreds of times a second and does little work per peer, so letting it spread across every core costs more in thread wake-up and GC-poll traffic than it saves. Watch the 'peer-update N/M workers' figures and the pass time in the [CPU] log line before overriding. "));
250252
t.Fields.Add(new FieldDoc("MaxSendSockets", " Ceiling on sockets the server may add at runtime when the network path is what limits it. int; 0 = auto (half the CPU cores, floored at 4, never above the core count). Linux only - needs SO_REUSEPORT. Each socket is both an extra send path and an extra receive thread. Sends: the send loop gets SLOWER with more threads on one socket - measured at 1000 players, 8 to 16 to 32 send workers took the update phase from 6.1 to 12.9 to 15.4 ms per tick while throughput fell from 497 to 393 MB/s - so another socket, not another core, is what adds capacity. Receives: one receive thread is one core's worth of syscall throughput, and past that the kernel discards inbound datagrams; this never appears as high CPU because the thread is pinned either way, so it is detected from the kernel's RcvbufErrors counter. On machines with many weak cores the useful range runs to 64 sockets, which is where the auto derivation lands on a 128-core host. Growth needs sustained pressure, except on receive drops which act immediately - and a socket added for drops is then checked: if the drop rate does not fall, growth stops, because the cause is not something more receive threads can fix (raise sysctl net.core.rmem_max, or the link is full). Grow-only. "));

0 commit comments

Comments
 (0)