Add global worker statistics param (QUIC_PARAM_GLOBAL_WORKER_STATISTICS)

docs/PreviewFeatures.md

@@ -1,4 +1,4 @@
-GPreview features
+Preview features
 =========
 > [!IMPORTANT]
 >

docs/Settings.md

@@ -141,6 +141,7 @@ These parameters are accessed by calling [GetParam](./api/GetParam.md) or [SetPa
 | `QUIC_PARAM_GLOBAL_STATISTICS_V2_SIZES`<br> 12    | uint32_t[]               | Get-only  | Array of well-known sizes for each version of the QUIC_STATISTICS_V2 struct. The output array length is variable; pass a buffer of uint32_t and check BufferLength for the number of sizes returned. See GetParam documentation for usage details. |
 | `QUIC_PARAM_GLOBAL_VERSION_NEGOTIATION_ENABLED`<br> (preview) | uint8_t (BOOLEAN) | Both | Globally enable the version negotiation extension for all client and server connections. |
 | `QUIC_PARAM_GLOBAL_STATELESS_RETRY_CONFIG`<br> 13    | [QUIC_STATELESS_RETRY_CONFIG](./api/QUIC_STATELESS_RETRY_CONFIG.md) | Set-Only | Configure the stateless retry token secret, key algorithm, and key rotation interval. The secret length *must* match the AEAD algorithm key length. |
+| `QUIC_PARAM_GLOBAL_WORKER_STATISTICS`<br> 15 (preview) | QUIC_WORKER_STATISTICS_LIST | Get-only | Per-worker active and wall time statistics. The output length is variable; pass a `QUIC_WORKER_STATISTICS_LIST` buffer and use the double-call pattern to determine the required size. Not supported in kernel mode. See [GetParam](./api/GetParam.md#quic_param_global_worker_statistics) for usage details. |
 
 ## Registration Parameters
 

docs/api/GetParam.md

@@ -103,6 +103,51 @@ uint32_t NumSizes = BufferLength / sizeof(uint32_t);
 
 See also: [Settings.md](../Settings.md#global-parameters)
 
+## QUIC_PARAM_GLOBAL_WORKER_STATISTICS
+
+**Preview:** This parameter is in [preview](../PreviewFeatures.md).
+
+Returns per-worker statistics for the global worker pool, allowing an application to observe how busy each worker thread is. The returned buffer is a `QUIC_WORKER_STATISTICS_LIST` header followed by one `QUIC_WORKER_STATISTICS` entry per worker:
+
+- `CumulativeActiveTimeUs` - total time (in microseconds) the worker spent active (not idle).
+- `CumulativeWallTimeUs` - wall time (in microseconds) since the worker thread started.
+- `IdealProcessor` - the CPU the worker's partition is affinitized to.
+
+Dividing `CumulativeActiveTimeUs` by `CumulativeWallTimeUs` yields the worker's utilization; sampling the values over an interval yields the utilization for that interval.
+
+- **Type:** `QUIC_WORKER_STATISTICS_LIST`
+- **Get-only**
+- **Variable-length:** Use a double-call pattern to determine the required buffer size.
+
+Iterate the entries using the `WorkerStatsSize` stride, more fields may be added to `QUIC_WORKER_STATISTICS` in the future.
+
+**Sample usage:**
+```c
+uint32_t BufferLength = 0;
+QUIC_STATUS Status =
+    MsQuic->GetParam(NULL, QUIC_PARAM_GLOBAL_WORKER_STATISTICS, &BufferLength, NULL);
+
+if (Status != QUIC_STATUS_BUFFER_TOO_SMALL) {
+    // Unexpected, handle errors
+}
+
+QUIC_WORKER_STATISTICS_LIST* List =
+    (QUIC_WORKER_STATISTICS_LIST*)malloc(BufferLength);
+
+Status = MsQuic->GetParam(NULL, QUIC_PARAM_GLOBAL_WORKER_STATISTICS, &BufferLength, List);
+if (!QUIC_SUCCEEDED(Status)) {
+    // Handle errors
+}
+uint8_t* Entry = (uint8_t*)List + sizeof(QUIC_WORKER_STATISTICS_LIST);
+for (uint32_t i = 0; i < List->WorkerCount; i++) {
+    QUIC_WORKER_STATISTICS* WorkerStats = (QUIC_WORKER_STATISTICS*)Entry;
+    // Use Worker->CumulativeActiveTimeUs, etc.
+    Entry += List->WorkerStatsSize;
+}
+```
+
+See also: [Settings.md](../Settings.md#global-parameters)
+
 # See Also
 
 [Settings](../Settings.md#api-object-parameters)<br>

src/core/library.c

@@ -1539,6 +1539,83 @@ QuicLibrarySetGlobalParam(
     return Status;
 }
 
+//
+// Fills the caller's buffer with the per-worker statistics from the worker pool.
+//
+_IRQL_requires_max_(PASSIVE_LEVEL)
+QUIC_STATUS
+QuicLibraryGetGlobalWorkerStatistics(
+    _Inout_ uint32_t* BufferLength,
+    _Out_writes_bytes_opt_(*BufferLength)
+        void* Buffer
+    )
+{
+#ifdef _KERNEL_MODE
+    //
+    // Worker statistics are not supported in kernel mode, where the worker
+    // threads are not owned by the platform worker pool.
+    //
+    UNREFERENCED_PARAMETER(BufferLength);
+    UNREFERENCED_PARAMETER(Buffer);
+    return QUIC_STATUS_NOT_SUPPORTED;
+#else
+    QUIC_STATUS Status;
+
+    //
+    // Ensure the worker pool is not swapped / deleted concurrently
+    // (e.g. by MsQuicExecutionDelete).
+    //
+    CxPlatLockAcquire(&MsQuicLib.Lock);
+    CXPLAT_WORKER_POOL* WorkerPool = MsQuicLib.WorkerPool;
+    if (WorkerPool == NULL || !CxPlatWorkerPoolAddRef(WorkerPool, CXPLAT_WORKER_POOL_REF_STATS)) {
+        CxPlatLockRelease(&MsQuicLib.Lock);
+        return QUIC_STATUS_INVALID_STATE;
+    }
+    CxPlatLockRelease(&MsQuicLib.Lock);
+
+    uint32_t WorkerCount = CxPlatWorkerPoolGetCount(WorkerPool);
+    uint32_t RequiredSize =
+        sizeof(QUIC_WORKER_STATISTICS_LIST) +
+        WorkerCount * sizeof(QUIC_WORKER_STATISTICS);
+
+    if (*BufferLength < RequiredSize) {
+        *BufferLength = RequiredSize;
+        Status = QUIC_STATUS_BUFFER_TOO_SMALL;
+        goto Release;
+    }
+
+    if (Buffer == NULL) {
+        Status = QUIC_STATUS_INVALID_PARAMETER;
+        goto Release;
+    }
+
+    QUIC_WORKER_STATISTICS_LIST* List = (QUIC_WORKER_STATISTICS_LIST*)Buffer;
+    List->WorkerCount = WorkerCount;
+    List->WorkerStatsSize = sizeof(QUIC_WORKER_STATISTICS);
+
+    QUIC_WORKER_STATISTICS* Stats =
+        (QUIC_WORKER_STATISTICS*)((uint8_t*)Buffer + sizeof(QUIC_WORKER_STATISTICS_LIST));
+
+    const uint64_t Now = CxPlatTimeUs64();
+    for (uint32_t i = 0; i < WorkerCount; i++) {
+        CXPLAT_WORKER_STATISTICS WorkerStats = {0};
+        CxPlatWorkerPoolGetStatistics(WorkerPool, i, Now, &WorkerStats);
+        Stats[i].IdealProcessor = WorkerStats.IdealProcessor;
+        Stats[i].CumulativeActiveTimeUs = WorkerStats.CumulativeActiveTimeUs;
+        Stats[i].CumulativeWallTimeUs = WorkerStats.CumulativeWallTimeUs;
+    }
+
+    *BufferLength = RequiredSize;
+    Status = QUIC_STATUS_SUCCESS;
+
+Release:
+
+    CxPlatWorkerPoolRelease(WorkerPool, CXPLAT_WORKER_POOL_REF_STATS);
+
+    return Status;
+#endif // _KERNEL_MODE
+}
+
 _IRQL_requires_max_(PASSIVE_LEVEL)
 _Success_(return == QUIC_STATUS_SUCCESS)
 QUIC_STATUS
@@ -1914,6 +1991,10 @@ QuicLibraryGetGlobalParam(
         break;
     }
 
+    case QUIC_PARAM_GLOBAL_WORKER_STATISTICS:
+        Status = QuicLibraryGetGlobalWorkerStatistics(BufferLength, Buffer);
+        break;
+
     default:
         Status = QUIC_STATUS_INVALID_PARAMETER;
         break;
@@ -2916,14 +2997,24 @@ MsQuicExecutionCreate(
         //
         // Clean up any previous worker pool and create a new one.
         //
-        CxPlatWorkerPoolDelete(MsQuicLib.WorkerPool, CXPLAT_WORKER_POOL_REF_EXTERNAL);
-        MsQuicLib.WorkerPool =
+        CxPlatLockAcquire(&MsQuicLib.Lock);
+        CXPLAT_WORKER_POOL* OldWorkerPool = MsQuicLib.WorkerPool;
+        MsQuicLib.WorkerPool = NULL;
+        CxPlatLockRelease(&MsQuicLib.Lock);
+
+        CxPlatWorkerPoolDelete(OldWorkerPool, CXPLAT_WORKER_POOL_REF_EXTERNAL);
+
+        CXPLAT_WORKER_POOL* NewWorkerPool =
             CxPlatWorkerPoolCreateExternal(Count, Configs, Executions);
-        if (MsQuicLib.WorkerPool == NULL) {
+
+        if (NewWorkerPool == NULL) {
             Status = QUIC_STATUS_OUT_OF_MEMORY;
+        } else {
+            CxPlatLockAcquire(&MsQuicLib.Lock);
+            MsQuicLib.WorkerPool = NewWorkerPool;
+            MsQuicLib.CustomExecutions = TRUE;
+            CxPlatLockRelease(&MsQuicLib.Lock);
         }
-
-        MsQuicLib.CustomExecutions = TRUE;
     }
 
     QuicTraceEvent(
@@ -2951,9 +3042,13 @@ MsQuicExecutionDelete(
     UNREFERENCED_PARAMETER(Count);
     UNREFERENCED_PARAMETER(Executions);
 
-    CxPlatWorkerPoolDelete(MsQuicLib.WorkerPool, CXPLAT_WORKER_POOL_REF_EXTERNAL);
+    CxPlatLockAcquire(&MsQuicLib.Lock);
+    CXPLAT_WORKER_POOL* WorkerPool = MsQuicLib.WorkerPool;
     MsQuicLib.WorkerPool = NULL;
     MsQuicLib.CustomExecutions = FALSE;
+    CxPlatLockRelease(&MsQuicLib.Lock);
+
+    CxPlatWorkerPoolDelete(WorkerPool, CXPLAT_WORKER_POOL_REF_EXTERNAL);
 
     QuicTraceEvent(
         ApiExit,

src/inc/msquic.h

@@ -370,6 +370,27 @@ typedef struct QUIC_XDP_MAP_CONFIG {
 
 #endif // QUIC_API_ENABLE_PREVIEW_FEATURES
 
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+
+//
+// Per-worker statistics.
+//
+typedef struct QUIC_WORKER_STATISTICS {
+    uint64_t CumulativeActiveTimeUs;     // total time the worker spent active (not idle)
+    uint64_t CumulativeWallTimeUs;       // wall time since the worker thread started
+    uint16_t IdealProcessor;             // CPU the partition is affinitized to
+
+    // -- append-only beyond this point --
+} QUIC_WORKER_STATISTICS;
+
+typedef struct QUIC_WORKER_STATISTICS_LIST {
+    uint32_t WorkerCount;        // number of QUIC_WORKER_STATISTICS entries that follow
+    uint32_t WorkerStatsSize;    // sizeof(QUIC_WORKER_STATISTICS) as written by this msquic
+    // QUIC_WORKER_STATISTICS Workers[WorkerCount];   // stride == WorkerStatsSize
+} QUIC_WORKER_STATISTICS_LIST;
+
+#endif // QUIC_API_ENABLE_PREVIEW_FEATURES
+
 typedef struct QUIC_REGISTRATION_CONFIG { // All fields may be NULL/zero.
     const char* AppName;
     QUIC_EXECUTION_PROFILE ExecutionProfile;
@@ -1008,6 +1029,7 @@ void
 #define QUIC_PARAM_GLOBAL_STATELESS_RETRY_CONFIG        0x0100000D  // QUIC_STATELESS_RETRY_CONFIG
 #ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
 #define QUIC_PARAM_GLOBAL_XDP_MAP_CONFIG                0x0100000E  // QUIC_XDP_MAP_CONFIG[]
+#define QUIC_PARAM_GLOBAL_WORKER_STATISTICS             0x0100000F  // QUIC_WORKER_STATISTICS_LIST
 #endif
 
 //

src/inc/quic_platform.h

@@ -510,6 +510,7 @@ typedef enum CXPLAT_WORKER_POOL_REF {
     CXPLAT_WORKER_POOL_REF_RAW,
     CXPLAT_WORKER_POOL_REF_WINSOCK,
     CXPLAT_WORKER_POOL_REF_TOOL,
+    CXPLAT_WORKER_POOL_REF_STATS,       // Transient ref for querying worker statistics
 
     CXPLAT_WORKER_POOL_REF_COUNT
 
@@ -576,6 +577,23 @@ CxPlatWorkerPoolWorkerPoll(
     _In_ QUIC_EXECUTION* Execution
     );
 
+typedef struct CXPLAT_WORKER_STATISTICS {
+    uint64_t CumulativeActiveTimeUs;    // Time the worker spent active (not idle).
+    uint64_t CumulativeWallTimeUs;      // Wall time since the worker thread started.
+    uint16_t IdealProcessor;            // CPU the worker is affinitized to.
+} CXPLAT_WORKER_STATISTICS;
+
+//
+// Gets the statistics for a worker in the pool
+//
+void
+CxPlatWorkerPoolGetStatistics(
+    _In_ CXPLAT_WORKER_POOL* WorkerPool,
+    _In_ uint32_t Index,
+    _In_ uint64_t TimeNow,
+    _Out_ CXPLAT_WORKER_STATISTICS* Stats
+    );
+
 //
 // Supports more dynamic operations, but must be submitted to the platform worker
 // to manage.