update docs based on feedback; improve log level

Author: ProjectsByJackHe

docs/CIBIR.md

@@ -2,9 +2,9 @@
 
 ## What is it
 
-See [XDP](./XDP.md) first to understand the context.
+See the [draft IETF](https://datatracker.ietf.org/doc/html/draft-banks-quic-cibir) for CIBIR.
 
-When CIBIR is used, rather than programming XDP to filter and demux packets based on on address and port number,
+When CIBIR is used, rather than programming [XDP](./XDP.md) to filter and demux packets based on on address and port number,
 XDP with CIBIR will instead filter and de-mux packets based on address, port number, and QUIC connection ID.
 
 What CIBIR allows for is 2 or more separate server processes to share a single
@@ -17,10 +17,19 @@ port on the same machine, as long as their CIBIR ID is different.
 - The responsbility of book-keeping shared ports and ensuring robust protection for those shared ports is delegated to the application.
 
 
-## Port protection recommendation for shared ports
+## Port protection recommendations for shared ports
+
+### Option 1: Persistent port reservations (Recommended)
 
 MsQuic strongly recommends applications leverage the Windows [persistent port reservations API](https://learn.microsoft.com/en-us/windows/win32/api/iphlpapi/nf-iphlpapi-createpersistentudpportreservation) to secure shared CIBIR ports prior to serving multi-process CIBIR traffic on a shared port.
 - One time setup by a system admin to create the persistent reservation.
     > A good option for book-keeping persistent port reservations is via registry keys.
-- Persistent port reservations survive reboots, allowing for robust portection in the event of crashes.
-- Having a persistent reservation makes sure critical ports are taken out of the ephemeral port pool, so an unsuspecting application process won't get accidently assigned an ephemeral port that collides with a CIBIR port.
+- Persistent port reservations survive reboots, allowing for robust protection in the event of crashes.
+- Having a persistent reservation makes sure CIBIR ports are taken out of the ephemeral port pool and forbids sockets from binding to it unless it is associated with a persistent reservation token, which can only happen in an elevated process.
+    - This way, an unsuspecting application process won't get accidently assigned an ephemeral port that collides with a CIBIR port.
+
+### Option 2: WFP ALE (Application Layer Enforcement) filters
+
+As an alternative, applications can use the [Windows Filtering Platform (WFP)](https://learn.microsoft.com/en-us/windows/win32/fwp/windows-filtering-platform-start-page) to create ALE filters that block unauthorized bind attempts to CIBIR ports.
+
+ALE filters operate at the [bind and connect authorization layers](https://learn.microsoft.com/en-us/windows/win32/fwp/ale-layers) (`FWPM_LAYER_ALE_AUTH_RECV_ACCEPT_V4/V6`, `FWPM_LAYER_ALE_RESOURCE_ASSIGNMENT_V4/V6`). A filter can be configured to block any process from binding to a specific UDP port unless it matches an allowed application path or security descriptor.

docs/XDP.md

@@ -1,7 +1,7 @@
 # MsQuic over XDP
 
-To avoid confusion, "XDP" refers to [XDP-for-windows](https://github.com/microsoft/xdp-for-windows). While Linux XDP has been experimented
-upon in the past and shown some promise for running MsQuic, it is NOT a stable datapath actively being maintained today.
+To avoid confusion, "XDP" refers to [XDP-for-windows](https://github.com/microsoft/xdp-for-windows).
+MsQuic does not support Linux XDP as a datapath.
 
 ## What is XDP
 

src/generated/linux/datapath_xplat.c.clog.h

@@ -22,6 +22,10 @@
 #define _clog_MACRO_QuicTraceLogVerbose  1
 #define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__)))
 #endif
+#ifndef _clog_MACRO_QuicTraceLogError
+#define _clog_MACRO_QuicTraceLogError  1
+#define QuicTraceLogError(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__)))
+#endif
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -41,22 +45,6 @@ tracepoint(CLOG_DATAPATH_XPLAT_C, WarnFallbackToOsSockets );\
 
 
 
-/*----------------------------------------------------------
-// Decoder Ring for ErrNoXdpForRaw
-// [sock] Error: app requested QTIP but XDP not enabled/available/initialized.
-// QuicTraceLogWarning(
-                ErrNoXdpForRaw,
-                "[sock] Error: app requested QTIP but XDP not enabled/available/initialized.");
-----------------------------------------------------------*/
-#ifndef _clog_2_ARGS_TRACE_ErrNoXdpForRaw
-#define _clog_2_ARGS_TRACE_ErrNoXdpForRaw(uniqueId, encoded_arg_string)\
-tracepoint(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForRaw );\
-
-#endif
-
-
-
-
 /*----------------------------------------------------------
 // Decoder Ring for WarnNoXdpForCibirSockets
 // [sock] Warning: app requested CIBIR but XDP not enabled/available/initialized. \
@@ -126,6 +114,22 @@ tracepoint(CLOG_DATAPATH_XPLAT_C, RawSockCreateFail , arg2);\
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for ErrNoXdpForQtip
+// [sock] Error: app requested QTIP but XDP not enabled/available/initialized.
+// QuicTraceLogError(
+                ErrNoXdpForQtip,
+                "[sock] Error: app requested QTIP but XDP not enabled/available/initialized.");
+----------------------------------------------------------*/
+#ifndef _clog_2_ARGS_TRACE_ErrNoXdpForQtip
+#define _clog_2_ARGS_TRACE_ErrNoXdpForQtip(uniqueId, encoded_arg_string)\
+tracepoint(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForQtip );\
+
+#endif
+
+
+
+
 #ifdef __cplusplus
 }
 #endif

src/generated/linux/datapath_xplat.c.clog.h.lttng.h

@@ -17,22 +17,6 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, WarnFallbackToOsSockets,
 
 
 
-/*----------------------------------------------------------
-// Decoder Ring for ErrNoXdpForRaw
-// [sock] Error: app requested QTIP but XDP not enabled/available/initialized.
-// QuicTraceLogWarning(
-                ErrNoXdpForRaw,
-                "[sock] Error: app requested QTIP but XDP not enabled/available/initialized.");
-----------------------------------------------------------*/
-TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForRaw,
-    TP_ARGS(
-), 
-    TP_FIELDS(
-    )
-)
-
-
-
 /*----------------------------------------------------------
 // Decoder Ring for WarnNoXdpForCibirSockets
 // [sock] Warning: app requested CIBIR but XDP not enabled/available/initialized. \
@@ -102,3 +86,19 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, RawSockCreateFail,
         ctf_integer(int, arg2, arg2)
     )
 )
+
+
+
+/*----------------------------------------------------------
+// Decoder Ring for ErrNoXdpForQtip
+// [sock] Error: app requested QTIP but XDP not enabled/available/initialized.
+// QuicTraceLogError(
+                ErrNoXdpForQtip,
+                "[sock] Error: app requested QTIP but XDP not enabled/available/initialized.");
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForQtip,
+    TP_ARGS(
+), 
+    TP_FIELDS(
+    )
+)

src/manifest/clog.sidecar

@@ -4222,6 +4222,13 @@
       ],
       "macroName": "QuicTraceLogVerbose"
     },
+    "ErrNoXdpForQtip": {
+      "ModuleProperites": {},
+      "TraceString": "[sock] Error: app requested QTIP but XDP not enabled/available/initialized.",
+      "UniqueId": "ErrNoXdpForQtip",
+      "splitArgs": [],
+      "macroName": "QuicTraceLogError"
+    },
     "ErrNoXdpForRaw": {
       "ModuleProperites": {},
       "TraceString": "[sock] Error: app requested QTIP but XDP not enabled/available/initialized.",
@@ -14742,6 +14749,11 @@
         "TraceID": "EpollSocketRelease",
         "EncodingString": "[data][%p] Socket Freed"
       },
+      {
+        "UniquenessHash": "b93c5e28-04ab-7e11-fe17-83c24bf864d3",
+        "TraceID": "ErrNoXdpForQtip",
+        "EncodingString": "[sock] Error: app requested QTIP but XDP not enabled/available/initialized."
+      },
       {
         "UniquenessHash": "c8f7a72c-21b5-b5b0-8978-883ca5df98ee",
         "TraceID": "ErrNoXdpForRaw",

src/platform/datapath_xplat.c

@@ -166,7 +166,7 @@ CxPlatSocketCreateUdp(
                 if (IsWildcardAddr && RequiresQtip) {
                     //
                     // This retry loop is purely for QTIP listener sockets that try to reserve both a UDP/TCP port,
-                    // which may run into a WSA_ADDR_INUSE for TCP if the UDP ephemeral port collides with something
+                    // which may run into a port collision for TCP if the UDP ephemeral port collides with something
                     // in the TCP pool. So just try it again.
                     //
                     CxPlatSocketDelete(*NewSocket);
@@ -191,8 +191,8 @@ CxPlatSocketCreateUdp(
                 goto Error;
             }
         } else if (RequiresQtip) {
-            QuicTraceLogWarning(
-                ErrNoXdpForRaw,
+            QuicTraceLogError(
+                ErrNoXdpForQtip,
                 "[sock] Error: app requested QTIP but XDP not enabled/available/initialized.");
             CxPlatSocketDelete(*NewSocket);
             Status = QUIC_STATUS_INVALID_STATE;

src/test/lib/HandshakeTest.cpp

@@ -3920,25 +3920,12 @@ QuicTestCibirExtension(
         // CIBIR + XDP requires an explicit local port. Reserve an ephemeral port
         // up front so the listener always binds to a known port.
         //
-        if (PortReservation.Port != 0) {
-            ServerLocalAddr.SetPort(PortReservation.Port);
-        }
-    } else {
-        //
-        // If no XDP, CIBIR just falls back to normal OS sockets.
-        // We can just rely upon the Listener created sockets. Passing in
-        // a local port of 0 is fine here.
-        //
-        PortReservation.Release();
+        ServerLocalAddr.SetPort(PortReservation.Port);
     }
 #endif
     MsQuicAutoAcceptListener Listener(Registration, ServerConfiguration, MsQuicConnection::NoOpCallback);
     if (Mode & 1) {
         TEST_QUIC_SUCCEEDED(Listener.SetCibirId(CibirId, CibirIdLength));
-    } else {
-#ifdef _WIN32
-        PortReservation.Release();
-#endif
     }
 
     TEST_QUIC_SUCCEEDED(Listener.Start("MsQuicTest", &ServerLocalAddr.SockAddr));
@@ -4491,15 +4478,14 @@ QuicTestConnectionPoolCreate(
     TEST_QUIC_SUCCEEDED(ClientConfiguration.GetInitStatus());
 
     QuicAddr ServerAddr(QuicAddrFamily);
-#ifdef _WIN32
-    QuicTestPortReservation PortReservation(QuicAddrFamily);
-#endif
+
     if (XdpSupported) {
         QuicAddrSetToDuoNic(&ServerAddr.SockAddr);
     }
 
 #ifdef _WIN32
-    if (PortReservation.Port != 0 && UseDuoNic && TestCibirSupport) {
+    QuicTestPortReservation PortReservation(QuicAddrFamily);
+    if (UseDuoNic && TestCibirSupport) {
         //
         // If Cibir+XDP mode is active, we can't pass in a 0 local port
         // hoping the stack will assign us an ephemeral port.

src/test/lib/TestHelpers.h

@@ -102,10 +102,6 @@ QuitTestIsFeatureSupported(uint32_t Feature) {
 // is destroyed). This minimizes the TOCTOU window between discovering a free
 // port and having the listener bind to it.
 //
-// In kernel mode, raw socket APIs are unavailable, so this is a no-op
-// (Port stays 0 and the listener falls back to OS-assigned ephemeral ports).
-//
-#ifdef _WIN32
 struct QuicTestPortReservation {
     uint16_t Port{0};
 
@@ -115,7 +111,6 @@ struct QuicTestPortReservation {
         _In_ QUIC_ADDRESS_FAMILY Family
         )
     {
-#ifndef _KERNEL_MODE
         int af = (Family == QUIC_ADDRESS_FAMILY_INET) ? AF_INET : AF_INET6;
 
         Sock = socket(af, SOCK_DGRAM, IPPROTO_UDP);
@@ -140,9 +135,7 @@ struct QuicTestPortReservation {
         if (Port == 0) {
             Release();
         }
-#else
         UNREFERENCED_PARAMETER(Family);
-#endif
     }
 
     ~QuicTestPortReservation() { Release(); }
@@ -156,17 +149,14 @@ struct QuicTestPortReservation {
     //
     void Release()
     {
-#ifndef _KERNEL_MODE
-        if (Sock != INVALID_SOCKET) { closesocket(Sock); Sock = INVALID_SOCKET; }
-#endif
+        if (Sock != INVALID_SOCKET) {
+            closesocket(Sock); Sock = INVALID_SOCKET;
+        }
     }
 
 private:
-#ifndef _KERNEL_MODE
     SOCKET Sock{INVALID_SOCKET};
-#endif
 };
-#endif
 
 #define OLD_SUPPORTED_VERSION       QUIC_VERSION_1_MS_H
 #define LATEST_SUPPORTED_VERSION    QUIC_VERSION_LATEST_H