Trace shared-socket QUIC routing

Author: richard-ramos

generate_lsquic_ffi.nim

@@ -7,21 +7,29 @@ from os import parentDir, `/`
 
 import boringssl
 
-proc dropGeneratedCEnumsImpl(opirOutput: JsonNode): JsonNode =
+const preludeTypes = ["struct_lsquic_cid", "lsquic_cid_t"]
+
+proc dropPreludeTypesAndGeneratedCEnumsImpl(opirOutput: JsonNode): JsonNode =
   var resp = newJArray()
   for node in opirOutput:
     # enums are generated manually to avoid issue described in
     # https://github.com/PMunch/futhark/issues/152
     if node{"kind"}.getStr("") == "enum":
       continue
+
+    # Futhark incorrectly maps the struct alignment on lsquic_cid_t to field alignment,
+    # so the prelude supplies the correct native layout instead.
+    if node{"name"}.getStr("") in preludeTypes:
+      continue
+
     resp.add node
   resp
 
 importc:
   outputPath currentSourcePath.parentDir / "tmp_lsquic_ffi.nim"
   path currentSourcePath.parentDir / "libs/lsquic/include"
   addopircallback proc(opirOutput: JsonNode): JsonNode {.closure.} =
-    dropGeneratedCEnumsImpl(opirOutput)
+    dropPreludeTypesAndGeneratedCEnumsImpl(opirOutput)
   rename FILE, CFile # Rename `FILE` that STB uses to `CFile` which is the Nim equivalent
   rename struct_sockaddr, SockAddr # Rename `struct_sockaddr` for chronos SockAddr
   "lsquic.h"

lsquic/context/client.nim

@@ -107,9 +107,9 @@ method dial*(
   if conn.isNil:
     GC_unref(quicClientConn)
     return err("could not dial: " & $remote)
-  ctx.trackConnectionCid(conn)
 
   quicClientConn.lsquicConn = conn
+  ctx.trackConnectionCid(conn)
   ctx.processWhenReady()
 
   ok(quicClientConn)
@@ -120,9 +120,9 @@ const Adaptive = 3
 proc new*(T: typedesc[ClientContext], tlsConfig: TLSConfig): Result[T, string] =
   var ctx = ClientContext()
   ctx.tlsConfig = tlsConfig
-  ctx.initCidTracking()
   ctx.running = true
   ctx.setupSSLContext()
+  ctx.initCidTracking()
 
   lsquic_engine_init_settings(addr ctx.settings, 0)
   ctx.settings.es_versions = 1.cuint shl LSQVER_I001.cuint #IETF QUIC v1

lsquic/context/context.nim

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 # Copyright (c) Status Research & Development GmbH 
 
-import std/[deques, hashes, sets]
+import std/[deques, hashes, sets, strutils]
 import boringssl
 import chronos
 import chronos/osdefs
@@ -17,14 +17,7 @@ type
     len*: uint8
     bytes*: array[MAX_CID_LEN, uint8]
 
-  # The generated FFI type currently aligns fields, not just the C struct.
-  # Read CIDs through the native wire layout until the binding is regenerated.
-  RawLsquicCid = object
-    buf: array[MAX_CID_LEN, uint8]
-    len: uint8
-    padding: array[3, uint8]
-
-  RawLsquicCidArray = UncheckedArray[RawLsquicCid]
+  LsquicCidArray = UncheckedArray[lsquic_cid_t]
 
   QuicContext* = ref object of RootObj
     settings*: struct_lsquic_engine_settings
@@ -40,16 +33,25 @@ type
     ownedCids*: HashSet[CidKey]
 
 static:
-  doAssert sizeof(RawLsquicCid) == 24
-  doAssert offsetOf(RawLsquicCid, len) == 20
+  doAssert sizeof(lsquic_cid_t) == 24
+  doAssert offsetOf(lsquic_cid_t, len) == 20
 
 func hash*(cid: CidKey): Hash =
   var h = hash(cid.len)
   for i in 0 ..< cid.len.int:
     h = h !& hash(cid.bytes[i])
   !$h
 
-func toCidKey(cid: RawLsquicCid, key: var CidKey): bool =
+func shortLog*(cid: CidKey): string =
+  var ret = $cid.len & ":"
+  for i in 0 ..< min(cid.len.int, 8):
+    ret.add(toHex(cid.bytes[i], 2))
+  ret
+
+chronicles.formatIt(CidKey):
+  shortLog(it)
+
+func toCidKey(cid: lsquic_cid_t, key: var CidKey): bool =
   if cid.len == 0 or cid.len.int > MAX_CID_LEN:
     return false
 
@@ -68,11 +70,12 @@ proc addCids*(
   if quicCtx.isNil or cids.isNil:
     return
 
-  let cidsArr = cast[ptr RawLsquicCidArray](cids)
+  let cidsArr = cast[ptr LsquicCidArray](cids)
   for i in 0 ..< nCids.int:
     var key: CidKey
     if toCidKey(cidsArr[i], key):
       quicCtx.ownedCids.incl(key)
+      trace "Registered CID", cid = key, cidCount = quicCtx.ownedCids.len
 
 proc removeCids*(
     ctx: pointer, peerCtxs: ptr pointer, cids: ptr lsquic_cid_t, nCids: cuint
@@ -81,15 +84,14 @@ proc removeCids*(
   if quicCtx.isNil or cids.isNil:
     return
 
-  let cidsArr = cast[ptr RawLsquicCidArray](cids)
+  let cidsArr = cast[ptr LsquicCidArray](cids)
   for i in 0 ..< nCids.int:
     var key: CidKey
     if toCidKey(cidsArr[i], key):
       quicCtx.ownedCids.excl(key)
+      trace "Removed CID", cid = key, cidCount = quicCtx.ownedCids.len
 
-proc trackConnectionCid*(
-    ctx: QuicContext, conn: ptr lsquic_conn_t
-) {.raises: [].} =
+proc trackConnectionCid*(ctx: QuicContext, conn: ptr lsquic_conn_t) {.raises: [].} =
   if ctx.isNil or conn.isNil:
     return
 
@@ -98,8 +100,9 @@ proc trackConnectionCid*(
     return
 
   var key: CidKey
-  if toCidKey(cast[ptr RawLsquicCid](cid)[], key):
+  if toCidKey(cid[], key):
     ctx.ownedCids.incl(key)
+    trace "Tracked connection CID", cid = key, cidCount = ctx.ownedCids.len
 
 proc ownsCid*(ctx: QuicContext, cid: CidKey): bool {.raises: [].} =
   not ctx.isNil and cid in ctx.ownedCids

lsquic/context/io.nim

@@ -3,6 +3,7 @@
 
 import chronos
 import chronos/osdefs
+import chronicles
 import ./context
 import ../[lsquic_ffi, datagram]
 import ../helpers/[openarray, sequninit, transportaddr]
@@ -144,6 +145,7 @@ proc sendPacketsOut*(
 
       let res = sendmsg(SocketHandle(quicCtx.fd), msg.addr, 0)
       if res < 0:
+        trace "sendmsg failed", sent, nspecs
         break
 
     sent.inc

lsquic/context/server.nim

@@ -57,9 +57,9 @@ proc new*(T: typedesc[ServerContext], tlsConfig: TLSConfig): Result[T, string] =
   var ctx = ServerContext()
   ctx.tlsConfig = tlsConfig
   ctx.running = true
-  ctx.initCidTracking()
   ctx.incoming = newAsyncQueue[QuicConnection]()
   ctx.setupSSLContext()
+  ctx.initCidTracking()
 
   lsquic_engine_init_settings(addr ctx.settings, LSENG_SERVER)
   ctx.settings.es_versions = 1.cuint shl LSQVER_I001.cuint #IETF QUIC v1

lsquic/endpoint.nim

@@ -39,10 +39,12 @@ proc createClientContext(
   context.fd = fd
   context
 
-proc serverCidLen(endpoint: QuicEndpoint): cuint {.raises: [].} =
-  if not endpoint.serverContext.isNil and endpoint.serverContext.settings.es_scid_len != 0:
+proc scidLen(endpoint: QuicEndpoint): cuint {.raises: [].} =
+  if not endpoint.serverContext.isNil and
+      endpoint.serverContext.settings.es_scid_len != 0:
     return endpoint.serverContext.settings.es_scid_len
-  if not endpoint.clientContext.isNil and endpoint.clientContext.settings.es_scid_len != 0:
+  if not endpoint.clientContext.isNil and
+      endpoint.clientContext.settings.es_scid_len != 0:
     return endpoint.clientContext.settings.es_scid_len
   LSQUIC_DF_SCID_LEN.cuint
 
@@ -54,10 +56,7 @@ proc packetDcid(
 
   var cidLen: uint8
   let offset = lsquic_dcid_from_packet(
-    unsafeAddr packet[0],
-    packet.len.csize_t,
-    endpoint.serverCidLen(),
-    addr cidLen,
+    unsafeAddr packet[0], packet.len.csize_t, endpoint.scidLen(), addr cidLen
   )
   if offset < 0:
     return false
@@ -80,15 +79,23 @@ proc receiveDatagram(
   var cid: CidKey
   if endpoint.packetDcid(data, cid):
     if not endpoint.clientContext.isNil and endpoint.clientContext.ownsCid(cid):
+      trace "Routing datagram to client context", cid
       endpoint.clientContext.receive(Datagram(data: data), local, remote)
       return
 
     if not endpoint.serverContext.isNil and endpoint.serverContext.ownsCid(cid):
+      trace "Routing datagram to server context", cid
       endpoint.serverContext.receive(Datagram(data: data), local, remote)
       return
 
   # Unknown CIDs can be new handshakes; each engine gets a chance to claim them.
   # Endpoints can have both contexts; each engine needs to see the packet.
+  trace "Routing datagram with unknown CID",
+    bytes = data.len,
+    hasClientContext = not endpoint.clientContext.isNil,
+    hasServerContext = not endpoint.serverContext.isNil,
+    local,
+    remote
   if not endpoint.clientContext.isNil:
     endpoint.clientContext.receive(Datagram(data: data), local, remote)
   if not endpoint.serverContext.isNil:

lsquic/lsquic_ffi.nim

@@ -11,7 +11,9 @@ import chronos/osdefs
 import zlib
 import boringssl
 
-type ptrdiff_t* {.importc: "ptrdiff_t", header: "<stddef.h>".} = int
+type
+  ptrdiff_t* {.importc: "ptrdiff_t", header: "<stddef.h>".} = int
+  uint_fast8_t* {.importc: "uint_fast8_t", header: "<stdint.h>".} = uint8
 
 # enums are generated manually to avoid issue described in
 # https://github.com/PMunch/futhark/issues/152
@@ -43,6 +45,9 @@ borrowCEnumOps(enum_lsquic_conn_param)
 borrowCEnumOps(enum_LSQUIC_CONN_STATUS)
 
 const
+  MAX_CID_LEN* = 20
+  GQUIC_CID_LEN* = 8
+
   LSQVER_043* = enum_lsquic_version(0)
   LSQVER_046* = enum_lsquic_version(1)
   LSQVER_050* = enum_lsquic_version(2)
@@ -107,6 +112,16 @@ when defined(windows):
 {.passc: "-I" & lshpack.}
 {.passc: "-I" & xxhash.}
 
+type
+  struct_lsquic_cid* {.
+    importc: "struct lsquic_cid", header: "lsquic_types.h", bycopy, completeStruct
+  .} = object
+    buf* {.importc: "buf".}: array[MAX_CID_LEN, uint8]
+    len* {.importc: "len".}: uint_fast8_t
+    padding: array[3, uint8]
+
+  lsquic_cid_t* = struct_lsquic_cid
+
 const HAVE_BORINGSSL = "-DHAVE_BORINGSSL"
 const XXH_HEADER_NAME = "-DXXH_HEADER_NAME=\"<lsquic_xxhash.h>\""
 
@@ -357,14 +372,7 @@ else:
       "Declaration of " & "LSQUIC_DF_CFCW_CLIENT" & " already exists, not redeclaring"
     )
 type
-  struct_lsquic_cid_570425834 {.pure, inheritable, bycopy.} = object
-    buf* {.align(8'i64).}: array[20'i64, uint8]
-      ## Generated based on /home/r/vacp2p/nim-lsquic/libs/lsquic/include/lsquic_types.h:27:28
-    len* {.align(8'i64).}: uint_fast8_t_570425837
-
   uint_fast8_t_570425836 = uint8 ## Generated based on /usr/include/stdint.h:60:24
-  lsquic_cid_t_570425838 = struct_lsquic_cid_570425835
-    ## Generated based on /home/r/vacp2p/nim-lsquic/libs/lsquic/include/lsquic_types.h:32:3
   lsquic_stream_id_t_570425840 = uint64
     ## Generated based on /home/r/vacp2p/nim-lsquic/libs/lsquic/include/lsquic_types.h:40:18
   lsquic_engine_t_570425842 = struct_lsquic_engine
@@ -553,9 +561,8 @@ type
       proc(a0: pointer, a1: pointer, a2: pointer, a3: cschar): void {.cdecl.}
     pmi_return*: proc(a0: pointer, a1: pointer, a2: pointer, a3: cschar): void {.cdecl.}
 
-  lsquic_cids_update_f_570425876 = proc(
-    a0: pointer, a1: ptr pointer, a2: ptr lsquic_cid_t_570425839, a3: cuint
-  ): void {.cdecl.}
+  lsquic_cids_update_f_570425876 =
+    proc(a0: pointer, a1: ptr pointer, a2: ptr lsquic_cid_t, a3: cuint): void {.cdecl.}
     ## Generated based on /home/r/vacp2p/nim-lsquic/libs/lsquic/include/lsquic.h:1307:16
   struct_lsquic_hset_if_570425878 {.pure, inheritable, bycopy.} = object
     hsi_create_header_set*:
@@ -733,17 +740,6 @@ type
     else:
       struct_lsquic_conn_info_570425895
   )
-  struct_lsquic_cid_570425835 = (
-    when declared(struct_lsquic_cid):
-      when ownSizeof(struct_lsquic_cid) != ownSizeof(struct_lsquic_cid_570425834):
-        static:
-          warning(
-            "Declaration of " & "struct_lsquic_cid" & " exists but with different size"
-          )
-      struct_lsquic_cid
-    else:
-      struct_lsquic_cid_570425834
-  )
   lsquic_stream_id_t_570425841 = (
     when declared(lsquic_stream_id_t):
       when ownSizeof(lsquic_stream_id_t) != ownSizeof(lsquic_stream_id_t_570425840):
@@ -910,17 +906,6 @@ type
     else:
       lsquic_stream_ctx_t_570425850
   )
-  lsquic_cid_t_570425839 = (
-    when declared(lsquic_cid_t):
-      when ownSizeof(lsquic_cid_t) != ownSizeof(lsquic_cid_t_570425838):
-        static:
-          warning(
-            "Declaration of " & "lsquic_cid_t" & " exists but with different size"
-          )
-      lsquic_cid_t
-    else:
-      lsquic_cid_t_570425838
-  )
   uint_fast8_t_570425837 = (
     when declared(uint_fast8_t):
       when ownSizeof(uint_fast8_t) != ownSizeof(uint_fast8_t_570425836):
@@ -1046,11 +1031,6 @@ else:
     hint(
       "Declaration of " & "struct_lsquic_conn_info" & " already exists, not redeclaring"
     )
-when not declared(struct_lsquic_cid):
-  type struct_lsquic_cid* = struct_lsquic_cid_570425834
-else:
-  static:
-    hint("Declaration of " & "struct_lsquic_cid" & " already exists, not redeclaring")
 when not declared(lsquic_stream_id_t):
   type lsquic_stream_id_t* = lsquic_stream_id_t_570425840
 else:
@@ -1138,11 +1118,6 @@ when not declared(lsquic_stream_ctx_t):
 else:
   static:
     hint("Declaration of " & "lsquic_stream_ctx_t" & " already exists, not redeclaring")
-when not declared(lsquic_cid_t):
-  type lsquic_cid_t* = lsquic_cid_t_570425838
-else:
-  static:
-    hint("Declaration of " & "lsquic_cid_t" & " already exists, not redeclaring")
 when not declared(uint_fast8_t):
   type uint_fast8_t* = uint_fast8_t_570425836
 else:
@@ -2670,7 +2645,7 @@ else:
 when not declared(lsquic_conn_id):
   proc lsquic_conn_id*(
     c: ptr lsquic_conn_t_570425845
-  ): ptr lsquic_cid_t_570425839 {.cdecl, importc: "lsquic_conn_id".}
+  ): ptr lsquic_cid_t {.cdecl, importc: "lsquic_conn_id".}
 
 else:
   static:
@@ -2968,7 +2943,7 @@ else:
     )
 when not declared(lsquic_cid_from_packet):
   proc lsquic_cid_from_packet*(
-    a0: ptr uint8, bufsz: csize_t, cid: ptr lsquic_cid_t_570425839
+    a0: ptr uint8, bufsz: csize_t, cid: ptr lsquic_cid_t
   ): cint {.cdecl, importc: "lsquic_cid_from_packet".}
 
 else: