Trace shared-socket QUIC routing

Author: richard-ramos

generate_lsquic_ffi.nim

@@ -7,21 +7,29 @@ from os import parentDir, `/`
 
 import boringssl
 
-proc dropGeneratedCEnumsImpl(opirOutput: JsonNode): JsonNode =
+const preludeTypes = ["struct_lsquic_cid", "lsquic_cid_t"]
+
+proc dropPreludeTypesAndGeneratedCEnumsImpl(opirOutput: JsonNode): JsonNode =
   var resp = newJArray()
   for node in opirOutput:
     # enums are generated manually to avoid issue described in
     # https://github.com/PMunch/futhark/issues/152
     if node{"kind"}.getStr("") == "enum":
       continue
+
+    # Futhark incorrectly maps the struct alignment on lsquic_cid_t to field alignment,
+    # so the prelude supplies the correct native layout instead.
+    if node{"name"}.getStr("") in preludeTypes:
+      continue
+
     resp.add node
   resp
 
 importc:
   outputPath currentSourcePath.parentDir / "tmp_lsquic_ffi.nim"
   path currentSourcePath.parentDir / "libs/lsquic/include"
   addopircallback proc(opirOutput: JsonNode): JsonNode {.closure.} =
-    dropGeneratedCEnumsImpl(opirOutput)
+    dropPreludeTypesAndGeneratedCEnumsImpl(opirOutput)
   rename FILE, CFile # Rename `FILE` that STB uses to `CFile` which is the Nim equivalent
   rename struct_sockaddr, SockAddr # Rename `struct_sockaddr` for chronos SockAddr
   "lsquic.h"

lsquic/context/client.nim

@@ -107,9 +107,9 @@ method dial*(
   if conn.isNil:
     GC_unref(quicClientConn)
     return err("could not dial: " & $remote)
-  ctx.trackConnectionCid(conn)
 
   quicClientConn.lsquicConn = conn
+  ctx.trackConnectionCid(conn)
   ctx.processWhenReady()
 
   ok(quicClientConn)
@@ -120,9 +120,9 @@ const Adaptive = 3
 proc new*(T: typedesc[ClientContext], tlsConfig: TLSConfig): Result[T, string] =
   var ctx = ClientContext()
   ctx.tlsConfig = tlsConfig
-  ctx.initCidTracking()
   ctx.running = true
   ctx.setupSSLContext()
+  ctx.initCidTracking()
 
   lsquic_engine_init_settings(addr ctx.settings, 0)
   ctx.settings.es_versions = 1.cuint shl LSQVER_I001.cuint #IETF QUIC v1

lsquic/context/context.nim

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 # Copyright (c) Status Research & Development GmbH 
 
-import std/[deques, hashes, sets]
+import std/[deques, hashes, sets, strutils]
 import boringssl
 import chronos
 import chronos/osdefs
@@ -17,14 +17,7 @@ type
     len*: uint8
     bytes*: array[MAX_CID_LEN, uint8]
 
-  # The generated FFI type currently aligns fields, not just the C struct.
-  # Read CIDs through the native wire layout until the binding is regenerated.
-  RawLsquicCid = object
-    buf: array[MAX_CID_LEN, uint8]
-    len: uint8
-    padding: array[3, uint8]
-
-  RawLsquicCidArray = UncheckedArray[RawLsquicCid]
+  LsquicCidArray = UncheckedArray[lsquic_cid_t]
 
   QuicContext* = ref object of RootObj
     settings*: struct_lsquic_engine_settings
@@ -39,17 +32,22 @@ type
     running*: bool
     ownedCids*: HashSet[CidKey]
 
-static:
-  doAssert sizeof(RawLsquicCid) == 24
-  doAssert offsetOf(RawLsquicCid, len) == 20
-
 func hash*(cid: CidKey): Hash =
   var h = hash(cid.len)
   for i in 0 ..< cid.len.int:
     h = h !& hash(cid.bytes[i])
   !$h
 
-func toCidKey(cid: RawLsquicCid, key: var CidKey): bool =
+func shortLog*(cid: CidKey): string =
+  var ret = $cid.len & ":"
+  for i in 0 ..< min(cid.len.int, 8):
+    ret.add(toHex(cid.bytes[i], 2))
+  ret
+
+chronicles.formatIt(CidKey):
+  shortLog(it)
+
+func toCidKey(cid: lsquic_cid_t, key: var CidKey): bool =
   if cid.len == 0 or cid.len.int > MAX_CID_LEN:
     return false
 
@@ -68,11 +66,12 @@ proc addCids*(
   if quicCtx.isNil or cids.isNil:
     return
 
-  let cidsArr = cast[ptr RawLsquicCidArray](cids)
+  let cidsArr = cast[ptr LsquicCidArray](cids)
   for i in 0 ..< nCids.int:
     var key: CidKey
     if toCidKey(cidsArr[i], key):
       quicCtx.ownedCids.incl(key)
+      trace "Registered CID", cid = key, cidCount = quicCtx.ownedCids.len
 
 proc removeCids*(
     ctx: pointer, peerCtxs: ptr pointer, cids: ptr lsquic_cid_t, nCids: cuint
@@ -81,15 +80,14 @@ proc removeCids*(
   if quicCtx.isNil or cids.isNil:
     return
 
-  let cidsArr = cast[ptr RawLsquicCidArray](cids)
+  let cidsArr = cast[ptr LsquicCidArray](cids)
   for i in 0 ..< nCids.int:
     var key: CidKey
     if toCidKey(cidsArr[i], key):
       quicCtx.ownedCids.excl(key)
+      trace "Removed CID", cid = key, cidCount = quicCtx.ownedCids.len
 
-proc trackConnectionCid*(
-    ctx: QuicContext, conn: ptr lsquic_conn_t
-) {.raises: [].} =
+proc trackConnectionCid*(ctx: QuicContext, conn: ptr lsquic_conn_t) {.raises: [].} =
   if ctx.isNil or conn.isNil:
     return
 
@@ -98,8 +96,9 @@ proc trackConnectionCid*(
     return
 
   var key: CidKey
-  if toCidKey(cast[ptr RawLsquicCid](cid)[], key):
+  if toCidKey(cid[], key):
     ctx.ownedCids.incl(key)
+    trace "Tracked connection CID", cid = key, cidCount = ctx.ownedCids.len
 
 proc ownsCid*(ctx: QuicContext, cid: CidKey): bool {.raises: [].} =
   not ctx.isNil and cid in ctx.ownedCids

lsquic/context/io.nim

@@ -3,6 +3,7 @@
 
 import chronos
 import chronos/osdefs
+import chronicles
 import ./context
 import ../[lsquic_ffi, datagram]
 import ../helpers/[openarray, sequninit, transportaddr]
@@ -144,6 +145,7 @@ proc sendPacketsOut*(
 
       let res = sendmsg(SocketHandle(quicCtx.fd), msg.addr, 0)
       if res < 0:
+        trace "sendmsg failed", sent, nspecs
         break
 
     sent.inc

lsquic/context/server.nim

@@ -57,9 +57,9 @@ proc new*(T: typedesc[ServerContext], tlsConfig: TLSConfig): Result[T, string] =
   var ctx = ServerContext()
   ctx.tlsConfig = tlsConfig
   ctx.running = true
-  ctx.initCidTracking()
   ctx.incoming = newAsyncQueue[QuicConnection]()
   ctx.setupSSLContext()
+  ctx.initCidTracking()
 
   lsquic_engine_init_settings(addr ctx.settings, LSENG_SERVER)
   ctx.settings.es_versions = 1.cuint shl LSQVER_I001.cuint #IETF QUIC v1

lsquic/endpoint.nim

@@ -39,10 +39,12 @@ proc createClientContext(
   context.fd = fd
   context
 
-proc serverCidLen(endpoint: QuicEndpoint): cuint {.raises: [].} =
-  if not endpoint.serverContext.isNil and endpoint.serverContext.settings.es_scid_len != 0:
+proc scidLen(endpoint: QuicEndpoint): cuint {.raises: [].} =
+  if not endpoint.serverContext.isNil and
+      endpoint.serverContext.settings.es_scid_len != 0:
     return endpoint.serverContext.settings.es_scid_len
-  if not endpoint.clientContext.isNil and endpoint.clientContext.settings.es_scid_len != 0:
+  if not endpoint.clientContext.isNil and
+      endpoint.clientContext.settings.es_scid_len != 0:
     return endpoint.clientContext.settings.es_scid_len
   LSQUIC_DF_SCID_LEN.cuint
 
@@ -54,10 +56,7 @@ proc packetDcid(
 
   var cidLen: uint8
   let offset = lsquic_dcid_from_packet(
-    unsafeAddr packet[0],
-    packet.len.csize_t,
-    endpoint.serverCidLen(),
-    addr cidLen,
+    unsafeAddr packet[0], packet.len.csize_t, endpoint.scidLen(), addr cidLen
   )
   if offset < 0:
     return false
@@ -71,28 +70,48 @@ proc packetDcid(
     cid.bytes[i] = packet[start + i]
   true
 
+func isIetfInitial(packet: seq[byte]): bool {.raises: [].} =
+  if packet.len == 0:
+    return false
+  (packet[0] and 0xC0'u8) == 0xC0'u8 and (packet[0] and 0x30'u8) == 0
+
 proc receiveDatagram(
     endpoint: QuicEndpoint, data: seq[byte], local, remote: TransportAddress
 ) {.raises: [].} =
   if endpoint.isNil or endpoint.stopped:
     return
 
+  let
+    hasClientContext = not endpoint.clientContext.isNil
+    hasServerContext = not endpoint.serverContext.isNil
+
   var cid: CidKey
   if endpoint.packetDcid(data, cid):
-    if not endpoint.clientContext.isNil and endpoint.clientContext.ownsCid(cid):
+    if hasClientContext and endpoint.clientContext.ownsCid(cid):
+      trace "Routing datagram to client context", cid
       endpoint.clientContext.receive(Datagram(data: data), local, remote)
       return
 
-    if not endpoint.serverContext.isNil and endpoint.serverContext.ownsCid(cid):
+    if hasServerContext and endpoint.serverContext.ownsCid(cid):
+      trace "Routing datagram to server context", cid
       endpoint.serverContext.receive(Datagram(data: data), local, remote)
       return
 
-  # Unknown CIDs can be new handshakes; each engine gets a chance to claim them.
-  # Endpoints can have both contexts; each engine needs to see the packet.
-  if not endpoint.clientContext.isNil:
+  if hasClientContext and not hasServerContext:
     endpoint.clientContext.receive(Datagram(data: data), local, remote)
-  if not endpoint.serverContext.isNil:
+    return
+
+  if hasServerContext and not hasClientContext:
+    endpoint.serverContext.receive(Datagram(data: data), local, remote)
+    return
+
+  if hasServerContext and data.isIetfInitial():
+    trace "Routing initial datagram with unknown CID to server context",
+      bytes = data.len, local, remote
     endpoint.serverContext.receive(Datagram(data: data), local, remote)
+    return
+
+  trace "Dropping datagram with unknown CID", bytes = data.len, local, remote
 
 proc receiveFromUdp(
     endpoint: QuicEndpoint, udp: DatagramTransport, remote: TransportAddress