feat: ffiHandle

(cherry picked from commit 498416b)

Author: gmelodie

ffi.nim

@@ -3,12 +3,12 @@ import chronos, chronicles
 import
   ffi/internal/[ffi_library, ffi_macro],
   ffi/[
-    alloc, ffi_types, ffi_events, ffi_context, ffi_context_pool, ffi_thread_request,
-    cbor_serial,
+    alloc, ffi_types, ffi_events, ffi_handles, ffi_context, ffi_context_pool,
+    ffi_thread_request, cbor_serial,
   ]
 
 export atomics, tables
 export chronos, chronicles
 export
-  atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_events, ffi_context,
-  ffi_context_pool, ffi_thread_request, cbor_serial
+  atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_events, ffi_handles,
+  ffi_context, ffi_context_pool, ffi_thread_request, cbor_serial

ffi/codegen/cddl.nim

@@ -92,7 +92,7 @@ proc emitObjectFields(t: FFITypeMeta): string =
 proc emitReqFields(p: FFIProcMeta): string =
   var fields: seq[tuple[name: string, typeName: string, isPtr: bool]] = @[]
   for ep in p.extraParams:
-    fields.add((name: ep.name, typeName: ep.typeName, isPtr: ep.isPtr))
+    fields.add((name: ep.name, typeName: ep.typeName, isPtr: ep.ridesAsPtr()))
   emitMap(fields)
 
 proc responseRule(p: FFIProcMeta): string =
@@ -106,7 +106,7 @@ proc responseRule(p: FFIProcMeta): string =
     # The dtor has no meaningful payload — handleRes sends a CBOR null sentinel.
     "nil"
   of FFIKind.FFI:
-    if p.returnIsPtr:
+    if p.returnRidesAsPtr():
       "uint"
     else:
       nimTypeToCddl(p.returnTypeName)

ffi/codegen/cpp.nim

@@ -299,7 +299,7 @@ proc generateCppHeader*(
     lines.add("struct $1 {" % [reqName])
     for ep in p.extraParams:
       let cppType =
-        if ep.isPtr:
+        if ep.ridesAsPtr():
           CppPtrType
         else:
           nimTypeToCpp(ep.typeName)
@@ -308,7 +308,7 @@ proc generateCppHeader*(
     var fields: seq[(string, string)] = @[]
     for ep in p.extraParams:
       let cppType =
-        if ep.isPtr:
+        if ep.ridesAsPtr():
           CppPtrType
         else:
           nimTypeToCpp(ep.typeName)
@@ -388,7 +388,7 @@ proc generateCppHeader*(
     var epNames: seq[string] = @[]
     for ep in ctor.extraParams:
       let cppType =
-        if ep.isPtr:
+        if ep.ridesAsPtr():
           CppPtrType
         else:
           nimTypeToCpp(ep.typeName)
@@ -492,7 +492,7 @@ proc generateCppHeader*(
   for m in methods:
     let methodName = stripLibPrefixCpp(m.procName, libName)
     let retCppType =
-      if m.returnIsPtr:
+      if m.returnRidesAsPtr():
         CppPtrType
       else:
         nimTypeToCpp(m.returnTypeName)
@@ -502,7 +502,7 @@ proc generateCppHeader*(
     var methParamNames: seq[string] = @[]
     for ep in m.extraParams:
       let cppType =
-        if ep.isPtr:
+        if ep.ridesAsPtr():
           CppPtrType
         else:
           nimTypeToCpp(ep.typeName)

ffi/codegen/meta.nim

@@ -6,6 +6,7 @@ type
     name*: string # Nim param name, e.g. "req"
     typeName*: string # Nim type name, e.g. "EchoRequest"
     isPtr*: bool # true if the type is `ptr T`
+    isHandle*: bool # true if the type is an {.ffiHandle.} type (wire form uint64)
 
   FFIKind* {.pure.} = enum
     FFI
@@ -20,6 +21,7 @@ type
     extraParams*: seq[FFIParamMeta] # all params except the lib param
     returnTypeName*: string # e.g. "EchoResponse", "string", "pointer"
     returnIsPtr*: bool # true if return type is ptr T
+    returnIsHandle*: bool # true if return type is an {.ffiHandle.} type
 
   FFIFieldMeta* = object
     name*: string # e.g. "delayMs"
@@ -46,6 +48,24 @@ var ffiTypeRegistry* {.compileTime.}: seq[FFITypeMeta]
 var ffiEventRegistry* {.compileTime.}: seq[FFIEventMeta]
 var currentLibName* {.compileTime.}: string
 
+# Lib type name (set by declareLibrary) so handle-receiver procs resolve the pool.
+var currentLibType* {.compileTime.}: string
+
+# Names of types marked `{.ffiHandle.}` (wire form uint64).
+var ffiHandleTypeNames* {.compileTime.}: seq[string]
+
+proc isFFIHandleTypeName*(name: string): bool {.compileTime.} =
+  name in ffiHandleTypeNames
+
+proc ridesAsPtr*(ep: FFIParamMeta): bool =
+  ## True if the param crosses the wire as an opaque uint64 — a raw `ptr` or an
+  ## `{.ffiHandle.}` id. Both share the codegen pointer type.
+  ep.isPtr or ep.isHandle
+
+proc returnRidesAsPtr*(p: FFIProcMeta): bool =
+  ## True if the return crosses the wire as an opaque uint64 (raw `ptr` or handle).
+  p.returnIsPtr or p.returnIsHandle
+
 # Target language for binding generation; override with -d:targetLang=cpp
 const targetLang* {.strdefine.} = "rust"
 

ffi/codegen/rust.nim

@@ -256,7 +256,7 @@ proc generateTypesRs*(types: seq[FFITypeMeta], procs: seq[FFIProcMeta]): string
       for ep in p.extraParams:
         let snake = camelToSnakeCase(ep.name)
         let rustType =
-          if ep.isPtr:
+          if ep.ridesAsPtr():
             RustPtrType
           else:
             nimTypeToRust(ep.typeName)
@@ -550,7 +550,7 @@ proc generateApiRs*(
     for ep in ctor.extraParams:
       let snake = camelToSnakeCase(ep.name)
       let rustType =
-        if ep.isPtr:
+        if ep.ridesAsPtr():
           RustPtrType
         else:
           nimTypeToRust(ep.typeName)
@@ -711,7 +711,7 @@ proc generateApiRs*(
     for ep in m.extraParams:
       let snake = camelToSnakeCase(ep.name)
       let rustType =
-        if ep.isPtr:
+        if ep.ridesAsPtr():
           RustPtrType
         else:
           nimTypeToRust(ep.typeName)
@@ -729,7 +729,8 @@ proc generateApiRs*(
       else:
         reqName & " {}"
 
-    let retTypeForApi = if m.returnIsPtr: RustPtrType else: retRustType
+    let retTypeForApi =
+      if m.returnRidesAsPtr(): RustPtrType else: retRustType
 
     # -- blocking method --
     lines.add(

ffi/ffi_context.nim

@@ -8,9 +8,15 @@
 
 import std/[atomics, locks, options, tables]
 import chronicles, chronos, chronos/threadsync, taskpools/channels_spsc_single, results
-import ./ffi_types, ./ffi_events, ./ffi_thread_request, ./logging, ./cbor_serial
+import
+  ./ffi_types,
+  ./ffi_events,
+  ./ffi_handles,
+  ./ffi_thread_request,
+  ./logging,
+  ./cbor_serial
 
-export ffi_events
+export ffi_events, ffi_handles
 
 type FFIContext*[T] = object
   myLib*: ptr T # main library object (Waku, LibP2P, SDS, …)
@@ -27,6 +33,7 @@ type FFIContext*[T] = object
   eventThreadExitSignal: ThreadSignalPtr # mirrors threadExitSignal for the event thread
   userData*: pointer
   eventRegistry*: FFIEventRegistry
+  handles*: FFIHandleRegistry # live {.ffiHandle.} objects, keyed by uint64 id
   eventQueue*: EventQueue
   ffiHeartbeat*: Atomic[int64]
     # advanced each FFI-thread loop; event thread reads for liveness
@@ -57,6 +64,7 @@ proc deinitContextResources*[T](ctx: ptr FFIContext[T]): Result[void, string] =
   ## fields are nil'd after close so re-init on the same slot is safe.
   ctx.lock.deinitLock()
   deinitEventRegistry(ctx[].eventRegistry)
+  deinitHandleRegistry(ctx[].handles)
   deinitEventQueue(ctx[].eventQueue)
   when defined(gcRefc):
     # ThreadSignalPtr.close() under refc traps in safeUnregisterAndCloseFd
@@ -95,6 +103,7 @@ proc initContextResources*[T](ctx: ptr FFIContext[T]): Result[void, string] =
   ctx.eventThreadExitSignal = nil
   ctx.lock.initLock()
   initEventRegistry(ctx[].eventRegistry)
+  initHandleRegistry(ctx[].handles)
   initEventQueue(ctx[].eventQueue)
   ctx.ffiHeartbeat.store(0)
   ctx.eventQueueStuck.store(false)

ffi/ffi_handles.nim

@@ -0,0 +1,55 @@
+## Per-context registry of live `{.ffiHandle.}` objects. The object stays here,
+## in `FFIContext.handles`; only its `uint64` id crosses the boundary. Ids are
+## monotonic and never recycled (0 = null), so a stale/forged id misses cleanly.
+## FFI-thread-only access, so no locking.
+
+import std/tables
+
+type
+  FFIHandleRoot* = ref object of RootObj
+    ## Base every `{.ffiHandle.}` type inherits from, so handle refs are storable
+    ## under one static type.
+
+  FFIHandleEntry = object
+    obj: FFIHandleRoot
+    typeName: string
+
+  FFIHandleRegistry* = object
+    nextId*: uint64
+    byHandle*: Table[uint64, FFIHandleEntry]
+
+proc initHandleRegistry*(reg: var FFIHandleRegistry) =
+  reg.nextId = 0'u64
+  reg.byHandle = initTable[uint64, FFIHandleEntry]()
+
+proc deinitHandleRegistry*(reg: var FFIHandleRegistry) =
+  reg.byHandle = default(Table[uint64, FFIHandleEntry])
+  reg.nextId = 0'u64
+
+proc register*(
+    reg: var FFIHandleRegistry, obj: FFIHandleRoot, typeName: string
+): uint64 =
+  ## Stores `obj`, returns its fresh handle id (>0).
+  reg.nextId.inc()
+  reg.byHandle[reg.nextId] = FFIHandleEntry(obj: obj, typeName: typeName)
+  reg.nextId
+
+proc lookup*(
+    reg: var FFIHandleRegistry, handle: uint64, typeName: string
+): FFIHandleRoot =
+  ## Live ref for `handle`, or nil if absent or registered under another type.
+  let entry = reg.byHandle.getOrDefault(handle)
+  if entry.obj.isNil() or entry.typeName != typeName:
+    return nil
+  entry.obj
+
+proc release*(reg: var FFIHandleRegistry, handle: uint64): bool {.discardable.} =
+  ## Drops the entry; true iff it existed.
+  if not reg.byHandle.hasKey(handle):
+    return false
+  reg.byHandle.del(handle)
+  true
+
+proc releaseAll*(reg: var FFIHandleRegistry) =
+  ## Drops every entry. Must run on the FFI thread that allocated the refs.
+  reg.byHandle.clear()

ffi/ffi_thread.nim

@@ -101,6 +101,8 @@ proc ffiThreadBody[T](ctx: ptr FFIContext[T]) {.thread.} =
 
   defer:
     onFFIThread = false
+    # Free handle refs on the FFI thread that allocated them (refc heap is thread-local).
+    ctx[].handles.releaseAll()
     # Unblocks destroyFFIContext's bounded wait so cleanup can proceed.
     let fireRes = ctx.threadExitSignal.fireSync()
     if fireRes.isErr():

ffi/internal/ffi_library.nim

@@ -121,11 +121,19 @@ macro declareLibrary*(libraryName: static[string], libType: untyped): untyped =
   ## `libType` is the Nim type of the main library object, used to type
   ## the `ctx: ptr FFIContext[libType]` parameter. See
   ## `examples/timer/timer.nim` for a working call site.
+  currentLibType = $libType # so handle-receiver `.ffi.` procs can resolve the pool
+
   var stmts = newStmtList()
 
   # Emit the base bootstrap (pragmas, linker flags, NimMain, initializeLibrary)
   stmts.add(newCall(ident("declareLibraryBase"), newStrLitNode(libraryName)))
 
+  # The pool the generated wrappers validate against; ffiCtor/ffiDtor guard alike.
+  let poolIdent = ident($libType & "FFIPool")
+  stmts.add quote do:
+    when not declared(`poolIdent`):
+      var `poolIdent`*: FFIContextPool[`libType`]
+
   let ctxType = nnkPtrTy.newTree(nnkBracketExpr.newTree(ident("FFIContext"), libType))
   let cdeclExportPragma = newTree(
     nnkPragma,