Skip to content

Commit 021f469

Browse files
authored
feat: {.ffiHandle.} — export complex live objects as opaque handles (#81)
1 parent 16dc1b3 commit 021f469

14 files changed

Lines changed: 399 additions & 38 deletions

File tree

.github/workflows/ci.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,8 @@ jobs:
133133
uses: actions/cache@v4
134134
with:
135135
path: tests/e2e/cpp/build/_deps
136-
key: ${{ runner.os }}-cpp-e2e-deps-${{ hashFiles('tests/e2e/cpp/CMakeLists.txt') }}
136+
# ImageVersion busts the cache when the toolchain (e.g. MSVC) bumps.
137+
key: ${{ runner.os }}-cpp-e2e-deps-${{ env.ImageVersion }}-${{ hashFiles('tests/e2e/cpp/CMakeLists.txt') }}
137138

138139
- name: Run C++ e2e tests
139140
shell: bash

.github/workflows/tests-sanitized.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ jobs:
8383
uses: actions/cache@v4
8484
with:
8585
path: tests/e2e/cpp/build/_deps
86-
key: ${{ runner.os }}-${{ runner.arch }}-cpp-e2e-deps-${{ inputs.sanitizer }}-${{ hashFiles('tests/e2e/cpp/CMakeLists.txt') }}
86+
# ImageVersion busts the cache when the toolchain bumps.
87+
key: ${{ runner.os }}-${{ runner.arch }}-cpp-e2e-deps-${{ inputs.sanitizer }}-${{ env.ImageVersion }}-${{ hashFiles('tests/e2e/cpp/CMakeLists.txt') }}
8788

8889
- name: Run unit tests (${{ inputs.sanitizer }})
8990
run: nimble test_sanitized -y

ffi.nim

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ import chronos, chronicles
33
import
44
ffi/internal/[ffi_library, ffi_macro],
55
ffi/[
6-
alloc, ffi_types, ffi_events, ffi_context, ffi_context_pool, ffi_thread_request,
7-
cbor_serial,
6+
alloc, ffi_types, ffi_events, ffi_handles, ffi_context, ffi_context_pool,
7+
ffi_thread_request, cbor_serial,
88
]
99

1010
export atomics, tables
1111
export chronos, chronicles
1212
export
13-
atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_events, ffi_context,
14-
ffi_context_pool, ffi_thread_request, cbor_serial
13+
atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_events, ffi_handles,
14+
ffi_context, ffi_context_pool, ffi_thread_request, cbor_serial

ffi.nimble

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ requires "nim >= 2.2.4"
1111
requires "chronos"
1212
requires "chronicles"
1313
requires "taskpools"
14-
requires "cbor_serialization"
14+
requires "cbor_serialization == 0.3.0"
1515

1616
const nimFlagsOrc = "--mm:orc -d:chronicles_log_level=WARN"
1717
const nimFlagsRefc = "--mm:refc -d:chronicles_log_level=WARN"

ffi/codegen/cddl.nim

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ proc emitObjectFields(t: FFITypeMeta): string =
9292
proc emitReqFields(p: FFIProcMeta): string =
9393
var fields: seq[tuple[name: string, typeName: string, isPtr: bool]] = @[]
9494
for ep in p.extraParams:
95-
fields.add((name: ep.name, typeName: ep.typeName, isPtr: ep.isPtr))
95+
fields.add((name: ep.name, typeName: ep.typeName, isPtr: ep.ridesAsPtr()))
9696
emitMap(fields)
9797

9898
proc responseRule(p: FFIProcMeta): string =
@@ -106,7 +106,7 @@ proc responseRule(p: FFIProcMeta): string =
106106
# The dtor has no meaningful payload — handleRes sends a CBOR null sentinel.
107107
"nil"
108108
of FFIKind.FFI:
109-
if p.returnIsPtr:
109+
if p.returnRidesAsPtr():
110110
"uint"
111111
else:
112112
nimTypeToCddl(p.returnTypeName)

ffi/codegen/cpp.nim

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ proc generateCppHeader*(
299299
lines.add("struct $1 {" % [reqName])
300300
for ep in p.extraParams:
301301
let cppType =
302-
if ep.isPtr:
302+
if ep.ridesAsPtr():
303303
CppPtrType
304304
else:
305305
nimTypeToCpp(ep.typeName)
@@ -308,7 +308,7 @@ proc generateCppHeader*(
308308
var fields: seq[(string, string)] = @[]
309309
for ep in p.extraParams:
310310
let cppType =
311-
if ep.isPtr:
311+
if ep.ridesAsPtr():
312312
CppPtrType
313313
else:
314314
nimTypeToCpp(ep.typeName)
@@ -388,7 +388,7 @@ proc generateCppHeader*(
388388
var epNames: seq[string] = @[]
389389
for ep in ctor.extraParams:
390390
let cppType =
391-
if ep.isPtr:
391+
if ep.ridesAsPtr():
392392
CppPtrType
393393
else:
394394
nimTypeToCpp(ep.typeName)
@@ -492,7 +492,7 @@ proc generateCppHeader*(
492492
for m in methods:
493493
let methodName = stripLibPrefixCpp(m.procName, libName)
494494
let retCppType =
495-
if m.returnIsPtr:
495+
if m.returnRidesAsPtr():
496496
CppPtrType
497497
else:
498498
nimTypeToCpp(m.returnTypeName)
@@ -502,7 +502,7 @@ proc generateCppHeader*(
502502
var methParamNames: seq[string] = @[]
503503
for ep in m.extraParams:
504504
let cppType =
505-
if ep.isPtr:
505+
if ep.ridesAsPtr():
506506
CppPtrType
507507
else:
508508
nimTypeToCpp(ep.typeName)

ffi/codegen/meta.nim

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ type
66
name*: string # Nim param name, e.g. "req"
77
typeName*: string # Nim type name, e.g. "EchoRequest"
88
isPtr*: bool # true if the type is `ptr T`
9+
isHandle*: bool # true if the type is an {.ffiHandle.} type (wire form uint64)
910

1011
FFIKind* {.pure.} = enum
1112
FFI
@@ -20,6 +21,7 @@ type
2021
extraParams*: seq[FFIParamMeta] # all params except the lib param
2122
returnTypeName*: string # e.g. "EchoResponse", "string", "pointer"
2223
returnIsPtr*: bool # true if return type is ptr T
24+
returnIsHandle*: bool # true if return type is an {.ffiHandle.} type
2325

2426
FFIFieldMeta* = object
2527
name*: string # e.g. "delayMs"
@@ -46,6 +48,24 @@ var ffiTypeRegistry* {.compileTime.}: seq[FFITypeMeta]
4648
var ffiEventRegistry* {.compileTime.}: seq[FFIEventMeta]
4749
var currentLibName* {.compileTime.}: string
4850

51+
# Lib type name (set by declareLibrary) so handle-receiver procs resolve the pool.
52+
var currentLibType* {.compileTime.}: string
53+
54+
# Names of types marked `{.ffiHandle.}` (wire form uint64).
55+
var ffiHandleTypeNames* {.compileTime.}: seq[string]
56+
57+
proc isFFIHandleTypeName*(name: string): bool {.compileTime.} =
58+
name in ffiHandleTypeNames
59+
60+
proc ridesAsPtr*(ep: FFIParamMeta): bool =
61+
## True if the param crosses the wire as an opaque uint64 — a raw `ptr` or an
62+
## `{.ffiHandle.}` id. Both share the codegen pointer type.
63+
ep.isPtr or ep.isHandle
64+
65+
proc returnRidesAsPtr*(p: FFIProcMeta): bool =
66+
## True if the return crosses the wire as an opaque uint64 (raw `ptr` or handle).
67+
p.returnIsPtr or p.returnIsHandle
68+
4969
# Target language for binding generation; override with -d:targetLang=cpp
5070
const targetLang* {.strdefine.} = "rust"
5171

ffi/codegen/rust.nim

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ proc generateTypesRs*(types: seq[FFITypeMeta], procs: seq[FFIProcMeta]): string
256256
for ep in p.extraParams:
257257
let snake = camelToSnakeCase(ep.name)
258258
let rustType =
259-
if ep.isPtr:
259+
if ep.ridesAsPtr():
260260
RustPtrType
261261
else:
262262
nimTypeToRust(ep.typeName)
@@ -550,7 +550,7 @@ proc generateApiRs*(
550550
for ep in ctor.extraParams:
551551
let snake = camelToSnakeCase(ep.name)
552552
let rustType =
553-
if ep.isPtr:
553+
if ep.ridesAsPtr():
554554
RustPtrType
555555
else:
556556
nimTypeToRust(ep.typeName)
@@ -711,7 +711,7 @@ proc generateApiRs*(
711711
for ep in m.extraParams:
712712
let snake = camelToSnakeCase(ep.name)
713713
let rustType =
714-
if ep.isPtr:
714+
if ep.ridesAsPtr():
715715
RustPtrType
716716
else:
717717
nimTypeToRust(ep.typeName)
@@ -729,7 +729,7 @@ proc generateApiRs*(
729729
else:
730730
reqName & " {}"
731731

732-
let retTypeForApi = if m.returnIsPtr: RustPtrType else: retRustType
732+
let retTypeForApi = if m.returnRidesAsPtr(): RustPtrType else: retRustType
733733

734734
# -- blocking method --
735735
lines.add(

ffi/ffi_context.nim

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,15 @@
88

99
import std/[atomics, locks, options, tables]
1010
import chronicles, chronos, chronos/threadsync, taskpools/channels_spsc_single, results
11-
import ./ffi_types, ./ffi_events, ./ffi_thread_request, ./logging, ./cbor_serial
11+
import
12+
./ffi_types,
13+
./ffi_events,
14+
./ffi_handles,
15+
./ffi_thread_request,
16+
./logging,
17+
./cbor_serial
1218

13-
export ffi_events
19+
export ffi_events, ffi_handles
1420

1521
type FFIContext*[T] = object
1622
myLib*: ptr T # main library object (Waku, LibP2P, SDS, …)
@@ -27,6 +33,7 @@ type FFIContext*[T] = object
2733
eventThreadExitSignal: ThreadSignalPtr # mirrors threadExitSignal for the event thread
2834
userData*: pointer
2935
eventRegistry*: FFIEventRegistry
36+
handles*: FFIHandleRegistry # live {.ffiHandle.} objects, keyed by uint64 id
3037
eventQueue*: EventQueue
3138
ffiHeartbeat*: Atomic[int64]
3239
# advanced each FFI-thread loop; event thread reads for liveness
@@ -57,6 +64,7 @@ proc deinitContextResources*[T](ctx: ptr FFIContext[T]): Result[void, string] =
5764
## fields are nil'd after close so re-init on the same slot is safe.
5865
ctx.lock.deinitLock()
5966
deinitEventRegistry(ctx[].eventRegistry)
67+
deinitHandleRegistry(ctx[].handles)
6068
deinitEventQueue(ctx[].eventQueue)
6169
when defined(gcRefc):
6270
# ThreadSignalPtr.close() under refc traps in safeUnregisterAndCloseFd
@@ -95,6 +103,7 @@ proc initContextResources*[T](ctx: ptr FFIContext[T]): Result[void, string] =
95103
ctx.eventThreadExitSignal = nil
96104
ctx.lock.initLock()
97105
initEventRegistry(ctx[].eventRegistry)
106+
initHandleRegistry(ctx[].handles)
98107
initEventQueue(ctx[].eventQueue)
99108
ctx.ffiHeartbeat.store(0)
100109
ctx.eventQueueStuck.store(false)

ffi/ffi_handles.nim

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
## Per-context registry of live `{.ffiHandle.}` objects. The object stays here,
2+
## in `FFIContext.handles`; only its `uint64` id crosses the boundary. Ids are
3+
## monotonic and never recycled (0 = null), so a stale/forged id misses cleanly.
4+
## FFI-thread-only access, so no locking.
5+
6+
import std/tables
7+
import results
8+
import ./cbor_serial
9+
10+
type
11+
FFIHandleRoot* = ref object of RootObj
12+
## Base every `{.ffiHandle.}` type inherits from, so handle refs are storable
13+
## under one static type.
14+
15+
FFIHandleEntry = object
16+
obj: FFIHandleRoot
17+
typeName: string
18+
19+
FFIHandleRegistry* = object
20+
nextId*: uint64
21+
byHandle*: Table[uint64, FFIHandleEntry]
22+
23+
proc initHandleRegistry*(reg: var FFIHandleRegistry) =
24+
reg.nextId = 0'u64
25+
reg.byHandle = initTable[uint64, FFIHandleEntry]()
26+
27+
proc deinitHandleRegistry*(reg: var FFIHandleRegistry) =
28+
reg.byHandle = default(Table[uint64, FFIHandleEntry])
29+
reg.nextId = 0'u64
30+
31+
proc register*(
32+
reg: var FFIHandleRegistry, obj: FFIHandleRoot, typeName: string
33+
): uint64 =
34+
## Stores `obj`, returns its fresh handle id (>0).
35+
reg.nextId.inc()
36+
reg.byHandle[reg.nextId] = FFIHandleEntry(obj: obj, typeName: typeName)
37+
reg.nextId
38+
39+
proc lookup*(
40+
reg: var FFIHandleRegistry, handle: uint64, typeName: string
41+
): Result[FFIHandleRoot, string] =
42+
## Live ref for `handle`; err if absent or registered under another type.
43+
let entry = reg.byHandle.getOrDefault(handle)
44+
if entry.obj.isNil():
45+
return err("no ffiHandle with id " & $handle)
46+
if entry.typeName != typeName:
47+
return err(
48+
"ffiHandle " & $handle & " has type '" & entry.typeName & "', expected '" &
49+
typeName & "'"
50+
)
51+
ok(entry.obj)
52+
53+
proc release*(reg: var FFIHandleRegistry, handle: uint64): bool {.discardable.} =
54+
## Drops the entry; true iff it existed.
55+
if not reg.byHandle.hasKey(handle):
56+
return false
57+
reg.byHandle.del(handle)
58+
return true
59+
60+
proc releaseAll*(reg: var FFIHandleRegistry) =
61+
## Drops every entry. Must run on the FFI thread that allocated the refs.
62+
reg.byHandle.clear()
63+
64+
proc encodeHandle*(id: uint64): seq[byte] =
65+
## Wire-form bytes for a handle id. Single ABI seam for future format changes.
66+
cborEncode(id)

0 commit comments

Comments
 (0)