chore(cbind): nim-ffi migration [4/9] — streams & custom protocols (+ echo example)

cbind/.pinned

@@ -1,3 +1,3 @@
 taskpools;https://github.com/status-im/nim-taskpools@#9e8ccc754631ac55ac2fd495e167e74e86293edb
 cbor_serialization;https://github.com/vacp2p/nim-cbor-serialization@#1664160e04d153573373afddc552b9cbf6fbe4dc
-ffi;https://github.com/logos-messaging/nim-ffi@#3e57751e3a5fd8e0f9b66a9a01ef24a9447863aa
+ffi;https://github.com/logos-messaging/nim-ffi@#7ef58f4bf5cf683cfdf2abbdb3d7688727763d65

cbind/Makefile

@@ -1,6 +1,14 @@
-.PHONY: all deps build clean
+.PHONY: all build genbindings deps nixbuild clean
 
-all: build
+all: build genbindings
+
+# Builds ../build/liblibp2p.so plus the C/CDDL bindings. Run `nimble setup` once first.
+build:
+	nimble buildffi
+
+genbindings:
+	nimble genbindings_c
+	nimble genbindings_cddl
 
 nimble.lock:
 	nimble lock
@@ -10,9 +18,9 @@ nimble.lock:
 
 deps: ../nix/cbind-deps.nix
 
-build: deps
+nixbuild: deps
 	nix build .#cbind
 
 clean:
 	$(RM) nimble.lock ../nix/cbind-deps.nix
-
+	$(RM) -r c_bindings cddl_bindings nimcache nimcache_c nimcache_cddl

cbind/cbind.nimble

@@ -55,42 +55,37 @@ task libDynamic, "Generate dynamic bindings":
 task libStatic, "Generate static bindings":
   buildCBindings "static", ""
 
-proc findNatPkgDir(): string =
-  # Match the top-level Makefile: nimble installs deps under pkgs2 on newer
-  # versions and pkgs on older ones; resolve from either.
-  for base in ["../nimbledeps/pkgs2", "../nimbledeps/pkgs"]:
-    if dirExists(base):
-      for d in listDirs(base):
-        if d.extractFilename().startsWith("nat_traversal-"):
-          return d
-  quit "nat_traversal package not found under ../nimbledeps/pkgs2 or " &
-    "../nimbledeps/pkgs; run 'nimble install_pinned' first"
-
-task examples, "Build and run C bindings examples":
-  buildCBindings "static", ""
-  # libp2p.a transitively references miniupnpc and libnatpmp via nat_traversal.
-  # Build the vendored .a's via the parent Makefile and link them in.
-  exec "make -C .. nat_libs"
-  let natPkg = findNatPkgDir()
-  # miniupnpc's unix Makefile drops the .a under build/, but its Makefile.mingw
-  # drops it at the package root. Match the parent Makefile's per-OS choice.
-  let upnpA =
-    when defined(windows):
-      natPkg / "vendor/miniupnp/miniupnpc/libminiupnpc.a"
-    else:
-      natPkg / "vendor/miniupnp/miniupnpc/build/libminiupnpc.a"
-  let pmpA = natPkg / "vendor/libnatpmp-upstream/libnatpmp.a"
-  let natLibs = upnpA & " " & pmpA
-  exec "g++ -I. -o ../build/cbindings ./examples/cbindings.c ../build/libp2p.a " &
-    natLibs & " -pthread"
-  exec "g++ -I. -o ../build/echo ./examples/echo.c ../build/libp2p.a " & natLibs &
-    " -pthread"
-  exec "../build/cbindings"
-  exec "../build/echo"
-
 # nim-ffi library, built in parallel to the legacy cbind above (see libp2p_ffi.nim).
 # Renamed over libp2p.nim at the flip PR, which drops everything above this line.
 
+proc findInstalledPkgDir(prefix: string): string =
+  ## Path of an installed dep dir matching `prefix` (e.g. "ffi-"). install_pinned
+  ## drops cbind's pinned deps under the project-local `nimbledeps/pkgs2`; a plain
+  ## `nimble install` uses the global store. Check both.
+  var bases = @["nimbledeps/pkgs2", "../nimbledeps/pkgs2"]
+  let home = getEnv("HOME")
+  if home.len > 0:
+    bases.add home & "/.nimble/pkgs2"
+  for base in bases:
+    if not dirExists(base):
+      continue
+    for entry in listDirs(base):
+      if entry.extractFilename().startsWith(prefix):
+        return entry
+  raise newException(
+    IOError,
+    "could not locate installed package '" & prefix &
+      "*'; run `nimble install_pinned` first",
+  )
+
+proc ffiDepPaths(): string =
+  # ffi and cbor_serialization aren't cbind `requires` (they're not in the nimble
+  # registry, so setup can't resolve them onto nimble.paths); point the compiler
+  # at the installed copies. Their transitive deps (chronos, serialization, stew,
+  # results, faststreams, …) are libp2p deps already on the inherited root paths.
+  " --path:" & findInstalledPkgDir("ffi-") & " --path:" &
+    findInstalledPkgDir("cbor_serialization-")
+
 proc ffiLibExt(): string =
   when defined(windows):
     "dll"
@@ -103,14 +98,17 @@ proc buildFfiLib() =
   let buildDir = "../build"
   if not dirExists(buildDir):
     mkDir(buildDir)
+  # liblibp2p.so transitively references miniupnpc and libnatpmp via
+  # nat_traversal's {.passL.} of their vendored .a's; build them first.
+  exec "make -C .. nat_libs"
   # Name the output `lib<name>` so the file matches the soname nim derives from
   # the module; `--nimMainPrefix:liblibp2p` matches the `liblibp2pNimMain` symbol
   # nim-ffi's `declareLibrary` imports.
   # ffiThreadExitTimeoutMs: bound the FFI thread's graceful-shutdown wait; the
   # 1500ms default is too tight for libp2pDestroy's switch.stop() over many conns.
   exec "nim c --out:" & buildDir & "/liblibp2p." & ffiLibExt() &
     " --threads:on --app:lib --opt:size --noMain --mm:refc -d:metrics" &
-    " -d:ffiThreadExitTimeoutMs=5000" &
+    " -d:ffiThreadExitTimeoutMs=5000" & ffiDepPaths() &
     " --nimMainPrefix:liblibp2p --nimcache:nimcache libp2p_ffi.nim"
 
 task buildffi, "Build the FFI shared library":
@@ -119,11 +117,43 @@ task buildffi, "Build the FFI shared library":
 proc genBindingsFor(lang, outDir: string) =
   exec "nim c --threads:on --app:lib --noMain --mm:refc -d:metrics" &
     " --nimMainPrefix:liblibp2p -d:ffiGenBindings -d:targetLang=" & lang &
-    " -d:ffiOutputDir=" & outDir & " -d:ffiSrcPath=libp2p_ffi.nim" &
+    " -d:ffiOutputDir=" & outDir & " -d:ffiSrcPath=libp2p_ffi.nim" & ffiDepPaths() &
     " --nimcache:nimcache_" & lang & " -o:/dev/null libp2p_ffi.nim"
 
 task genbindings_c, "Generate C bindings (cbind/c_bindings)":
   genBindingsFor("c", "c_bindings")
 
 task genbindings_cddl, "Generate CDDL schema (cbind/cddl_bindings)":
   genBindingsFor("cddl", "cddl_bindings")
+
+proc findFfiVendorDir(): string =
+  ## TinyCBOR sources vendored inside the installed nim-ffi package.
+  let vendor = findInstalledPkgDir("ffi-") & "/ffi/codegen/templates/cpp/vendor"
+  if not fileExists(vendor & "/tinycbor/cbor.h"):
+    raise newException(IOError, "vendored tinycbor missing under " & vendor)
+  vendor
+
+task examples, "Build and run the C bindings examples":
+  let lib = "../build/liblibp2p." & ffiLibExt()
+  if not fileExists(lib):
+    buildFfiLib()
+  if not fileExists("c_bindings/libp2p.h"):
+    genBindingsFor("c", "c_bindings")
+
+  let vendor = findFfiVendorDir()
+  var cborObjs: seq[string]
+  for name in [
+    "cborencoder", "cborencoder_close_container_checked", "cborparser",
+    "cborparser_dup_string", "cborerrorstrings",
+  ]:
+    let obj = "../build/" & name & ".o"
+    exec "gcc -std=c99 -O2 -fPIC -I " & vendor & " -I " & vendor & "/tinycbor -c " &
+      vendor & "/tinycbor/" & name & ".c -o " & obj
+    cborObjs.add obj
+  let cborObjsStr = cborObjs.join(" ")
+
+  for example in ["echo"]:
+    let outBin = "../build/" & example
+    exec "gcc -std=c11 -O2 -I c_bindings -I " & vendor & " examples/" & example & ".c " &
+      cborObjsStr & " " & lib & " -pthread -Wl,-rpath,'$ORIGIN' -o " & outBin
+    exec outBin

cbind/examples/CMakeLists.txt

@@ -0,0 +1,73 @@
+cmake_minimum_required(VERSION 3.14)
+project(libp2p_cbind_examples C)
+
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_C_STANDARD_REQUIRED ON)
+
+set(CBIND_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..")
+set(GEN_DIR "${CBIND_DIR}/c_bindings")
+set(BUILD_DIR "${CBIND_DIR}/../build")
+
+if(APPLE)
+  set(LIB_FILE "${BUILD_DIR}/liblibp2p.dylib")
+elseif(WIN32)
+  set(LIB_FILE "${BUILD_DIR}/liblibp2p.dll")
+else()
+  set(LIB_FILE "${BUILD_DIR}/liblibp2p.so")
+endif()
+
+if(NOT EXISTS "${GEN_DIR}/libp2p.h")
+  message(FATAL_ERROR
+    "Generated header ${GEN_DIR}/libp2p.h not found. "
+    "Run `nimble genbindings_c` in cbind/ first.")
+endif()
+if(NOT EXISTS "${LIB_FILE}")
+  message(FATAL_ERROR
+    "Runtime library ${LIB_FILE} not found. Run `nimble buildffi` in cbind/ first.")
+endif()
+
+# ── Locate TinyCBOR (vendored inside the nim-ffi package) ────────────────────
+# The generated header includes <tinycbor/cbor.h> and defers to the vendored
+# sources. Override with -DTINYCBOR_VENDOR=/path/to/vendor if auto-detection
+# fails.
+if(NOT TINYCBOR_VENDOR)
+  file(GLOB _ffi_vendors
+    "${CBIND_DIR}/nimbledeps/pkgs2/ffi-*/ffi/codegen/templates/cpp/vendor"
+    "${CBIND_DIR}/../nimbledeps/pkgs2/ffi-*/ffi/codegen/templates/cpp/vendor"
+    "$ENV{HOME}/.nimble/pkgs2/ffi-*/ffi/codegen/templates/cpp/vendor")
+  foreach(_v ${_ffi_vendors})
+    if(EXISTS "${_v}/tinycbor/cbor.h")
+      set(TINYCBOR_VENDOR "${_v}")
+      break()
+    endif()
+  endforeach()
+endif()
+if(NOT TINYCBOR_VENDOR OR NOT EXISTS "${TINYCBOR_VENDOR}/tinycbor/cbor.h")
+  message(FATAL_ERROR
+    "Could not locate the vendored TinyCBOR sources. "
+    "Pass -DTINYCBOR_VENDOR=/path/to/ffi/codegen/templates/cpp/vendor")
+endif()
+
+add_library(tinycbor STATIC
+  "${TINYCBOR_VENDOR}/tinycbor/cborencoder.c"
+  "${TINYCBOR_VENDOR}/tinycbor/cborencoder_close_container_checked.c"
+  "${TINYCBOR_VENDOR}/tinycbor/cborparser.c"
+  "${TINYCBOR_VENDOR}/tinycbor/cborparser_dup_string.c"
+  "${TINYCBOR_VENDOR}/tinycbor/cborerrorstrings.c")
+target_include_directories(tinycbor PUBLIC
+  "${TINYCBOR_VENDOR}"
+  "${TINYCBOR_VENDOR}/tinycbor")
+set_property(TARGET tinycbor PROPERTY C_STANDARD 99)
+set_property(TARGET tinycbor PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+find_package(Threads REQUIRED)
+
+foreach(example echo)
+  add_executable(${example} "${example}.c")
+  target_include_directories(${example} PRIVATE "${GEN_DIR}")
+  target_link_libraries(${example} PRIVATE "${LIB_FILE}" tinycbor Threads::Threads)
+  # nanosleep needs a POSIX feature level that strict `-std=c11` hides.
+  target_compile_definitions(${example} PRIVATE _POSIX_C_SOURCE=200809L)
+  # So the loader finds libp2p.{so,dylib} next to it at runtime.
+  set_target_properties(${example} PROPERTIES BUILD_RPATH "${BUILD_DIR}")
+endforeach()

cbind/examples/README.md

@@ -0,0 +1,39 @@
+# cbind examples
+
+Worked examples of consuming nim-libp2p from C through the FFI bindings generated by [nim-ffi](https://github.com/logos-messaging/nim-ffi).
+
+The bindings are a header-only C API over a C ABI. Every generated method is asynchronous: it takes a result callback and returns immediately, and the callback fires from the library's dispatch thread. Library-initiated events (incoming streams, pub/sub messages) arrive through `libp2p_ctx_add_on_…_listener` callbacks. The examples turn each async call back into a blocking step with the small helpers in `common.h`.
+
+- `echo.c` — two TCP nodes; the server mounts a custom `/cbind/echo/1.0.0` protocol and echoes the bytes it reads, the client dials it and verifies the round-trip. The incoming-stream handler only hands the stream id to `main`, which serves it inline, since calling back into the library from the dispatch thread would deadlock.
+
+## Build
+
+From `cbind/`, produce the runtime library and the generated header first:
+
+```sh
+nimble setup
+nimble buildffi       # -> ../build/liblibp2p.so
+nimble genbindings_c  # -> c_bindings/libp2p.h
+```
+
+Then build the examples:
+
+```sh
+cd examples
+cmake -S . -B build
+cmake --build build
+```
+
+If the vendored TinyCBOR sources can't be located automatically, point CMake at them:
+
+```sh
+cmake -S . -B build -DTINYCBOR_VENDOR=$(nimble path ffi)/ffi/codegen/templates/cpp/vendor
+```
+
+## Run
+
+```sh
+./build/echo
+```
+
+Each program exits `0` on success and prints what it sent and received.