chore: replace minprotobuf with nim-protobuf-serialization

* convert all codecs except sds_persistency to proto2
* add common/protobuf_ext.nim: generic array[N,byte] protobuf extension
* encode PeerId and MultiAddress via libp2p extensions
* encode PublicKey as seq[byte]
* change encode() to return seq[byte] and update all .encode().buffer call sites
* reduce common/protobuf.nim to ProtobufError/ProtobufResult
* add wire-format tests for message, filter v2, lightpush v3/legacy, rln proof
* adapt chat2, chat2mix, chat2bridge, wakustealthcommitments
* delete tests/common/test_protobuf_validation
* add protobuf_serialization to logos_delivery.nimble

Author: fcecin

apps/chat2/chat2.nim

@@ -27,11 +27,12 @@ import
     peerinfo,
       # manage the information of a peer, such as peer ID and public / private key
     peerid, # Implement how peers interact
-    protobuf/minprotobuf, # message serialisation/deserialisation from and to protobufs
     nameresolving/dnsresolver,
   ] # define DNS resolution
+import protobuf_serialization, protobuf_serialization/pkg/results
 import
   logos_delivery/waku/[
+    common/protobuf,
     waku_core,
     waku_lightpush_legacy/common,
     waku_lightpush_legacy/rpc,
@@ -87,27 +88,36 @@ type
     nick*: string
     payload*: seq[byte]
 
-proc init*(T: type Chat2Message, buffer: seq[byte]): ProtoResult[T] =
-  var msg = Chat2Message()
-  let pb = initProtoBuffer(buffer)
+type Chat2MessagePB {.proto2.} = object
+  timestamp {.fieldNumber: 1, pint.}: Opt[uint64]
+  nick {.fieldNumber: 2.}: Opt[string]
+  payload {.fieldNumber: 3.}: Opt[seq[byte]]
 
-  var timestamp: uint64
-  discard ?pb.getField(1, timestamp)
-  msg.timestamp = int64(timestamp)
-
-  discard ?pb.getField(2, msg.nick)
-  discard ?pb.getField(3, msg.payload)
-
-  ok(msg)
-
-proc encode*(message: Chat2Message): ProtoBuffer =
-  var serialised = initProtoBuffer()
+proc decodeChat2Message(buffer: seq[byte]): ProtobufResult[Chat2Message] =
+  var pb: Chat2MessagePB
+  try:
+    pb = Protobuf.decode(buffer, Chat2MessagePB)
+  except SerializationError:
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
+  ok(
+    Chat2Message(
+      timestamp: int64(pb.timestamp.get(0'u64)),
+      nick: pb.nick.get(""),
+      payload: pb.payload.get(@[]),
+    )
+  )
 
-  serialised.write(1, uint64(message.timestamp))
-  serialised.write(2, message.nick)
-  serialised.write(3, message.payload)
+proc init*(T: type Chat2Message, buffer: seq[byte]): ProtobufResult[T] =
+  decodeChat2Message(buffer)
 
-  return serialised
+proc encode*(message: Chat2Message): seq[byte] =
+  Protobuf.encode(
+    Chat2MessagePB(
+      timestamp: Opt.some(uint64(message.timestamp)),
+      nick: Opt.some(message.nick),
+      payload: Opt.some(message.payload),
+    )
+  )
 
 proc toString*(message: Chat2Message): string =
   # Get message date and timestamp in local time
@@ -183,7 +193,7 @@ proc publish(c: Chat, line: string) =
     trace "lightpush response received", response = response
 
   var message = WakuMessage(
-    payload: chat2pb.buffer,
+    payload: chat2pb,
     contentTopic: c.contentTopic,
     version: 0,
     timestamp: getNanosecondTime(time),

apps/chat2bridge/chat2bridge.nim

@@ -89,7 +89,7 @@ proc toWakuMessage(
     payload: msgFields["text"].getStr().toBytes(),
   ).encode()
 
-  WakuMessage(payload: chat2pb.buffer, contentTopic: cmb.contentTopic, version: 0)
+  WakuMessage(payload: chat2pb, contentTopic: cmb.contentTopic, version: 0)
 
 proc toChat2(cmb: Chat2MatterBridge, jsonNode: JsonNode) {.async.} =
   let msg = cmb.toWakuMessage(jsonNode)

apps/chat2mix/chat2mix.nim

@@ -27,14 +27,15 @@ import
     peerinfo,
       # manage the information of a peer, such as peer ID and public / private key
     peerid, # Implement how peers interact
-    protobuf/minprotobuf, # message serialisation/deserialisation from and to protobufs
     protocols/kademlia/types,
     protocols/service_discovery/types as sd_types,
     nameresolving/dnsresolver,
   ] # define DNS resolution
 import libp2p_mix/[curve25519, mix_protocol]
+import protobuf_serialization, protobuf_serialization/pkg/results
 import
   logos_delivery/waku/[
+    common/protobuf,
     waku_core,
     waku_core/peers,
     waku_lightpush/common,
@@ -108,27 +109,36 @@ proc getPubsubTopic*(
       return "" #TODO: fix this.
   return $RelayShard(clusterId: conf.clusterId, shardId: shard.shardId)
 
-proc init*(T: type Chat2Message, buffer: seq[byte]): ProtoResult[T] =
-  var msg = Chat2Message()
-  let pb = initProtoBuffer(buffer)
+type Chat2MessagePB {.proto2.} = object
+  timestamp {.fieldNumber: 1, pint.}: Opt[uint64]
+  nick {.fieldNumber: 2.}: Opt[string]
+  payload {.fieldNumber: 3.}: Opt[seq[byte]]
 
-  var timestamp: uint64
-  discard ?pb.getField(1, timestamp)
-  msg.timestamp = int64(timestamp)
-
-  discard ?pb.getField(2, msg.nick)
-  discard ?pb.getField(3, msg.payload)
-
-  ok(msg)
-
-proc encode*(message: Chat2Message): ProtoBuffer =
-  var serialised = initProtoBuffer()
+proc decodeChat2Message(buffer: seq[byte]): ProtobufResult[Chat2Message] =
+  var pb: Chat2MessagePB
+  try:
+    pb = Protobuf.decode(buffer, Chat2MessagePB)
+  except SerializationError:
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
+  ok(
+    Chat2Message(
+      timestamp: int64(pb.timestamp.get(0'u64)),
+      nick: pb.nick.get(""),
+      payload: pb.payload.get(@[]),
+    )
+  )
 
-  serialised.write(1, uint64(message.timestamp))
-  serialised.write(2, message.nick)
-  serialised.write(3, message.payload)
+proc init*(T: type Chat2Message, buffer: seq[byte]): ProtobufResult[T] =
+  decodeChat2Message(buffer)
 
-  return serialised
+proc encode*(message: Chat2Message): seq[byte] =
+  Protobuf.encode(
+    Chat2MessagePB(
+      timestamp: Opt.some(uint64(message.timestamp)),
+      nick: Opt.some(message.nick),
+      payload: Opt.some(message.payload),
+    )
+  )
 
 proc `$`*(message: Chat2Message): string =
   # Get message date and timestamp in local time
@@ -203,7 +213,7 @@ proc publish(c: Chat, line: string) {.async.} =
     trace "lightpush response received", response = response
 
   var message = WakuMessage(
-    payload: chat2pb.buffer,
+    payload: chat2pb,
     contentTopic: c.contentTopic,
     version: 0,
     timestamp: getNanosecondTime(time),

examples/wakustealthcommitments/wire_spec.nim

@@ -1,10 +1,10 @@
 import std/times
 import confutils, chronicles, chronos, results
+import protobuf_serialization, protobuf_serialization/pkg/results
 
 import logos_delivery/waku/[waku_core, common/protobuf]
-import libp2p/protobuf/minprotobuf
 
-export times, confutils, chronicles, chronos, results, waku_core, protobuf, minprotobuf
+export times, confutils, chronicles, chronos, results, waku_core, protobuf
 
 type SerializedKey* = seq[byte]
 
@@ -16,57 +16,47 @@ type WakuStealthCommitmentMsg* = object
   stealthCommitment*: Opt[SerializedKey]
   viewTag*: Opt[uint64]
 
-proc decode*(T: type WakuStealthCommitmentMsg, buffer: seq[byte]): ProtoResult[T] =
+type WakuStealthCommitmentMsgPB {.proto2.} = object
+  request {.fieldNumber: 1, pint.}: Opt[uint64]
+  spendingPubKey {.fieldNumber: 2.}: Opt[seq[byte]]
+  viewingPubKey {.fieldNumber: 3.}: Opt[seq[byte]]
+  stealthCommitment {.fieldNumber: 4.}: Opt[seq[byte]]
+  ephemeralPubKey {.fieldNumber: 5.}: Opt[seq[byte]]
+  viewTag {.fieldNumber: 6, pint.}: Opt[uint64]
+
+proc nonEmptyKey(o: Opt[seq[byte]]): Opt[SerializedKey] =
+  if o.isSome() and o.get().len > 0:
+    o
+  else:
+    Opt.none(SerializedKey)
+
+proc decode*(T: type WakuStealthCommitmentMsg, buffer: seq[byte]): ProtobufResult[T] =
+  var pb: WakuStealthCommitmentMsgPB
+  try:
+    pb = Protobuf.decode(buffer, WakuStealthCommitmentMsgPB)
+  except SerializationError:
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
+
   var msg = WakuStealthCommitmentMsg()
-  let pb = initProtoBuffer(buffer)
-
-  var request: uint64
-  discard ?pb.getField(1, request)
-  msg.request = request == 1
-  var spendingPubKey = newSeq[byte]()
-  discard ?pb.getField(2, spendingPubKey)
-  msg.spendingPubKey =
-    if spendingPubKey.len > 0:
-      Opt.some(spendingPubKey)
-    else:
-      Opt.none(SerializedKey)
-  var viewingPubKey = newSeq[byte]()
-  discard ?pb.getField(3, viewingPubKey)
-  msg.viewingPubKey =
-    if viewingPubKey.len > 0:
-      Opt.some(viewingPubKey)
-    else:
-      Opt.none(SerializedKey)
+  msg.request = pb.request.get(0'u64) == 1
+  msg.spendingPubKey = nonEmptyKey(pb.spendingPubKey)
+  msg.viewingPubKey = nonEmptyKey(pb.viewingPubKey)
 
   if msg.spendingPubKey.isSome() and msg.viewingPubKey.isSome():
     msg.stealthCommitment = Opt.none(SerializedKey)
     msg.viewTag = Opt.none(uint64)
     return ok(msg)
   if msg.spendingPubKey.isSome() and msg.viewingPubKey.isNone():
-    return err(ProtoError.RequiredFieldMissing)
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
   if msg.spendingPubKey.isNone() and msg.viewingPubKey.isSome():
-    return err(ProtoError.RequiredFieldMissing)
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
   if msg.request == true and msg.spendingPubKey.isNone() and msg.viewingPubKey.isNone():
-    return err(ProtoError.RequiredFieldMissing)
-
-  var stealthCommitment = newSeq[byte]()
-  discard ?pb.getField(4, stealthCommitment)
-  msg.stealthCommitment =
-    if stealthCommitment.len > 0:
-      Opt.some(stealthCommitment)
-    else:
-      Opt.none(SerializedKey)
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
 
-  var ephemeralPubKey = newSeq[byte]()
-  discard ?pb.getField(5, ephemeralPubKey)
-  msg.ephemeralPubKey =
-    if ephemeralPubKey.len > 0:
-      Opt.some(ephemeralPubKey)
-    else:
-      Opt.none(SerializedKey)
+  msg.stealthCommitment = nonEmptyKey(pb.stealthCommitment)
+  msg.ephemeralPubKey = nonEmptyKey(pb.ephemeralPubKey)
 
-  var viewTag: uint64
-  discard ?pb.getField(6, viewTag)
+  let viewTag = pb.viewTag.get(0'u64)
   msg.viewTag =
     if viewTag != 0:
       Opt.some(viewTag)
@@ -75,37 +65,31 @@ proc decode*(T: type WakuStealthCommitmentMsg, buffer: seq[byte]): ProtoResult[T
 
   if msg.stealthCommitment.isNone() and msg.viewTag.isNone() and
       msg.ephemeralPubKey.isNone():
-    return err(ProtoError.RequiredFieldMissing)
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
 
   if msg.stealthCommitment.isSome() and msg.viewTag.isNone():
-    return err(ProtoError.RequiredFieldMissing)
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
 
   if msg.stealthCommitment.isNone() and msg.viewTag.isSome():
-    return err(ProtoError.RequiredFieldMissing)
+    return err(protobuf.ProtobufError(kind: ProtobufErrorKind.DecodeFailure))
 
   if msg.stealthCommitment.isSome() and msg.viewTag.isSome():
     msg.spendingPubKey = Opt.none(SerializedKey)
     msg.viewingPubKey = Opt.none(SerializedKey)
 
   ok(msg)
 
-proc encode*(msg: WakuStealthCommitmentMsg): ProtoBuffer =
-  var serialised = initProtoBuffer()
-
-  serialised.write(1, uint64(msg.request))
-
-  if msg.spendingPubKey.isSome():
-    serialised.write(2, msg.spendingPubKey.get())
-  if msg.viewingPubKey.isSome():
-    serialised.write(3, msg.viewingPubKey.get())
-  if msg.stealthCommitment.isSome():
-    serialised.write(4, msg.stealthCommitment.get())
-  if msg.ephemeralPubKey.isSome():
-    serialised.write(5, msg.ephemeralPubKey.get())
-  if msg.viewTag.isSome():
-    serialised.write(6, msg.viewTag.get())
-
-  return serialised
+proc encode*(msg: WakuStealthCommitmentMsg): seq[byte] =
+  Protobuf.encode(
+    WakuStealthCommitmentMsgPB(
+      request: Opt.some(uint64(msg.request)),
+      spendingPubKey: msg.spendingPubKey,
+      viewingPubKey: msg.viewingPubKey,
+      stealthCommitment: msg.stealthCommitment,
+      ephemeralPubKey: msg.ephemeralPubKey,
+      viewTag: msg.viewTag,
+    )
+  )
 
 func toByteSeq*(str: string): seq[byte] {.inline.} =
   ## Converts a string to the corresponding byte sequence.

logos_delivery.nimble

@@ -26,6 +26,7 @@ requires "nim >= 2.2.4",
   "serialization",
   "json_serialization",
   "toml_serialization",
+  "protobuf_serialization",
   "faststreams",
   # Networking & P2P
   "https://github.com/vacp2p/nim-libp2p.git#v2.0.0",

logos_delivery/waku/common/protobuf.nim

@@ -1,10 +1,9 @@
-# Extensions for libp2p's protobuf library implementation
+# Waku protobuf result/error types.
 
 {.push raises: [].}
 
-import results, libp2p/protobuf/minprotobuf, libp2p/varint
-
-export minprotobuf, varint
+import results
+export results
 
 ## Custom errors
 
@@ -17,48 +16,23 @@ type
   ProtobufError* = object
     case kind*: ProtobufErrorKind
     of DecodeFailure:
-      error*: minprotobuf.ProtoError
+      discard
     of MissingRequiredField, InvalidLengthField:
       field*: string
 
   ProtobufResult*[T] = Result[T, ProtobufError]
 
-converter toProtobufError*(err: minprotobuf.ProtoError): ProtobufError =
-  case err
-  of minprotobuf.ProtoError.RequiredFieldMissing:
-    ProtobufError(kind: ProtobufErrorKind.MissingRequiredField, field: "unknown")
-  else:
-    ProtobufError(kind: ProtobufErrorKind.DecodeFailure, error: err)
-
 proc missingRequiredField*(T: type ProtobufError, field: string): T =
   ProtobufError(kind: ProtobufErrorKind.MissingRequiredField, field: field)
 
 proc invalidLengthField*(T: type ProtobufError, field: string): T =
   ProtobufError(kind: ProtobufErrorKind.InvalidLengthField, field: field)
 
-## Extension methods
-
-proc write3*(proto: var ProtoBuffer, field: int, value: auto) =
-  when value is Opt:
-    if value.isSome():
-      proto.write(field, value.get())
-  else:
-    proto.write(field, value)
-
-proc finish3*(proto: var ProtoBuffer) =
-  if proto.buffer.len > 0:
-    proto.finish()
-  else:
-    proto.offset = 0
-
-proc `==`*(a: zint64, b: zint64): bool =
-  int64(a) == int64(b)
-
 proc `$`*(err: ProtobufError): string =
   case err.kind
   of DecodeFailure:
-    return $err.error ## assume that ProtoError is pure
+    "DecodeFailure"
   of MissingRequiredField:
-    return "MissingRequiredField " & err.field
+    "MissingRequiredField " & err.field
   of InvalidLengthField:
-    return "InvalidLengthField " & err.field
+    "InvalidLengthField " & err.field