chore: retrieve cache of merkle roots from RLN contract (#3903)

* Only add new roots, not all received

* Fix error in removing recent roots not checking AcceptableWindowSize

* fix merging

* more merging fixes

* merge fixes

* add test for updated merkle roots window

* add pr re-add gauge for proof-generation-duration-seconds

* Decrease AcceptableRootWindowSize for testing

* debug spam log

* linting

* start trackRootChanges call loop immediately

* Fix 5s delay trackRootChanges

* set rpcDelay for root tracking to 10s

* add default params to sendEthCallWithParams

* improve recents roots retrieval and logs

* Use updateRecentRoots to track root changes

* simplify updateRecentRoots

* set root polling to 15s

* set rpc poll delay to 30s

* set acceptablerootwindowsize and root poll delay

* Improve test 'should fetch history correctly' for root cache

* Make root cache handling more efficient

* add contract root cache size as constant and function use fix

* updateRecentRoots comments update

* Update group_manager and tests

* fix linting

Author: stubbsta

tests/waku_rln_relay/test_rln_group_manager_onchain.nim

@@ -97,7 +97,8 @@ suite "Onchain group manager":
     check:
       merkleRootBefore != merkleRootAfter
 
-  test "trackRootChanges: should fetch history correctly":
+  test "trackRootChanges: should fetch history correctly: fetch single root()":
+    # basic check for the soon to be deprecated root contract function, is replaced by getRecentRoots()
     # TODO: We can't use `trackRootChanges()` directly in this test because its current implementation
     #       relies on a busy loop rather than event-based monitoring. but that busy loop fetch root every 5 seconds
     #       so we can't use it in this test. 
@@ -107,20 +108,92 @@ suite "Onchain group manager":
     (waitFor manager.init()).isOkOr:
       raiseAssert $error
 
-    let merkleRootBefore = waitFor manager.fetchMerkleRoot()
+    let merkleRootBefore = (waitFor manager.fetchMerkleRoot()).valueOr:
+      raiseAssert "Failed to fetch merkle root before: " & error
 
     for i in 0 ..< credentials.len():
       info "Registering credential", index = i, credential = credentials[i]
       (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
         assert false, "Failed to register credential " & $i & ": " & error
       discard waitFor manager.updateRoots()
 
-    let merkleRootAfter = waitFor manager.fetchMerkleRoot()
+    let merkleRootAfter = (waitFor manager.fetchMerkleRoot()).valueOr:
+      raiseAssert "Failed to fetch merkle root after: " & error
 
     check:
       merkleRootBefore != merkleRootAfter
       manager.validRoots.len() == credentialCount
 
+  test "trackRootChanges: should fetch history correctly: fetch root cache":
+    # Verify that the group_manager list of valid roots is updated correctly from the recent roots
+    # cache as new credentials are registered.
+    # TODO: We can't use `trackRootChanges()` directly in this test because its current implementation
+    #       relies on a busy loop rather than event-based monitoring. but that busy loop fetch root every 5 seconds
+    #       so we can't use it in this test.
+
+    const credentialCount = RlnContractRootCacheSize
+    let credentials = generateCredentials(credentialCount)
+    (waitFor manager.init()).isOkOr:
+      raiseAssert $error
+
+    let merkleRootCacheBefore = (waitFor manager.fetchMerkleRootsCache()).valueOr:
+      raiseAssert "Failed to fetch merkle root cache before: " & error
+
+    check:
+      merkleRootCacheBefore.len == RlnContractRootCacheSize * 32
+      merkleRootCacheBefore.allIt(it == 0'u8)
+      manager.validRoots.len() == 0
+
+    for i in 0 ..< credentials.len():
+      info "Registering credential", index = i, credential = credentials[i]
+      (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
+        assert false, "Failed to register credential " & $i & ": " & error
+      discard waitFor manager.updateRecentRoots()
+
+    let merkleRootCacheAfter = (waitFor manager.fetchMerkleRootsCache()).valueOr:
+      raiseAssert "Failed to fetch merkle root cache after: " & error
+
+    check:
+      merkleRootCacheAfter.len == RlnContractRootCacheSize * 32
+      not merkleRootCacheAfter.allIt(it == 0'u8)
+      manager.validRoots.len() == credentialCount
+      manager.validRoots.items().toSeq().allIt(it != default(MerkleNode))
+
+  test "trackRootChanges: oldest roots are evicted once the window is exceeded":
+    const
+      initialCount = AcceptableRootWindowSize - RlnContractRootCacheSize
+      additionalCount = RlnContractRootCacheSize + 1
+        # one more than the cache size to ensure eviction occurs
+    let credentials = generateCredentials(initialCount + additionalCount)
+    (waitFor manager.init()).isOkOr:
+      raiseAssert $error
+
+    # Register the first credentials and snapshot the 3 oldest roots.
+    for i in 0 ..< initialCount:
+      (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
+        assert false, "Failed to register credential " & $i & ": " & error
+      discard waitFor manager.updateRecentRoots()
+
+    check manager.validRoots.len() >= 3
+    let firstThreeBefore =
+      @[manager.validRoots[0], manager.validRoots[1], manager.validRoots[2]]
+
+    # Register the remaining credentials, pushing the deque past AcceptableRootWindowSize.
+    for i in initialCount ..< credentials.len():
+      (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
+        assert false, "Failed to register credential " & $i & ": " & error
+      discard waitFor manager.updateRecentRoots()
+
+    let rootsAfter = manager.validRoots.items().toSeq()
+
+    # AcceptableRootWindowSize + 1 registrations evicts exactly the single oldest root,
+    # so only the first of the original three is gone; the other two remain.
+    check:
+      manager.validRoots.len() == AcceptableRootWindowSize
+      firstThreeBefore[0] notin rootsAfter
+      firstThreeBefore[1] in rootsAfter
+      firstThreeBefore[2] in rootsAfter
+
   test "register: should guard against uninitialized state":
     let dummyCommitment = default(IDCommitment)
 
@@ -214,7 +287,7 @@ suite "Onchain group manager":
 
     waitFor fut
 
-    let rootUpdated = waitFor manager.updateRoots()
+    let rootUpdated = waitFor manager.updateRecentRoots()
 
     if rootUpdated:
       let proofResult = waitFor manager.fetchMerkleProofElements()
@@ -296,7 +369,7 @@ suite "Onchain group manager":
       assert false, "error returned when calling register: " & error
     waitFor fut
 
-    let rootUpdated = waitFor manager.updateRoots()
+    let rootUpdated = waitFor manager.updateRecentRoots()
 
     if rootUpdated:
       let proofResult = waitFor manager.fetchMerkleProofElements()
@@ -333,7 +406,7 @@ suite "Onchain group manager":
 
     let messageBytes = "Hello".toBytes()
 
-    let rootUpdated = waitFor manager.updateRoots()
+    let rootUpdated = waitFor manager.updateRecentRoots()
 
     manager.merkleProofCache = newSeq[byte](640)
     for i in 0 ..< 640:
@@ -362,7 +435,7 @@ suite "Onchain group manager":
       verified == false
 
   test "root queue should be updated correctly":
-    const credentialCount = 12
+    const credentialCount = 9
     let credentials = generateCredentials(credentialCount)
     (waitFor manager.init()).isOkOr:
       raiseAssert $error
@@ -391,7 +464,7 @@ suite "Onchain group manager":
     for i in 0 ..< credentials.len():
       (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
         assert false, "Failed to register credential " & $i & ": " & error
-      discard waitFor manager.updateRoots()
+      discard waitFor manager.updateRecentRoots()
 
     waitFor allFutures(futures)
 
@@ -436,7 +509,7 @@ suite "Onchain group manager":
     (waitFor manager.register(credentials, UserMessageLimit(20))).isOkOr:
       assert false, "register failed: " & error
 
-    discard waitFor manager.updateRoots()
+    discard waitFor manager.updateRecentRoots()
     let roots = manager.validRoots.items().toSeq()
     require:
       roots.len > 0

waku/waku_rln_relay/constants.nim

@@ -7,6 +7,9 @@ import ../waku_keystore
 # Acceptable roots for merkle root validation of incoming messages
 const AcceptableRootWindowSize* = 50
 
+#Size if RLN contract root cache
+const RlnContractRootCacheSize* = 5
+
 # RLN membership key and index files path
 const RlnCredentialsFilename* = "rlnCredentials.txt"
 

waku/waku_rln_relay/group_manager/on_chain/group_manager.nim

@@ -62,25 +62,43 @@ proc fetchMerkleProofElements*(
   let response = await sendEthCallWithParams(
     ethRpc = g.ethRpc.get(),
     functionSignature = methodSig,
-    params = paddedParam,
     fromAddress = g.ethRpc.get().defaultAccount,
     toAddress = fromHex(Address, g.ethContractAddress),
     chainId = g.chainId,
+    params = paddedParam,
   )
 
   return response
 
 proc fetchMerkleRoot*(
     g: OnchainGroupManager
 ): Future[Result[UInt256, string]] {.async.} =
-  let merkleRoot = await sendEthCallWithoutParams(
-    ethRpc = g.ethRpc.get(),
-    functionSignature = "root()",
-    fromAddress = g.ethRpc.get().defaultAccount,
-    toAddress = fromHex(Address, g.ethContractAddress),
-    chainId = g.chainId,
-  )
-  return merkleRoot
+  try:
+    let merkleRoot = await sendEthCallWithoutParams(
+      ethRpc = g.ethRpc.get(),
+      functionSignature = "root()",
+      fromAddress = g.ethRpc.get().defaultAccount,
+      toAddress = fromHex(Address, g.ethContractAddress),
+      chainId = g.chainId,
+    )
+    return merkleRoot
+  except CatchableError:
+    error "Failed to fetch Merkle root", error = getCurrentExceptionMsg()
+    return err("Failed to fetch merkle root: " & getCurrentExceptionMsg())
+
+proc fetchMerkleRootsCache*(
+    g: OnchainGroupManager
+): Future[Result[seq[byte], string]] {.async.} =
+  let
+    # using sendEthCallWithParams to get return type of seq[bytes] for getRecentRoots() function which returns an array of bytes32
+    merkleRoots = await sendEthCallWithParams(
+      ethRpc = g.ethRpc.get(),
+      functionSignature = "getRecentRoots()",
+      fromAddress = g.ethRpc.get().defaultAccount,
+      toAddress = fromHex(Address, g.ethContractAddress),
+      chainId = g.chainId,
+    )
+  return merkleRoots
 
 proc fetchNextFreeIndex*(
     g: OnchainGroupManager
@@ -102,10 +120,10 @@ proc fetchMembershipStatus*(
     await sendEthCallWithParams(
       ethRpc = g.ethRpc.get(),
       functionSignature = "isInMembershipSet(uint256)",
-      params = params,
       fromAddress = g.ethRpc.get().defaultAccount,
       toAddress = fromHex(Address, g.ethContractAddress),
       chainId = g.chainId,
+      params = params,
     )
   ).valueOr:
     return err("Failed to check membership: " & error)
@@ -148,14 +166,64 @@ proc updateRoots*(g: OnchainGroupManager): Future[bool] {.async.} =
 
   return false
 
+proc updateRecentRoots*(g: OnchainGroupManager): Future[bool] {.async.} =
+  ## Fetch recent roots from the contract roots cache and update the validRoots deque, ensuring we maintain a window of unique acceptable roots.
+  ## Contract returns array of uint256 roots, newest first, zero-padded to the cache size (e.g. 5).
+  let bytes = (await g.fetchMerkleRootsCache()).valueOr:
+    error "Failed to fetch current Merkle root", error = error
+    return false
+
+  if (bytes.len mod 32) != 0:
+    error "Invalid recent roots payload length", length = bytes.len
+    return false
+
+  let chunkCount = bytes.len div 32
+  if chunkCount != RlnContractRootCacheSize:
+    warn "Unexpected number of recent roots returned; proceeding anyway",
+      count = chunkCount
+
+  # Parse 32-byte chunks (contract returns newest-first) into MerkleNode values,
+  # reversing to oldest-first and skipping zero roots.
+  var newRootsDequeOrder: seq[MerkleNode] = @[]
+  for startIdx in countdown(bytes.len - 32, 0, 32):
+    let u = UInt256.fromBytesBE(bytes.toOpenArray(startIdx, startIdx + 31))
+    if u.isZero:
+      continue
+    newRootsDequeOrder.add(UInt256ToField(u))
+
+  if newRootsDequeOrder.len == 0:
+    debug "no non-zero recent roots to add; skipping update"
+    return false
+
+  # Determine overlap with existing tail so we only append truly new roots
+  let overlap = min(g.validRoots.len, newRootsDequeOrder.len)
+  var matchLen = 0
+  for startIdx in (g.validRoots.len - overlap) ..< g.validRoots.len:
+    if g.validRoots[startIdx] == newRootsDequeOrder[0]:
+      matchLen = g.validRoots.len - startIdx
+      break
+
+  let toAdd = newRootsDequeOrder[matchLen ..< newRootsDequeOrder.len]
+  if toAdd.len == 0:
+    return false
+
+  # Append new roots to the tail; trim happens below if we exceed the window.
+  for root in toAdd:
+    g.validRoots.addLast(root)
+  debug "appended recent roots to list of valid roots", count = toAdd.len, roots = toAdd
+
+  while g.validRoots.len > AcceptableRootWindowSize:
+    discard g.validRoots.popFirst()
+
+  return true
+
 proc trackRootChanges*(g: OnchainGroupManager): Future[Result[void, string]] {.async.} =
   ?checkInitialized(g)
 
-  const rpcDelay = 5.seconds
+  const rpcDelay = 10.seconds
 
   while true:
-    await sleepAsync(rpcDelay)
-    let rootUpdated = await g.updateRoots()
+    let rootUpdated = await g.updateRecentRoots()
 
     if rootUpdated:
       ## The membership set on-chain has changed (some new members have joined or some members have left)
@@ -174,6 +242,7 @@ proc trackRootChanges*(g: OnchainGroupManager): Future[Result[void, string]] {.a
 
       let memberCount = cast[int64](nextFreeIndex)
       waku_rln_number_registered_memberships.set(float64(memberCount))
+    await sleepAsync(rpcDelay)
 
 method register*(
     g: OnchainGroupManager, rateCommitment: RateCommitment
@@ -393,8 +462,11 @@ method generateProof*(
     external_nullifier: extNullifier,
   )
 
-  let output = generateRlnProofWithWitness(g.rlnInstance, witness, epoch, rlnIdentifier).valueOr:
-    return err("Failed to generate proof: " & error)
+  waku_rln_proof_generation_duration_seconds.nanosecondTime:
+    let output = generateRlnProofWithWitness(
+      g.rlnInstance, witness, epoch, rlnIdentifier
+    ).valueOr:
+      return err("Failed to generate proof: " & error)
 
   info "Proof generated successfully", proof = output
 

waku/waku_rln_relay/group_manager/on_chain/rpc_wrapper.nim

@@ -76,10 +76,10 @@ proc sendEthCallWithoutParams*(
 proc sendEthCallWithParams*(
     ethRpc: Web3,
     functionSignature: string,
-    params: seq[byte],
     fromAddress: Address,
     toAddress: Address,
     chainId: UInt256,
+    params: seq[byte] = @[],
 ): Future[Result[seq[byte], string]] {.async.} =
   ## Workaround for web3 chainId=null issue with parameterized contract calls
   let functionHash =