Merge branch 'master' into add-rust-overlay

Author: vinhtc27

Cargo.lock

@@ -13,11 +13,11 @@ path = "src/examples/stateless.rs"
 required-features = ["stateless"]
 
 [dependencies]
-rln = { path = "../rln", default-features = false }
+rln = { path = "../rln", default-features = true }
 zerokit_utils = { path = "../utils" }
-clap = { version = "4.5.35", features = ["cargo", "derive", "env"] }
+clap = { version = "4.5.38", features = ["cargo", "derive", "env"] }
 clap_derive = { version = "4.5.32" }
-color-eyre = "0.6.3"
+color-eyre = "0.6.4"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 

rln-cli/Cargo.toml

@@ -10,10 +10,7 @@ required-features = ["stateless"]
 
 [dependencies]
 rln = { path = "../rln", default-features = false }
-num-bigint = { version = "0.4.6", default-features = false, features = [
-    "rand",
-    "serde",
-] }
+num-bigint = { version = "0.4.6", default-features = false }
 js-sys = "0.3.77"
 wasm-bindgen = "0.2.100"
 serde-wasm-bindgen = "0.6.5"
@@ -27,7 +24,7 @@ console_error_panic_hook = { version = "0.1.7", optional = true }
 zerokit_utils = { path = "../utils" }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-getrandom = { version = "0.2.15", features = ["js"] }
+getrandom = { version = "0.2.16", features = ["js"] }
 
 [dev-dependencies]
 serde_json = "1.0"

rln-wasm/Cargo.toml

@@ -40,16 +40,13 @@ ark-serialize = { version = "0.5.0", default-features = false, features = [
 ] }
 
 # error handling
-color-eyre = "0.6.3"
+color-eyre = "0.6.4"
 thiserror = "2.0.12"
 
 # utilities
 byteorder = "1.5.0"
 cfg-if = "1.0"
-num-bigint = { version = "0.4.6", default-features = false, features = [
-    "rand",
-    "std",
-] }
+num-bigint = { version = "0.4.6", default-features = false, features = ["std"] }
 num-traits = "0.2.19"
 once_cell = "1.21.3"
 lazy_static = "1.5.0"
@@ -67,7 +64,6 @@ serde = { version = "1.0", features = ["derive"] }
 document-features = { version = "0.2.11", optional = true }
 
 [dev-dependencies]
-sled = "0.34.7"
 criterion = { version = "0.4.0", features = ["html_reports"] }
 
 [features]

rln/Cargo.toml

@@ -26,12 +26,6 @@ pub fn pmtree_benchmark(c: &mut Criterion) {
         })
     });
 
-    c.bench_function("Pmtree::compute_root", |b| {
-        b.iter(|| {
-            tree.compute_root().unwrap();
-        })
-    });
-
     c.bench_function("Pmtree::get", |b| {
         b.iter(|| {
             tree.get(0).unwrap();

rln/benches/pmtree_benchmark.rs

@@ -168,10 +168,6 @@ impl ZerokitMerkleTree for PmTree {
         self.tree.root()
     }
 
-    fn compute_root(&mut self) -> Result<FrOf<Self::Hasher>> {
-        Ok(self.tree.root())
-    }
-
     fn set(&mut self, index: usize, leaf: FrOf<Self::Hasher>) -> Result<()> {
         self.tree
             .set(index, leaf)

rln/src/pm_tree_adapter.rs

@@ -15,15 +15,14 @@ bench = false
 ark-ff = { version = "0.5.0", default-features = false, features = [
     "parallel",
 ] }
-num-bigint = { version = "0.4.6", default-features = false, features = [
-    "rand",
-] }
-color-eyre = "0.6.3"
+num-bigint = { version = "0.4.6", default-features = false }
+color-eyre = "0.6.4"
 pmtree = { package = "vacp2p_pmtree", version = "2.0.2", optional = true }
 sled = "0.34.7"
 serde = "1.0"
 lazy_static = "1.5.0"
-hex = "0.4"
+hex = "0.4.3"
+rayon = "1.7.0"
 
 [dev-dependencies]
 ark-bn254 = { version = "0.5.0", features = ["std"] }

utils/Cargo.toml

@@ -20,20 +20,23 @@ The crate supports two interchangeable Merkle tree implementations:
 - **OptimalMerkleTree**
   - Only stores nodes used to prove accumulation of set leaves
 
+Both OptimalMerkleTree and FullMerkleTree use [Rayon](https://crates.io/crates/rayon) internally to speed up computation through data parallelism. This provides significant performance improvements, especially during large Merkle tree updates.
+
 ### Implementation notes
 
 Glossary:
 
-* depth: level of leaves if we count from levels from 0
-* number of levels: depth + 1
-* capacity (== number of leaves) -- 1 << depth
-* total number of nodes: 1 << (depth + 1)) - 1
+- depth: level of leaves if we count from levels from 0
+- number of levels: depth + 1
+- capacity (number of leaves): 1 << depth
+- total number of nodes: 1 << (depth + 1) - 1
 
 So for instance:
-* depth: 3
-* number of levels: 4
-* capacity (number of leaves): 8
-* total number of nodes: 15
+
+- depth: 3
+- number of levels: 4
+- capacity (number of leaves): 8
+- total number of nodes: 15
 
 ```mermaid
 flowchart TD
@@ -53,7 +56,6 @@ flowchart TD
     N6 -->|Leaf| L8
 ```
 
-
 ## Poseidon Hash Implementation
 
 This crate provides an implementation to compute the Poseidon hash round constants and MDS matrices:

utils/README.md

@@ -1,5 +1,4 @@
 use criterion::{criterion_group, criterion_main, Criterion};
-use hex_literal::hex;
 use lazy_static::lazy_static;
 use std::{fmt::Display, str::FromStr};
 use tiny_keccak::{Hasher as _, Keccak};
@@ -47,55 +46,78 @@ impl FromStr for TestFr {
 }
 
 lazy_static! {
-    static ref LEAVES: [TestFr; 4] = [
-        hex!("0000000000000000000000000000000000000000000000000000000000000001"),
-        hex!("0000000000000000000000000000000000000000000000000000000000000002"),
-        hex!("0000000000000000000000000000000000000000000000000000000000000003"),
-        hex!("0000000000000000000000000000000000000000000000000000000000000004"),
-    ]
-    .map(TestFr);
+    static ref LEAVES: Vec<TestFr> = {
+        let mut leaves = Vec::with_capacity(1 << 20);
+        for i in 0..(1 << 20) {
+            let mut bytes = [0u8; 32];
+            bytes[28..].copy_from_slice(&(i as u32).to_be_bytes());
+            leaves.push(TestFr(bytes));
+        }
+        leaves
+    };
+    static ref INDICES: Vec<usize> = (0..(1 << 20)).collect();
 }
 
+const NOF_LEAVES: usize = 8192;
+
 pub fn optimal_merkle_tree_benchmark(c: &mut Criterion) {
     let mut tree =
-        OptimalMerkleTree::<Keccak256>::new(2, TestFr([0; 32]), OptimalMerkleConfig::default())
+        OptimalMerkleTree::<Keccak256>::new(20, TestFr([0; 32]), OptimalMerkleConfig::default())
             .unwrap();
 
+    for i in 0..NOF_LEAVES {
+        tree.set(i, LEAVES[i % LEAVES.len()]).unwrap();
+    }
+
     c.bench_function("OptimalMerkleTree::set", |b| {
+        let mut index = NOF_LEAVES;
         b.iter(|| {
-            tree.set(0, LEAVES[0]).unwrap();
+            tree.set(index % (1 << 20), LEAVES[index % LEAVES.len()])
+                .unwrap();
+            index = (index + 1) % (1 << 20);
         })
     });
 
     c.bench_function("OptimalMerkleTree::delete", |b| {
+        let mut index = 0;
         b.iter(|| {
-            tree.delete(0).unwrap();
-        })
-    });
-
-    c.bench_function("OptimalMerkleTree::override_range", |b| {
-        b.iter(|| {
-            tree.override_range(0, LEAVES.into_iter(), [0, 1, 2, 3].into_iter())
+            tree.delete(index % NOF_LEAVES).unwrap();
+            tree.set(index % NOF_LEAVES, LEAVES[index % LEAVES.len()])
                 .unwrap();
+            index = (index + 1) % NOF_LEAVES;
         })
     });
 
-    c.bench_function("OptimalMerkleTree::compute_root", |b| {
+    c.bench_function("OptimalMerkleTree::override_range", |b| {
+        let mut offset = 0;
         b.iter(|| {
-            tree.compute_root().unwrap();
+            let range = offset..offset + NOF_LEAVES;
+            tree.override_range(
+                offset,
+                LEAVES[range.clone()].iter().cloned(),
+                INDICES[range.clone()].iter().cloned(),
+            )
+            .unwrap();
+            offset = (offset + NOF_LEAVES) % (1 << 20);
         })
     });
 
     c.bench_function("OptimalMerkleTree::get", |b| {
+        let mut index = 0;
         b.iter(|| {
-            tree.get(0).unwrap();
+            tree.get(index % NOF_LEAVES).unwrap();
+            index = (index + 1) % NOF_LEAVES;
         })
     });
 
-    // check intermediate node getter which required additional computation of sub root index
     c.bench_function("OptimalMerkleTree::get_subtree_root", |b| {
+        let mut level = 1;
+        let mut index = 0;
         b.iter(|| {
-            tree.get_subtree_root(1, 0).unwrap();
+            tree.get_subtree_root(level % 20, index % (1 << (20 - (level % 20))))
+                .unwrap();
+            index = (index + 1) % (1 << (20 - (level % 20)));
+            level = 1 + (level % 20);
         })
     });
 
@@ -108,43 +130,61 @@ pub fn optimal_merkle_tree_benchmark(c: &mut Criterion) {
 
 pub fn full_merkle_tree_benchmark(c: &mut Criterion) {
     let mut tree =
-        FullMerkleTree::<Keccak256>::new(2, TestFr([0; 32]), FullMerkleConfig::default()).unwrap();
+        FullMerkleTree::<Keccak256>::new(20, TestFr([0; 32]), FullMerkleConfig::default()).unwrap();
+
+    for i in 0..NOF_LEAVES {
+        tree.set(i, LEAVES[i % LEAVES.len()]).unwrap();
+    }
 
     c.bench_function("FullMerkleTree::set", |b| {
+        let mut index = NOF_LEAVES;
         b.iter(|| {
-            tree.set(0, LEAVES[0]).unwrap();
+            tree.set(index % (1 << 20), LEAVES[index % LEAVES.len()])
+                .unwrap();
+            index = (index + 1) % (1 << 20);
         })
     });
 
     c.bench_function("FullMerkleTree::delete", |b| {
+        let mut index = 0;
         b.iter(|| {
-            tree.delete(0).unwrap();
-        })
-    });
-
-    c.bench_function("FullMerkleTree::override_range", |b| {
-        b.iter(|| {
-            tree.override_range(0, LEAVES.into_iter(), [0, 1, 2, 3].into_iter())
+            tree.delete(index % NOF_LEAVES).unwrap();
+            tree.set(index % NOF_LEAVES, LEAVES[index % LEAVES.len()])
                 .unwrap();
+            index = (index + 1) % NOF_LEAVES;
         })
     });
 
-    c.bench_function("FullMerkleTree::compute_root", |b| {
+    c.bench_function("FullMerkleTree::override_range", |b| {
+        let mut offset = 0;
         b.iter(|| {
-            tree.compute_root().unwrap();
+            let range = offset..offset + NOF_LEAVES;
+            tree.override_range(
+                offset,
+                LEAVES[range.clone()].iter().cloned(),
+                INDICES[range.clone()].iter().cloned(),
+            )
+            .unwrap();
+            offset = (offset + NOF_LEAVES) % (1 << 20);
         })
     });
 
     c.bench_function("FullMerkleTree::get", |b| {
+        let mut index = 0;
         b.iter(|| {
-            tree.get(0).unwrap();
+            tree.get(index % NOF_LEAVES).unwrap();
+            index = (index + 1) % NOF_LEAVES;
         })
     });
 
-    // check intermediate node getter which required additional computation of sub root index
     c.bench_function("FullMerkleTree::get_subtree_root", |b| {
+        let mut level = 1;
+        let mut index = 0;
         b.iter(|| {
-            tree.get_subtree_root(1, 0).unwrap();
+            tree.get_subtree_root(level % 20, index % (1 << (20 - (level % 20))))
+                .unwrap();
+            index = (index + 1) % (1 << (20 - (level % 20)));
+            level = 1 + (level % 20);
         })
     });