11.3 vibes

Author: SozinM

Cargo.lock

@@ -70,62 +70,73 @@ codegen-units = 1
 incremental = false
 
 [workspace.dependencies]
-reth = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-db-common = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646", features = [
+# default-features minus "rocksdb": we only read reth datadirs via MDBX + static files.
+# ("reth-revm/portable" from reth's default set is re-enabled on the reth-revm dep below.)
+reth = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b", default-features = false, features = [
+    "jemalloc",
+    "otlp",
+    "otlp-logs",
+    "js-tracer",
+    "keccak-cache-global",
+    "asm-keccak",
+    "min-debug-logs",
+] }
+reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-db-common = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b", features = [
     "test-utils",
 ] }
-reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-
-# compatible with reth 27a8c0f5a6dfb27dea84c5751776ecabdd069646 dependencies
-revm = { version = "31.0.2", features = [
+reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b", features = ["portable"] }
+reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-tasks = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+
+# compatible with reth v1.11.3 (d6324d63e27ef6b7c49cdc9b1977c1b808234c7b) dependencies
+revm = { version = "34.0.0", features = [
     "std",
     "secp256k1",
     "optional_balance_check",
 ], default-features = false }
-revm-inspectors = { version = "0.32.0", default-features = false }
+revm-inspectors = { version = "0.34.2", default-features = false }
 
 ethereum_ssz_derive = "0.9.0"
 ethereum_ssz = "0.9.0"
 
-alloy-primitives = { version = "1.4.1", default-features = false, features = [
+alloy-primitives = { version = "1.5.6", default-features = false, features = [
     "getrandom",
 ] }
-alloy-rlp = "0.3.10"
+alloy-rlp = "0.3.13"
 alloy-chains = "0.2.5"
-alloy-trie = { version = "0.8.1", default-features = false }
-alloy-evm = { version = "0.23.3", default-features = false }
-alloy-provider = { version = "1.0.41", features = ["ipc", "pubsub", "ws"] }
-alloy-eips = { version = "1.0.41" }
-alloy-rpc-types = { version = "1.0.41" }
-alloy-json-rpc = { version = "1.0.41" }
-alloy-network = { version = "1.0.41" }
-alloy-network-primitives = { version = "1.0.41" }
-alloy-node-bindings = { version = "1.0.41" }
-alloy-consensus = { version = "1.0.41", features = ["kzg"] }
-alloy-rpc-types-beacon = { version = "1.0.41", features = ["ssz"] }
-alloy-rpc-types-engine = { version = "1.0.41", features = ["ssz"] }
-alloy-rpc-types-eth = { version = "1.0.41" }
-alloy-signer = { version = "1.0.41" }
-alloy-signer-local = { version = "1.0.41" }
+alloy-trie = { version = "0.9.4", default-features = false }
+alloy-evm = { version = "0.27.2", default-features = false }
+alloy-provider = { version = "1.6.3", features = ["ipc", "pubsub", "ws"] }
+alloy-eips = { version = "1.6.3" }
+alloy-rpc-types = { version = "1.6.3" }
+alloy-json-rpc = { version = "1.6.3" }
+alloy-network = { version = "1.6.3" }
+alloy-network-primitives = { version = "1.6.3" }
+alloy-node-bindings = { version = "1.6.3" }
+alloy-consensus = { version = "1.6.3", features = ["kzg"] }
+alloy-rpc-types-beacon = { version = "1.6.3", features = ["ssz"] }
+alloy-rpc-types-engine = { version = "1.6.3", features = ["ssz"] }
+alloy-rpc-types-eth = { version = "1.6.3" }
+alloy-signer = { version = "1.6.3" }
+alloy-signer-local = { version = "1.6.3" }
 
 # Version required by ethereum-consensus beacon-api-client
 mev-share-sse = { git = "https://github.com/paradigmxyz/mev-share-rs", rev = "9eb2b0138ab3202b9eb3af4b19c7b3bf40b0faa8", default-features = false }
@@ -141,7 +152,7 @@ parking_lot = { version = "0.12.3" }
 tokio = { version = "1.40.0", default-features = false }
 futures = "0.3"
 futures-util = "0.3"
-reqwest = { version = "0.12.8" }
+reqwest = { version = "0.12.8", features = ["json"] }
 serde = { version = "1.0.210" }
 serde_json = { version = "1.0" }
 serde_with = { version = "3.8.1" }

Cargo.toml

@@ -17,7 +17,7 @@ rustc-hash = "2.0.0"
 rayon = "1.10.0"
 smallvec = "1.13.2"
 alloy-trie.workspace = true
-nybbles = { version = "0.3.3", features = ["serde"] }
+nybbles = { version = "0.4", features = ["serde"] }
 
 tracing.workspace = true
 
@@ -32,6 +32,7 @@ reth-node-api = { workspace = true, optional = true }
 reth-node-ethereum = { workspace = true, optional = true }
 reth-trie.workspace = true
 reth-trie-parallel = { workspace = true, optional = true }
+reth-tasks = { workspace = true, optional = true }
 reth-trie-db.workspace = true
 reth-provider.workspace = true
 reth-revm = { workspace = true, optional = true }
@@ -64,6 +65,7 @@ dev-tools = [
     "dep:reth-node-api",
     "dep:reth-node-ethereum",
     "dep:reth-trie-parallel",
+    "dep:reth-tasks",
     "dep:reth-revm",
 ]
 

crates/eth-sparse-mpt/Cargo.toml

@@ -92,14 +92,14 @@ fn insert_proofs(c: &mut Criterion) {
     let mut proofs: HashMap<Nibbles, Vec<(Nibbles, Bytes)>> = Default::default();
     for key in &byte_keys {
         let key = Nibbles::unpack(key);
-        let current_key_proofs = proofs.entry(key.clone()).or_default();
+        let current_key_proofs = proofs.entry(key).or_default();
         for (path, node) in &nodes {
             if key.starts_with(path) {
-                current_key_proofs.push((path.clone(), node.clone().into()));
+                current_key_proofs.push((*path, node.clone().into()));
             }
         }
-        current_key_proofs.sort_by_key(|(p, _)| p.clone());
-        current_key_proofs.dedup_by_key(|(p, _)| p.clone());
+        current_key_proofs.sort_by_key(|(p, _)| *p);
+        current_key_proofs.dedup_by_key(|(p, _)| *p);
     }
     for (path, proof) in proofs {
         proof_store.add_proof(path, proof).unwrap();

crates/eth-sparse-mpt/benches/trie_do_bench.rs

@@ -120,8 +120,8 @@ fn gather_nodes(c: &mut Criterion) {
             .into_iter()
             .flat_map(|mp| mp.account_subtree.into_iter().collect::<Vec<_>>())
             .collect();
-        account_proof.sort_by_key(|(p, _)| p.clone());
-        account_proof.dedup_by_key(|(p, _)| p.clone());
+        account_proof.sort_by_key(|(p, _)| *p);
+        account_proof.dedup_by_key(|(p, _)| *p);
         account_proof
     };
 

crates/eth-sparse-mpt/benches/trie_insert_bench.rs

@@ -14,12 +14,16 @@ use reth_evm_ethereum::EthEvmConfig;
 use reth_node_api::NodeTypesWithDBAdapter;
 use reth_node_ethereum::EthereumNode;
 use reth_provider::{
-    providers::{ConsistentDbView, OverlayStateProviderFactory, StaticFileProvider},
+    providers::{
+        ConsistentDbView, OverlayStateProviderFactory, RocksDBProvider, StaticFileProvider,
+        StaticFileProviderBuilder,
+    },
     BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, HeaderProvider,
     ProviderFactory, TransactionVariant,
 };
 use reth_revm::database::StateProviderDatabase;
 use reth_trie::TrieInput;
+use reth_trie_db::ChangesetCache;
 use reth_trie_parallel::root::ParallelStateRoot;
 use revm::database::BundleState;
 use std::{
@@ -347,13 +351,13 @@ fn calculate_reth_root(
     parent_hash: B256,
     outcome: &BundleState,
 ) -> Result<B256> {
-    let overlay = OverlayStateProviderFactory::new(factory.clone());
+    let overlay = OverlayStateProviderFactory::new(factory.clone(), ChangesetCache::new());
     let hasher = factory
         .history_by_block_hash(parent_hash)
         .with_context(|| format!("failed to open state provider at parent hash {parent_hash:?}"))?;
     let hashed_post_state = hasher.hashed_post_state(outcome);
     let trie_input = TrieInput::from_state(hashed_post_state);
-    ParallelStateRoot::new(overlay, trie_input.prefix_sets.freeze())
+    ParallelStateRoot::new(overlay, trie_input.prefix_sets.freeze(), task_runtime())
         .incremental_root()
         .with_context(|| "parallel state root failed")
 }
@@ -403,24 +407,46 @@ fn open_provider_factory(
         .with_context(|| format!("failed to open reth db at {}", db_path.display()))?,
     );
 
-    let static_files =
-        StaticFileProvider::read_only(&static_files_path, false).with_context(|| {
-            format!(
-                "failed to open static files at {}",
-                static_files_path.display()
-            )
-        })?;
     let static_files = if let Some(blocks_per_file) = static_file_blocks_per_file {
-        static_files.with_custom_blocks_per_file(blocks_per_file)
+        StaticFileProviderBuilder::read_only(&static_files_path)
+            .with_blocks_per_file(blocks_per_file)
+            .build()
     } else {
-        static_files
-    };
+        StaticFileProvider::read_only(&static_files_path, false)
+    }
+    .with_context(|| {
+        format!(
+            "failed to open static files at {}",
+            static_files_path.display()
+        )
+    })?;
 
+    let rocksdb_provider = RocksDBProvider::builder(datadir.join("rocksdb"))
+        .with_default_tables()
+        .with_read_only(true)
+        .build()
+        .context("failed to open rocksdb provider")?;
     Ok(ProviderFactory::<NodeTypes>::new(
         db,
         chain_spec,
         static_files,
-    ))
+        rocksdb_provider,
+        task_runtime(),
+    )?)
+}
+
+/// Process-wide reth task runtime for parallel provider/trie I/O.
+fn task_runtime() -> reth_tasks::Runtime {
+    use reth_tasks::{Runtime, RuntimeBuilder, RuntimeConfig};
+
+    static RUNTIME: std::sync::OnceLock<Runtime> = std::sync::OnceLock::new();
+    RUNTIME
+        .get_or_init(|| {
+            RuntimeBuilder::new(RuntimeConfig::default())
+                .build()
+                .expect("failed to build reth task runtime")
+        })
+        .clone()
 }
 
 fn parse_chain_spec(chain: &str) -> Result<Arc<ChainSpec>> {

crates/eth-sparse-mpt/src/bin/correctness-harness.rs

@@ -29,25 +29,23 @@ pub fn rlp_pointer(rlp_encode: Bytes) -> Bytes {
 }
 
 pub fn concat_path(p1: &Nibbles, p2: &[u8]) -> Nibbles {
-    let mut result = Nibbles::with_capacity(p1.len() + p2.len());
-    result.extend_from_slice_unchecked(p1);
+    let mut result = *p1;
     result.extend_from_slice_unchecked(p2);
     result
 }
 
 pub fn strip_first_nibble_mut(p: &mut Nibbles) -> u8 {
-    let nibble = p[0];
-    let vec = p.as_mut_vec_unchecked();
-    vec.remove(0);
+    let nibble = p.get_unchecked(0);
+    *p = p.slice_unchecked(1, p.len());
     nibble
 }
 
 #[inline]
 pub fn extract_prefix_and_suffix(p1: &Nibbles, p2: &Nibbles) -> (Nibbles, Nibbles, Nibbles) {
     let prefix_len = p1.common_prefix_length(p2);
-    let prefix = Nibbles::from_nibbles_unchecked(&p1[..prefix_len]);
-    let suffix1 = Nibbles::from_nibbles_unchecked(&p1[prefix_len..]);
-    let suffix2 = Nibbles::from_nibbles_unchecked(&p2[prefix_len..]);
+    let prefix = p1.slice_unchecked(0, prefix_len);
+    let suffix1 = p1.slice_unchecked(prefix_len, p1.len());
+    let suffix2 = p2.slice_unchecked(prefix_len, p2.len());
 
     (prefix, suffix1, suffix2)
 }
@@ -136,16 +134,3 @@ fn mismatch_chunks<const N: usize>(xs: &[u8], ys: &[u8]) -> usize {
         .take_while(|(x, y)| x == y)
         .count()
 }
-
-// rbuilder uses nybbles v3.3.0 and reth_trie uses nybbles v4.3.0. This is a temporary fix to convert between the two.
-// We can remove the below methods once rbuilder has been upgraded to nybbles v4.3.0.
-// nybbles v4.3.0 has a breaking change (byte array with 1 byte per nybble vs U256 packed data + length) in the API which breaks a lot of parts of the eth sparse trie code.
-#[inline]
-pub fn convert_reth_nybbles_to_nibbles(n: reth_trie::Nibbles) -> Nibbles {
-    Nibbles::from_nibbles(n.to_vec())
-}
-
-#[inline]
-pub fn convert_nibbles_to_reth_nybbles(n: Nibbles) -> reth_trie::Nibbles {
-    reth_trie::Nibbles::from_nibbles(n.as_slice())
-}

crates/eth-sparse-mpt/src/utils.rs

@@ -146,11 +146,11 @@ impl RethSparseTrieShareCacheInternal {
         multiproof: MultiProof,
     ) -> Result<(), AddNodeError> {
         let mut nodes: Vec<_> = multiproof.account_subtree.into_iter().collect();
-        nodes.sort_by_key(|(p, _)| p.clone());
+        nodes.sort_by_key(|(p, _)| *p);
         self.account_trie.add_nodes(&nodes)?;
         for (account, storge_proofs) in multiproof.storages {
             let mut nodes: Vec<_> = storge_proofs.subtree.into_iter().collect();
-            nodes.sort_by_key(|(p, _)| p.clone());
+            nodes.sort_by_key(|(p, _)| *p);
             let account = Bytes::copy_from_slice(account.as_slice());
             let storage_trie = self.storage_tries.entry(account).or_default();
             storage_trie.add_nodes(&nodes)?;

crates/eth-sparse-mpt/src/v1/reth_sparse_trie/shared_cache.rs

@@ -1,6 +1,6 @@
 use std::time::Instant;
 
-use crate::utils::{convert_reth_nybbles_to_nibbles, hash_map_with_capacity, HashMap, HashSet};
+use crate::utils::{hash_map_with_capacity, HashMap, HashSet};
 use alloy_primitives::map::HashSet as AlloyHashSet;
 use tracing::trace;
 
@@ -106,7 +106,9 @@ where
 }
 
 fn pad_path(mut path: Nibbles) -> B256 {
-    path.as_mut_vec_unchecked().resize(64, 0);
+    while path.len() < 64 {
+        path.push_unchecked(0);
+    }
     let mut res = B256::default();
     path.pack_to(res.as_mut_slice());
     res
@@ -164,16 +166,16 @@ fn merge_results(
     let mut result = MultiProof::default();
     for mut proof in multiproofs {
         result.account_subtree.append(&mut proof.account_subtree);
-        result.account_subtree.sort_by_key(|s| s.0.clone());
-        result.account_subtree.dedup_by_key(|s| s.0.clone());
+        result.account_subtree.sort_by_key(|s| s.0);
+        result.account_subtree.dedup_by_key(|s| s.0);
 
         for (account, mut storage_proof) in proof.storages {
             let result_storage_proof = result.storages.entry(account).or_default();
             result_storage_proof
                 .subtree
                 .append(&mut storage_proof.subtree);
-            result_storage_proof.subtree.sort_by_key(|s| s.0.clone());
-            result_storage_proof.subtree.dedup_by_key(|s| s.0.clone());
+            result_storage_proof.subtree.sort_by_key(|s| s.0);
+            result_storage_proof.subtree.dedup_by_key(|s| s.0);
         }
     }
 
@@ -192,9 +194,9 @@ fn convert_reth_multiproof(
 ) -> MultiProof {
     let mut account_subtree = Vec::with_capacity(reth_proof.account_subtree.len());
     for (k, v) in reth_proof.account_subtree.into_inner() {
-        account_subtree.push((convert_reth_nybbles_to_nibbles(k), v));
+        account_subtree.push((k, v));
     }
-    account_subtree.sort_by_key(|a| a.0.clone());
+    account_subtree.sort_by_key(|a| a.0);
     let mut storages = hash_map_with_capacity(reth_proof.storages.len());
     for (k, reth_storage_proof) in reth_proof.storages {
         if !all_requested_accounts.contains(&k) {
@@ -206,9 +208,9 @@ fn convert_reth_multiproof(
         let mut subtree = Vec::with_capacity(reth_storage_proof.subtree.len());
 
         for (k, v) in reth_storage_proof.subtree.into_inner() {
-            subtree.push((convert_reth_nybbles_to_nibbles(k), v));
+            subtree.push((k, v));
         }
-        subtree.sort_by_key(|a| a.0.clone());
+        subtree.sort_by_key(|a| a.0);
         let v = StorageMultiProof { subtree };
         storages.insert(k, v);
     }