STR-1308: EVM state diff(snapshot) prototype (#800)

* Implement a prototype of EE batch state diff.

* Implement exex for EE state diff generation.

* Implement an RPC to fetch block state diff.

* Implement StateDiff traits for DB.

* Use state diff as an exex in alpen-reth, guarded by the config flag.

* Implement unit tests for state diff DB table.

* Add basic fntest for state diff.

* Fmt and other small fixes.

* Add unit tests for state diff.

* Introduce a separate BatchStateDiff builder for a cleaner interface.

* Factor our mpt into its own separate crate.

* Use strata-mpt in the evm-ee-stf.

* Basic interative state reconstruction from batch state diffs.

* Extend exex DB with number to hash mapping to facilitate easier testing.

* Expose an additional RPC that reconstructs and returns the block state root based off state diffs.

* Minor fixes

* Prettify structure, factor out pieces.

* Split out chainspec into a separate microcrate and use it in statediffs to construct the genesis state.

* Add fntests for state reconstructure from state diffs using load gen env.

* Minor review fixes.

* Fix chain configs path in the fntests.

Author: evgenyzdanovich

Cargo.lock

@@ -11,12 +11,14 @@ members = [
   "crates/evmexec",
   "crates/key-derivation",
   "crates/l1tx",
+  "crates/mpt",
   "crates/primitives",
   "crates/proof-impl/btc-blockspace",
   "crates/proof-impl/checkpoint",
   "crates/proof-impl/cl-stf",
   "crates/proof-impl/evm-ee-stf",
   "crates/zkvm/hosts",
+  "crates/reth/chainspec",
   "crates/reth/db",
   "crates/reth/evm",
   "crates/reth/exex",
@@ -65,12 +67,14 @@ default-members = [
 resolver = "2"
 
 [workspace.dependencies]
+alpen-chainspec = { path = "crates/reth/chainspec" }
 alpen-reth-db = { path = "crates/reth/db" }
 alpen-reth-evm = { path = "crates/reth/evm" }
 alpen-reth-exex = { path = "crates/reth/exex" }
 alpen-reth-node = { path = "crates/reth/node" }
 alpen-reth-primitives = { path = "crates/reth/primitives" }
 alpen-reth-rpc = { path = "crates/reth/rpc" }
+alpen-reth-statediff = { path = "crates/reth/statediff" }
 strata-btcio = { path = "crates/btcio" }
 strata-chaintsn = { path = "crates/chaintsn" }
 strata-common = { path = "crates/common" }
@@ -83,6 +87,7 @@ strata-evmexec = { path = "crates/evmexec" }
 strata-key-derivation = { path = "crates/key-derivation" }
 strata-l1tx = { path = "crates/l1tx" }
 strata-mmr = { path = "crates/util/mmr" }
+strata-mpt = { path = "crates/mpt" }
 strata-primitives = { path = "crates/primitives" }
 strata-proofimpl-btc-blockspace = { path = "crates/proof-impl/btc-blockspace" }
 strata-proofimpl-checkpoint = { path = "crates/proof-impl/checkpoint" }

Cargo.toml

@@ -8,31 +8,24 @@ name = "alpen-reth"
 path = "src/main.rs"
 
 [dependencies]
+alpen-chainspec.workspace = true
 alpen-reth-db.workspace = true
 alpen-reth-exex.workspace = true
 alpen-reth-node.workspace = true
 alpen-reth-rpc.workspace = true
 
-alloy-genesis.workspace = true
 alloy-rpc-types.workspace = true
 async-trait.workspace = true
 clap.workspace = true
 eyre.workspace = true
 hex.workspace = true
-jsonrpsee-types.workspace = true
-reqwest.workspace = true
 reth.workspace = true
 reth-chainspec.workspace = true
-reth-cli.workspace = true
 reth-cli-commands.workspace = true
 reth-cli-util.workspace = true
 reth-db.workspace = true
 reth-primitives.workspace = true
-reth-provider.workspace = true
-reth-rpc-eth-api.workspace = true
-reth-rpc-eth-types.workspace = true
 rockbound.workspace = true
 serde_json.workspace = true
-shellexpand = "3.0.0"
 thiserror.workspace = true
 tracing.workspace = true

bin/alpen-reth/Cargo.toml

@@ -1,10 +1,10 @@
 mod db;
 
-use std::{fs, future::Future, path::PathBuf, sync::Arc};
+use std::{future::Future, sync::Arc};
 
-use alloy_genesis::Genesis;
+use alpen_chainspec::{chain_value_parser, StrataChainSpecParser};
 use alpen_reth_db::rocksdb::WitnessDB;
-use alpen_reth_exex::ProverWitnessGenerator;
+use alpen_reth_exex::{ProverWitnessGenerator, StateDiffGenerator};
 use alpen_reth_node::{args::StrataNodeArgs, StrataEthereumNode};
 use alpen_reth_rpc::{StrataRPC, StrataRpcApiServer};
 use clap::Parser;
@@ -14,14 +14,9 @@ use reth::{
     CliRunner,
 };
 use reth_chainspec::ChainSpec;
-use reth_cli::chainspec::ChainSpecParser;
 use reth_cli_commands::node::NodeCommand;
 use tracing::info;
 
-const DEFAULT_CHAIN_SPEC: &str = include_str!("../res/testnet-chain.json");
-const DEVNET_CHAIN_SPEC: &str = include_str!("../res/devnet-chain.json");
-const DEV_CHAIN_SPEC: &str = include_str!("../res/alpen-dev-chain.json");
-
 fn main() {
     reth_cli_util::sigsegv_handler::install();
 
@@ -48,17 +43,27 @@ fn main() {
 
         let mut extend_rpc = None;
 
-        // Install Prover Input ExEx, persist to DB, and add RPC for querying block witness.
-        if ext.enable_witness_gen {
+        if ext.enable_witness_gen || ext.enable_state_diff_gen {
             let rbdb = db::open_rocksdb_database(datadir.clone()).expect("open rocksdb");
             let db = Arc::new(WitnessDB::new(rbdb));
-            let rpc_db = db.clone();
-
-            extend_rpc.replace(StrataRPC::new(rpc_db));
+            // Add RPC for querying block witness and state diffs.
+            extend_rpc.replace(StrataRPC::new(db.clone()));
+
+            // Install Prover Input ExEx and persist to DB
+            if ext.enable_witness_gen {
+                let witness_db = db.clone();
+                node_builder = node_builder.install_exex("prover_input", |ctx| async {
+                    Ok(ProverWitnessGenerator::new(ctx, witness_db).start())
+                });
+            }
 
-            node_builder = node_builder.install_exex("prover_input", |ctx| async {
-                Ok(ProverWitnessGenerator::new(ctx, db).start())
-            });
+            // Install State Diff ExEx and persist to DB
+            if ext.enable_state_diff_gen {
+                let state_diff_db = db.clone();
+                node_builder = node_builder.install_exex("state_diffs", |ctx| async {
+                    Ok(StateDiffGenerator::new(ctx, state_diff_db).start())
+                });
+            }
         }
 
         // Note: can only add single hook
@@ -100,59 +105,14 @@ pub struct AdditionalConfig {
     #[arg(long, default_value_t = false)]
     pub enable_witness_gen: bool,
 
+    #[arg(long, default_value_t = false)]
+    pub enable_state_diff_gen: bool,
+
     /// Rpc of sequener's reth node to forward transactions to.
     #[arg(long, required = false)]
     pub sequencer_http: Option<String>,
 }
 
-#[derive(Debug, Clone, Default)]
-#[non_exhaustive]
-pub struct StrataChainSpecParser;
-
-impl ChainSpecParser for StrataChainSpecParser {
-    type ChainSpec = ChainSpec;
-
-    const SUPPORTED_CHAINS: &'static [&'static str] = &["dev", "devnet", "testnet"];
-
-    fn parse(s: &str) -> eyre::Result<Arc<Self::ChainSpec>> {
-        chain_value_parser(s)
-    }
-}
-
-pub fn chain_value_parser(s: &str) -> eyre::Result<Arc<ChainSpec>, eyre::Error> {
-    Ok(match s {
-        "testnet" => parse_chain_spec(DEFAULT_CHAIN_SPEC)?,
-        "devnet" => parse_chain_spec(DEVNET_CHAIN_SPEC)?,
-        "dev" => parse_chain_spec(DEV_CHAIN_SPEC)?,
-        _ => {
-            // try to read json from path first
-            let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) {
-                Ok(raw) => raw,
-                Err(io_err) => {
-                    // valid json may start with "\n", but must contain "{"
-                    if s.contains('{') {
-                        s.to_string()
-                    } else {
-                        return Err(io_err.into()); // assume invalid path
-                    }
-                }
-            };
-
-            // both serialized Genesis and ChainSpec structs supported
-            let genesis: Genesis = serde_json::from_str(&raw)?;
-
-            Arc::new(genesis.into())
-        }
-    })
-}
-
-fn parse_chain_spec(chain_json: &str) -> eyre::Result<Arc<ChainSpec>> {
-    // both serialized Genesis and ChainSpec structs supported
-    let genesis: Genesis = serde_json::from_str(chain_json)?;
-
-    Ok(Arc::new(genesis.into()))
-}
-
 /// Run node with logging
 /// based on reth::cli::Cli::run
 fn run<L, Fut>(

bin/alpen-reth/src/main.rs

@@ -8,6 +8,7 @@ name = "strata-datatool"
 path = "src/main.rs"
 
 [dependencies]
+alpen-chainspec.workspace = true
 strata-key-derivation.workspace = true
 strata-primitives.workspace = true
 strata-risc0-guest-builder = { path = "../../provers/risc0", optional = true }

bin/datatool/Cargo.toml

@@ -48,7 +48,7 @@ const OPKEY_ENVVAR: &str = "STRATA_OP_KEY";
 const DEFAULT_NETWORK: Network = Network::Signet;
 
 /// The default evm chainspec to use in params.
-const DEFAULT_CHAIN_SPEC: &str = include_str!("../../alpen-reth/res/alpen-dev-chain.json");
+const DEFAULT_CHAIN_SPEC: &str = alpen_chainspec::DEV_CHAIN_SPEC;
 
 /// Resolves a [`Network`] from a string.
 pub(super) fn resolve_network(arg: Option<&str>) -> anyhow::Result<Network> {

bin/datatool/src/util.rs

@@ -0,0 +1,19 @@
+[package]
+description = "Implementation of the Merkle-Patricia Trie"
+edition = "2021"
+name = "strata-mpt"
+version = "0.1.0"
+
+[dependencies]
+alloy-rlp.workspace = true
+alloy-rlp-derive.workspace = true
+alloy-rpc-types-eth.workspace = true
+
+anyhow.workspace = true
+revm-primitives.workspace = true
+rlp = "0.5.2"
+serde.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+hex-literal = "0.4"

crates/mpt/Cargo.toml

@@ -6,6 +6,7 @@ version = "0.3.0-alpha.1"
 [dependencies]
 alpen-reth-evm.workspace = true
 alpen-reth-primitives.workspace = true
+strata-mpt.workspace = true
 strata-primitives.workspace = true
 strata-state.workspace = true
 zkaleido.workspace = true