Merge branch 'main' into new-approach3

Author: Rimeeeeee

Cargo.lock

@@ -3,6 +3,7 @@ use clap::Parser;
 use reth_db::{
     static_file::{
         ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask,
+        TransactionSenderMask,
     },
     RawDupSort,
 };
@@ -75,6 +76,10 @@ impl Command {
                     StaticFileSegment::Receipts => {
                         (table_key::<tables::Receipts>(&key)?, <ReceiptMask<ReceiptTy<N>>>::MASK)
                     }
+                    StaticFileSegment::TransactionSenders => (
+                        table_key::<tables::TransactionSenders>(&key)?,
+                        <TransactionSenderMask>::MASK,
+                    ),
                 };
 
                 let content = tool
@@ -114,6 +119,13 @@ impl Command {
                                     )?;
                                     println!("{}", serde_json::to_string_pretty(&receipt)?);
                                 }
+                                StaticFileSegment::TransactionSenders => {
+                                    let sender =
+                                        <<tables::TransactionSenders as Table>::Value>::decompress(
+                                            content[0].as_slice(),
+                                        )?;
+                                    println!("{}", serde_json::to_string_pretty(&sender)?);
+                                }
                             }
                         }
                     }

crates/cli/commands/src/db/get.rs

@@ -44,6 +44,11 @@ pub enum SetCommand {
         #[clap(action(ArgAction::Set))]
         value: bool,
     },
+    /// Store transaction senders in static files instead of the database
+    TransactionSendersInStaticFiles {
+        #[clap(action(ArgAction::Set))]
+        value: bool,
+    },
 }
 
 impl Command {
@@ -83,8 +88,10 @@ impl Command {
             println!("No storage settings found, creating new settings.");
         }
 
-        let mut settings @ StorageSettings { receipts_in_static_files: _ } =
-            settings.unwrap_or_default();
+        let mut settings @ StorageSettings {
+            receipts_in_static_files: _,
+            transaction_senders_in_static_files: _,
+        } = settings.unwrap_or_else(StorageSettings::legacy);
 
         // Update the setting based on the key
         match cmd {
@@ -96,6 +103,14 @@ impl Command {
                 settings.receipts_in_static_files = value;
                 println!("Set receipts_in_static_files = {}", value);
             }
+            SetCommand::TransactionSendersInStaticFiles { value } => {
+                if settings.transaction_senders_in_static_files == value {
+                    println!("transaction_senders_in_static_files is already set to {}", value);
+                    return Ok(());
+                }
+                settings.transaction_senders_in_static_files = value;
+                println!("Set transaction_senders_in_static_files = {}", value);
+            }
         }
 
         // Write updated settings

crates/cli/commands/src/db/settings.rs

@@ -45,6 +45,7 @@ impl<C: ChainSpecParser> Command<C> {
             StageEnum::Headers => Some(StaticFileSegment::Headers),
             StageEnum::Bodies => Some(StaticFileSegment::Transactions),
             StageEnum::Execution => Some(StaticFileSegment::Receipts),
+            StageEnum::Senders => Some(StaticFileSegment::TransactionSenders),
             _ => None,
         };
 
@@ -68,17 +69,24 @@ impl<C: ChainSpecParser> Command<C> {
                     StaticFileSegment::Transactions => {
                         let to_delete = static_file_provider
                             .get_highest_static_file_tx(static_file_segment)
-                            .map(|tx| tx + 1)
+                            .map(|tx_num| tx_num + 1)
                             .unwrap_or_default();
                         writer.prune_transactions(to_delete, 0)?;
                     }
                     StaticFileSegment::Receipts => {
                         let to_delete = static_file_provider
                             .get_highest_static_file_tx(static_file_segment)
-                            .map(|receipt| receipt + 1)
+                            .map(|tx_num| tx_num + 1)
                             .unwrap_or_default();
                         writer.prune_receipts(to_delete, 0)?;
                     }
+                    StaticFileSegment::TransactionSenders => {
+                        let to_delete = static_file_provider
+                            .get_highest_static_file_tx(static_file_segment)
+                            .map(|tx_num| tx_num + 1)
+                            .unwrap_or_default();
+                        writer.prune_transaction_senders(to_delete, 0)?;
+                    }
                 }
             }
         }

crates/cli/commands/src/stage/drop.rs

@@ -436,14 +436,17 @@ pub struct BlocksPerFileConfig {
     pub transactions: Option<u64>,
     /// Number of blocks per file for the receipts segment.
     pub receipts: Option<u64>,
+    /// Number of blocks per file for the transaction senders segment.
+    pub transaction_senders: Option<u64>,
 }
 
 impl StaticFilesConfig {
     /// Validates the static files configuration.
     ///
     /// Returns an error if any blocks per file value is zero.
     pub fn validate(&self) -> eyre::Result<()> {
-        let BlocksPerFileConfig { headers, transactions, receipts } = self.blocks_per_file;
+        let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
+            self.blocks_per_file;
         eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
         eyre::ensure!(
             transactions != Some(0),
@@ -453,12 +456,17 @@ impl StaticFilesConfig {
             receipts != Some(0),
             "Receipts segment blocks per file must be greater than 0"
         );
+        eyre::ensure!(
+            transaction_senders != Some(0),
+            "Transaction senders segment blocks per file must be greater than 0"
+        );
         Ok(())
     }
 
     /// Converts the blocks per file configuration into a [`HashMap`] per segment.
     pub fn as_blocks_per_file_map(&self) -> HashMap<StaticFileSegment, u64> {
-        let BlocksPerFileConfig { headers, transactions, receipts } = self.blocks_per_file;
+        let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
+            self.blocks_per_file;
 
         let mut map = HashMap::new();
         // Iterating over all possible segments allows us to do an exhaustive match here,
@@ -468,6 +476,7 @@ impl StaticFilesConfig {
                 StaticFileSegment::Headers => headers,
                 StaticFileSegment::Transactions => transactions,
                 StaticFileSegment::Receipts => receipts,
+                StaticFileSegment::TransactionSenders => transaction_senders,
             };
 
             if let Some(blocks_per_file) = blocks_per_file {

crates/config/src/config.rs

@@ -15,6 +15,7 @@ reth-engine-primitives = { workspace = true, features = ["std"] }
 reth-ethereum-engine-primitives.workspace = true
 reth-payload-builder.workspace = true
 reth-payload-primitives.workspace = true
+reth-primitives-traits.workspace = true
 reth-storage-api.workspace = true
 reth-transaction-pool.workspace = true
 
@@ -43,4 +44,5 @@ op = [
     "dep:op-alloy-rpc-types-engine",
     "dep:reth-optimism-chainspec",
     "reth-payload-primitives/op",
+    "reth-primitives-traits/op",
 ]

crates/engine/local/Cargo.toml

@@ -1,6 +1,5 @@
 //! Contains the implementation of the mining mode for the local engine.
 
-use alloy_consensus::BlockHeader;
 use alloy_primitives::{TxHash, B256};
 use alloy_rpc_types_engine::ForkchoiceState;
 use eyre::OptionExt;
@@ -10,14 +9,15 @@ use reth_payload_builder::PayloadBuilderHandle;
 use reth_payload_primitives::{
     BuiltPayload, EngineApiMessageVersion, PayloadAttributesBuilder, PayloadKind, PayloadTypes,
 };
+use reth_primitives_traits::{HeaderTy, SealedHeaderFor};
 use reth_storage_api::BlockReader;
 use reth_transaction_pool::TransactionPool;
 use std::{
     collections::VecDeque,
     future::Future,
     pin::Pin,
     task::{Context, Poll},
-    time::{Duration, UNIX_EPOCH},
+    time::Duration,
 };
 use tokio::time::Interval;
 use tokio_stream::wrappers::ReceiverStream;
@@ -106,36 +106,39 @@ pub struct LocalMiner<T: PayloadTypes, B, Pool: TransactionPool + Unpin> {
     mode: MiningMode<Pool>,
     /// The payload builder for the engine
     payload_builder: PayloadBuilderHandle<T>,
-    /// Timestamp for the next block.
-    last_timestamp: u64,
+    /// Latest block in the chain so far.
+    last_header: SealedHeaderFor<<T::BuiltPayload as BuiltPayload>::Primitives>,
     /// Stores latest mined blocks.
     last_block_hashes: VecDeque<B256>,
 }
 
 impl<T, B, Pool> LocalMiner<T, B, Pool>
 where
     T: PayloadTypes,
-    B: PayloadAttributesBuilder<<T as PayloadTypes>::PayloadAttributes>,
+    B: PayloadAttributesBuilder<
+        T::PayloadAttributes,
+        HeaderTy<<T::BuiltPayload as BuiltPayload>::Primitives>,
+    >,
     Pool: TransactionPool + Unpin,
 {
     /// Spawns a new [`LocalMiner`] with the given parameters.
     pub fn new(
-        provider: impl BlockReader,
+        provider: impl BlockReader<Header = HeaderTy<<T::BuiltPayload as BuiltPayload>::Primitives>>,
         payload_attributes_builder: B,
         to_engine: ConsensusEngineHandle<T>,
         mode: MiningMode<Pool>,
         payload_builder: PayloadBuilderHandle<T>,
     ) -> Self {
-        let latest_header =
+        let last_header =
             provider.sealed_header(provider.best_block_number().unwrap()).unwrap().unwrap();
 
         Self {
             payload_attributes_builder,
             to_engine,
             mode,
             payload_builder,
-            last_timestamp: latest_header.timestamp(),
-            last_block_hashes: VecDeque::from([latest_header.hash()]),
+            last_block_hashes: VecDeque::from([last_header.hash()]),
+            last_header,
         }
     }
 
@@ -193,19 +196,11 @@ where
     /// Generates payload attributes for a new block, passes them to FCU and inserts built payload
     /// through newPayload.
     async fn advance(&mut self) -> eyre::Result<()> {
-        let timestamp = std::cmp::max(
-            self.last_timestamp.saturating_add(1),
-            std::time::SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .expect("cannot be earlier than UNIX_EPOCH")
-                .as_secs(),
-        );
-
         let res = self
             .to_engine
             .fork_choice_updated(
                 self.forkchoice_state(),
-                Some(self.payload_attributes_builder.build(timestamp)),
+                Some(self.payload_attributes_builder.build(&self.last_header)),
                 EngineApiMessageVersion::default(),
             )
             .await?;
@@ -222,16 +217,16 @@ where
             eyre::bail!("No payload")
         };
 
-        let block = payload.block();
+        let header = payload.block().sealed_header().clone();
         let payload = T::block_to_payload(payload.block().clone());
         let res = self.to_engine.new_payload(payload).await?;
 
         if !res.is_valid() {
             eyre::bail!("Invalid payload")
         }
 
-        self.last_timestamp = timestamp;
-        self.last_block_hashes.push_back(block.hash());
+        self.last_block_hashes.push_back(header.hash());
+        self.last_header = header;
         // ensure we keep at most 64 blocks
         if self.last_block_hashes.len() > 64 {
             self.last_block_hashes.pop_front();

crates/engine/local/src/miner.rs

@@ -1,10 +1,12 @@
 //! The implementation of the [`PayloadAttributesBuilder`] for the
 //! [`LocalMiner`](super::LocalMiner).
 
+use alloy_consensus::BlockHeader;
 use alloy_primitives::{Address, B256};
-use reth_chainspec::EthereumHardforks;
+use reth_chainspec::{EthChainSpec, EthereumHardforks};
 use reth_ethereum_engine_primitives::EthPayloadAttributes;
 use reth_payload_primitives::PayloadAttributesBuilder;
+use reth_primitives_traits::SealedHeader;
 use std::sync::Arc;
 
 /// The attributes builder for local Ethereum payload.
@@ -13,21 +15,36 @@ use std::sync::Arc;
 pub struct LocalPayloadAttributesBuilder<ChainSpec> {
     /// The chainspec
     pub chain_spec: Arc<ChainSpec>,
+
+    /// Whether to enforce increasing timestamp.
+    pub enforce_increasing_timestamp: bool,
 }
 
 impl<ChainSpec> LocalPayloadAttributesBuilder<ChainSpec> {
     /// Creates a new instance of the builder.
     pub const fn new(chain_spec: Arc<ChainSpec>) -> Self {
-        Self { chain_spec }
+        Self { chain_spec, enforce_increasing_timestamp: true }
+    }
+
+    /// Creates a new instance of the builder without enforcing increasing timestamps.
+    pub fn without_increasing_timestamp(self) -> Self {
+        Self { enforce_increasing_timestamp: false, ..self }
     }
 }
 
-impl<ChainSpec> PayloadAttributesBuilder<EthPayloadAttributes>
+impl<ChainSpec> PayloadAttributesBuilder<EthPayloadAttributes, ChainSpec::Header>
     for LocalPayloadAttributesBuilder<ChainSpec>
 where
-    ChainSpec: Send + Sync + EthereumHardforks + 'static,
+    ChainSpec: EthChainSpec + EthereumHardforks + 'static,
 {
-    fn build(&self, timestamp: u64) -> EthPayloadAttributes {
+    fn build(&self, parent: &SealedHeader<ChainSpec::Header>) -> EthPayloadAttributes {
+        let mut timestamp =
+            std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs();
+
+        if self.enforce_increasing_timestamp {
+            timestamp = std::cmp::max(parent.timestamp().saturating_add(1), timestamp);
+        }
+
         EthPayloadAttributes {
             timestamp,
             prev_randao: B256::random(),
@@ -45,14 +62,18 @@ where
 }
 
 #[cfg(feature = "op")]
-impl<ChainSpec> PayloadAttributesBuilder<op_alloy_rpc_types_engine::OpPayloadAttributes>
+impl<ChainSpec>
+    PayloadAttributesBuilder<op_alloy_rpc_types_engine::OpPayloadAttributes, ChainSpec::Header>
     for LocalPayloadAttributesBuilder<ChainSpec>
 where
-    ChainSpec: Send + Sync + EthereumHardforks + 'static,
+    ChainSpec: EthChainSpec + EthereumHardforks + 'static,
 {
-    fn build(&self, timestamp: u64) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
+    fn build(
+        &self,
+        parent: &SealedHeader<ChainSpec::Header>,
+    ) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
         op_alloy_rpc_types_engine::OpPayloadAttributes {
-            payload_attributes: self.build(timestamp),
+            payload_attributes: self.build(parent),
             // Add dummy system transaction
             transactions: Some(vec![
                 reth_optimism_chainspec::constants::TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056

crates/engine/local/src/payload.rs

@@ -9,7 +9,7 @@ use reth_db_api::{
     RawKey, RawTable, RawValue,
 };
 use reth_era::{
-    common::{decode::DecodeCompressed, file_ops::StreamReader},
+    common::{decode::DecodeCompressedRlp, file_ops::StreamReader},
     e2s::error::E2sError,
     era1::{
         file::{BlockTupleIterator, Era1Reader},

crates/era-utils/src/history.rs

@@ -5,7 +5,7 @@ use alloy_rlp::Decodable;
 use ssz::Decode;
 
 /// Extension trait for generic decoding from compressed data
-pub trait DecodeCompressed {
+pub trait DecodeCompressedRlp {
     /// Decompress and decode the data into the given type
     fn decode<T: Decodable>(&self) -> Result<T, E2sError>;
 }