feat: Batch RPC calls for execution simulation

Motivation: doing one simulation at a time is too slow. We want to batch per block to increase performance so that we can do more simulations in the same time.
Changes:
- Overwrites are now scoped for the entire block
- Simulations can be marked as:
  - Reverted - if the output from the RPC call is an error
  - Failed - if there was a problem in preparing/processing the simulation data. For example, if the balance/allowance storage slots could not be found.
  - Success - if everything went perfectly (as it should)

Took 6 hours 46 minutes


Took 13 seconds

Author: dianacarvalho1

Cargo.lock

@@ -40,7 +40,7 @@ tycho-ethereum = { version = ">=0.91.0", features = ["onchain_data"] }
 tycho-execution = ">=0.129.1"
 
 # EVM dependencies
-alloy = { version = "1.0.30", features = ["providers", "signer-local", "rpc-types-eth"] }
+alloy = { version = "1.0.30", features = ["providers", "signer-local", "rpc-types-eth", "provider-debug-api"] }
 
 [dependencies]
 # Serialization/Deserialization
@@ -112,8 +112,8 @@ approx = "0.5.1"
 rstest = "0.23.0"
 rstest_reuse = "0.7.0"
 tracing-subscriber = { workspace = true, default-features = false, features = [
-  "env-filter",
-  "fmt",
+    "env-filter",
+    "fmt",
 ] }
 tempfile = "3.13.0"
 
@@ -132,7 +132,7 @@ tracing-appender = "0.2.3"
 default = ["evm", "rfq"]
 network_tests = []
 evm = [
-  "dep:foundry-config", "dep:foundry-evm", "dep:revm", "dep:revm-inspectors", "dep:alloy",
+    "dep:foundry-config", "dep:foundry-evm", "dep:revm", "dep:revm-inspectors", "dep:alloy",
 ]
 rfq = ["dep:reqwest", "dep:async-trait", "dep:tokio-tungstenite", "dep:async-stream", "dep:http", "dep:prost"]
 

Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-alloy = { workspace = true, features = ["rpc-types-eth", "sol-types"] }
+alloy = { workspace = true, features = ["rpc-types-eth", "sol-types", "provider-debug-api"] }
 alloy-chains = "0.2.6"
 alloy-rpc-types-trace = "1.0.38"
 clap = { version = "4.5.48", features = ["derive"] }

tycho-integration-test/Cargo.toml

@@ -1,12 +1,12 @@
-use std::{str::FromStr, sync::Arc};
+use std::{collections::HashMap, str::FromStr, sync::Arc};
 
 use alloy::{
     rpc::types::{state::AccountOverride, Block, TransactionRequest},
     sol_types::SolValue,
 };
 use miette::{miette, IntoDiagnostic, WrapErr};
 use num_bigint::BigUint;
-use tracing::debug;
+use tracing::{debug, info};
 use tycho_common::{
     models::{token::Token, Chain},
     simulation::protocol_sim::ProtocolSim,
@@ -25,7 +25,12 @@ use tycho_simulation::{
     protocol::models::ProtocolComponent,
 };
 
-use crate::{execution::tenderly::OverwriteMetadata, RPCTools};
+use crate::{
+    execution::tenderly::{get_overwites_string, OverwriteMetadata},
+    RPCTools,
+};
+
+const USER_ADDR: &str = "0xf847a638E44186F3287ee9F8cAF73FF4d4B80784";
 
 pub fn encode_swap(
     component: &ProtocolComponent,
@@ -72,8 +77,7 @@ fn create_solution(
     amount_in: BigUint,
     expected_amount_out: BigUint,
 ) -> miette::Result<Solution> {
-    let user_address =
-        Bytes::from_str("0xf847a638E44186F3287ee9F8cAF73FF4d4B80784").into_diagnostic()?;
+    let user_address = Bytes::from_str(USER_ADDR).into_diagnostic()?;
 
     // Prepare data to encode. First we need to create a swap object
     let simple_swap =
@@ -168,121 +172,124 @@ fn encode_input(selector: &str, mut encoded_args: Vec<u8>) -> Vec<u8> {
 /// Returns both the overwrites and metadata for human-readable logging.
 pub(crate) async fn setup_user_overwrites(
     rpc_tools: &RPCTools,
-    solution: &Solution,
-    transaction: &Transaction,
+    user_balances: HashMap<Bytes, BigUint>,
     block: &Block,
-    user_address: Address,
-) -> miette::Result<(AddressHashMap<AccountOverride>, OverwriteMetadata)> {
+    to_address: Bytes,
+) -> miette::Result<(AddressHashMap<AccountOverride>, Vec<Bytes>)> {
     let mut overwrites = AddressHashMap::default();
     let mut metadata = OverwriteMetadata::new();
-    let token_address = Address::from_slice(&solution.given_token[..20]);
-    // If given token is ETH, add the given amount + 10 ETH for gas
-    if solution.given_token == Bytes::zero(20) {
-        let eth_balance = biguint_to_u256(&solution.given_amount) +
-            U256::from_str("10000000000000000000").unwrap(); // given_amount + 1 ETH for gas
-        overwrites.insert(user_address, AccountOverride::default().with_balance(eth_balance));
-    // if the given token is not ETH, do balance and allowance slots overwrites
-    } else {
-        let results = rpc_tools
-            .evm_balance_slot_detector
-            .detect_balance_slots(
-                std::slice::from_ref(&solution.given_token),
-                (**user_address).into(),
-                (*block.header.hash).into(),
-            )
-            .await;
+    let mut failed_tokens: Vec<Bytes> = Vec::new();
+    let user_address = Address::from_str(USER_ADDR).into_diagnostic()?;
+    let spender_address = Address::from_slice(&to_address[..20]);
+    for (token_address, balance) in user_balances {
+        // ETH
+        if token_address == Bytes::zero(20) {
+            overwrites.insert(
+                user_address,
+                AccountOverride::default().with_balance(biguint_to_u256(&balance)),
+            );
+        } else {
+            let results = rpc_tools
+                .evm_balance_slot_detector
+                .detect_balance_slots(
+                    std::slice::from_ref(&token_address),
+                    (**user_address).into(),
+                    (*block.header.hash).into(),
+                )
+                .await;
 
-        let (balance_storage_addr, balance_slot) =
-            if let Some(Ok((storage_addr, slot))) = results.get(&solution.given_token.clone()) {
-                (storage_addr, slot)
-            } else {
-                return Err(miette!("Couldn't find balance storage slot for token {token_address}"));
-            };
+            let (balance_storage_addr, balance_slot) =
+                if let Some(Ok((storage_addr, slot))) = results.get(&token_address) {
+                    (storage_addr, slot)
+                } else {
+                    failed_tokens.push(token_address.clone());
+                    continue;
+                };
 
-        let results = rpc_tools
-            .evm_allowance_slot_detector
-            .detect_allowance_slots(
-                std::slice::from_ref(&solution.given_token),
-                (**user_address).into(),
-                transaction.to.clone(), // tycho router
-                (*block.header.hash).into(),
-            )
-            .await;
+            let results = rpc_tools
+                .evm_allowance_slot_detector
+                .detect_allowance_slots(
+                    std::slice::from_ref(&token_address),
+                    (**user_address).into(),
+                    to_address.clone(), // tycho router
+                    (*block.header.hash).into(),
+                )
+                .await;
 
-        let (allowance_storage_addr, allowance_slot) = if let Some(Ok((storage_addr, slot))) =
-            results.get(&solution.given_token.clone())
-        {
-            (storage_addr, slot)
-        } else {
-            return Err(miette!("Couldn't find allowance storage slot for token {token_address}"));
-        };
+            let (allowance_storage_addr, allowance_slot) =
+                if let Some(Ok((storage_addr, slot))) = results.get(&token_address.clone()) {
+                    (storage_addr, slot)
+                } else {
+                    failed_tokens.push(token_address.clone());
+                    continue;
+                };
 
-        // Use the exact given amount for balance and allowance (no buffer, no max)
-        let token_balance = biguint_to_u256(&solution.given_amount);
-        let token_allowance = biguint_to_u256(&solution.given_amount);
+            // Use the exact given amount for balance and allowance (no buffer, no max)
+            let token_balance = biguint_to_u256(&balance);
+            let token_allowance = biguint_to_u256(&balance);
 
-        let balance_storage_address = Address::from_slice(&balance_storage_addr[..20]);
-        let allowance_storage_address = Address::from_slice(&allowance_storage_addr[..20]);
+            let balance_storage_address = Address::from_slice(&balance_storage_addr[..20]);
+            let allowance_storage_address = Address::from_slice(&allowance_storage_addr[..20]);
 
-        let balance_slot_b256 = alloy::primitives::B256::from_slice(balance_slot);
-        let allowance_slot_b256 = alloy::primitives::B256::from_slice(allowance_slot);
-        let spender_address = Address::from_slice(&transaction.to[..20]);
+            let balance_slot_b256 = alloy::primitives::B256::from_slice(balance_slot);
+            let allowance_slot_b256 = alloy::primitives::B256::from_slice(allowance_slot);
 
-        debug!(
+            debug!(
             "Setting token override for {token_address}: balance={}, allowance={}, balance_storage={}, allowance_storage={}",
             token_balance, token_allowance, balance_storage_address, allowance_storage_address
         );
 
-        // Add metadata for human-readable logging
-        metadata.add_balance(balance_storage_address, user_address, balance_slot_b256);
-        metadata.add_allowance(
-            allowance_storage_address,
-            user_address,
-            spender_address,
-            allowance_slot_b256,
-        );
+            // Add metadata for human-readable logging
+            metadata.add_balance(balance_storage_address, user_address, balance_slot_b256);
+            metadata.add_allowance(
+                allowance_storage_address,
+                user_address,
+                spender_address,
+                allowance_slot_b256,
+            );
 
-        // Apply balance and allowance overrides
-        // If both storage addresses are the same, combine them into one override
-        if balance_storage_address == allowance_storage_address {
-            overwrites.insert(
-                balance_storage_address,
-                AccountOverride::default().with_state_diff(vec![
-                    (
+            // Apply balance and allowance overrides
+            // If both storage addresses are the same, combine them into one override
+            if balance_storage_address == allowance_storage_address {
+                overwrites.insert(
+                    balance_storage_address,
+                    AccountOverride::default().with_state_diff(vec![
+                        (
+                            balance_slot_b256,
+                            alloy::primitives::B256::from_slice(&token_balance.to_be_bytes::<32>()),
+                        ),
+                        (
+                            allowance_slot_b256,
+                            alloy::primitives::B256::from_slice(
+                                &token_allowance.to_be_bytes::<32>(),
+                            ),
+                        ),
+                    ]),
+                );
+            } else {
+                // Different storage addresses, apply separately
+                overwrites.insert(
+                    balance_storage_address,
+                    AccountOverride::default().with_state_diff(vec![(
                         balance_slot_b256,
                         alloy::primitives::B256::from_slice(&token_balance.to_be_bytes::<32>()),
-                    ),
-                    (
+                    )]),
+                );
+                overwrites.insert(
+                    allowance_storage_address,
+                    AccountOverride::default().with_state_diff(vec![(
                         allowance_slot_b256,
                         alloy::primitives::B256::from_slice(&token_allowance.to_be_bytes::<32>()),
-                    ),
-                ]),
-            );
-        } else {
-            // Different storage addresses, apply separately
-            overwrites.insert(
-                balance_storage_address,
-                AccountOverride::default().with_state_diff(vec![(
-                    balance_slot_b256,
-                    alloy::primitives::B256::from_slice(&token_balance.to_be_bytes::<32>()),
-                )]),
-            );
-            overwrites.insert(
-                allowance_storage_address,
-                AccountOverride::default().with_state_diff(vec![(
-                    allowance_slot_b256,
-                    alloy::primitives::B256::from_slice(&token_allowance.to_be_bytes::<32>()),
-                )]),
-            );
+                    )]),
+                );
+            }
         }
-
-        // Add 10 ETH for gas for non-ETH token swaps
-        let eth_balance = U256::from_str("10000000000000000000").unwrap(); // 1 ETH for gas
-        overwrites.insert(user_address, AccountOverride::default().with_balance(eth_balance));
-        debug!("Setting ETH balance override for user {user_address}: {eth_balance} (for gas)");
     }
 
-    Ok((overwrites, metadata))
+    let overwrites_string = get_overwites_string(&overwrites, Some(&metadata));
+    info!("Block {} overwrites: {}", block.number(), overwrites_string);
+
+    Ok((overwrites, failed_tokens))
 }
 
 pub(crate) fn swap_request(