feat: fallback to single tx submission if batching fails

rust/main/chains/hyperlane-ethereum/src/contracts/multicall.rs

@@ -7,7 +7,9 @@ use itertools::{Either, Itertools};
 use tokio::sync::Mutex;
 use tracing::warn;
 
-use hyperlane_core::{ChainResult, HyperlaneDomain, HyperlaneProvider, H256, U256};
+use hyperlane_core::{
+    ChainCommunicationError, ChainResult, HyperlaneDomain, HyperlaneProvider, H256, U256,
+};
 
 use crate::{decode_revert_reason, EthereumProvider};
 
@@ -34,7 +36,7 @@ pub async fn build_multicall<M: Middleware + 'static>(
     domain: HyperlaneDomain,
     cache: Arc<Mutex<BatchCache>>,
     multicall_contract_address: H256,
-) -> eyre::Result<Multicall<M>> {
+) -> ChainResult<Multicall<M>> {
     let is_contract_cache = {
         let cache = cache.lock().await;
         cache.is_contract.get(&multicall_contract_address).cloned()
@@ -56,16 +58,19 @@ pub async fn build_multicall<M: Middleware + 'static>(
     };
 
     if !is_contract {
-        return Err(eyre::eyre!("Multicall contract not found at address"));
+        let err =
+            ChainCommunicationError::ContractNotFound(hex::encode(multicall_contract_address));
+        return Err(err);
     }
     let multicall =
         match Multicall::new(provider.clone(), Some(multicall_contract_address.into())).await {
             Ok(multicall) => multicall.version(MulticallVersion::Multicall3),
             Err(err) => {
-                return Err(eyre::eyre!(
-                    "Unable to build multicall contract: {}",
-                    err.to_string()
-                ))
+                tracing::error!(?err, "Unable to build multicall contract");
+                let err = ChainCommunicationError::CustomError(format!(
+                    "Unable to build multicall contract: {err}",
+                ));
+                return Err(err);
             }
         };
 

rust/main/chains/hyperlane-ethereum/src/rpc_clients/provider.rs

@@ -337,7 +337,7 @@ where
         &self,
         cache: Arc<Mutex<BatchCache>>,
         batch_contract_address: H256,
-    ) -> eyre::Result<Multicall<M>> {
+    ) -> ChainResult<Multicall<M>> {
         multicall::build_multicall(
             self.provider.clone(),
             self.domain.clone(),

rust/main/hyperlane-core/src/error.rs

@@ -72,6 +72,9 @@ pub enum ChainCommunicationError {
     /// An error with a contract call
     #[error(transparent)]
     ContractError(HyperlaneCustomErrorWrapper),
+    /// When a transaction is not found
+    #[error("Address is not a contract {0}")]
+    ContractNotFound(String),
     /// A transaction was dropped from the mempool
     #[error("Transaction dropped from mempool {0:?}")]
     TransactionDropped(H256),

rust/main/lander/src/adapter/chains/ethereum/adapter.rs

@@ -11,6 +11,7 @@ use ethers_core::abi::Function;
 use ethers_core::types::Eip1559TransactionRequest;
 use eyre::eyre;
 use futures_util::future;
+use hyperlane_core::ChainResult;
 use tokio::sync::Mutex;
 use tokio::try_join;
 use tracing::{debug, error, info, instrument, warn};
@@ -33,6 +34,9 @@ use hyperlane_ethereum::{
     multicall, EthereumReorgPeriod, EvmProviderForLander, LanderProviderBuilder,
 };
 
+use crate::adapter::chains::ethereum::metrics::{
+    LABEL_BATCHED_TRANSACTION_FAILED, LABEL_BATCHED_TRANSACTION_SUCCESS,
+};
 use crate::{
     adapter::{core::TxBuildingResult, AdaptsChain, GasLimit},
     dispatcher::{PayloadDb, PostInclusionMetricsSource, TransactionDb},
@@ -66,6 +70,7 @@ pub struct EthereumAdapter {
     pub payload_db: Arc<dyn PayloadDb>,
     pub signer: H160,
     pub minimum_time_between_resubmissions: Duration,
+    pub metrics: EthereumAdapterMetrics,
 }
 
 impl EthereumAdapter {
@@ -97,14 +102,16 @@ impl EthereumAdapter {
             .ok_or_else(|| eyre!("No signer found in provider for domain {}", domain))?;
 
         let metrics = EthereumAdapterMetrics::new(
+            conf.domain.clone(),
+            dispatcher_metrics.get_batched_transactions(),
             dispatcher_metrics.get_finalized_nonce(domain, &signer.to_string()),
             dispatcher_metrics.get_upper_nonce(domain, &signer.to_string()),
         );
 
         let payload_db = db.clone() as Arc<dyn PayloadDb>;
 
         let reorg_period = EthereumReorgPeriod::try_from(&conf.reorg_period)?;
-        let nonce_manager = NonceManager::new(&conf, db, provider.clone(), metrics).await?;
+        let nonce_manager = NonceManager::new(&conf, db, provider.clone(), metrics.clone()).await?;
 
         let adapter = Self {
             estimated_block_time: conf.estimated_block_time,
@@ -119,6 +126,7 @@ impl EthereumAdapter {
             payload_db,
             signer,
             minimum_time_between_resubmissions: DEFAULT_MINIMUM_TIME_BETWEEN_RESUBMISSIONS,
+            metrics,
         };
 
         Ok(adapter)
@@ -338,7 +346,7 @@ impl EthereumAdapter {
         &self,
         precursors: Vec<(TypedTransaction, Function)>,
         payload_details: Vec<PayloadDetails>,
-    ) -> Vec<TxBuildingResult> {
+    ) -> ChainResult<Vec<TxBuildingResult>> {
         use super::transaction::TransactionFactory;
 
         let multi_precursor = self
@@ -350,24 +358,19 @@ impl EthereumAdapter {
                 self.signer,
             )
             .await
-            .map(|(tx, f)| EthereumTxPrecursor::new(tx, f));
-
-        let multi_precursor = match multi_precursor {
-            Ok(precursor) => precursor,
-            Err(e) => {
-                error!(error = ?e, "Failed to batch payloads");
-                return vec![];
-            }
-        };
+            .map(|(tx, f)| EthereumTxPrecursor::new(tx, f))?;
 
         let transaction = TransactionFactory::build(multi_precursor, payload_details.clone());
 
         let tx_building_result = TxBuildingResult {
             payloads: payload_details,
             maybe_tx: Some(transaction),
         };
+        Ok(vec![tx_building_result])
+    }
 
-        vec![tx_building_result]
+    pub fn metrics(&self) -> &EthereumAdapterMetrics {
+        &self.metrics
     }
 }
 
@@ -403,14 +406,16 @@ impl AdaptsChain for EthereumAdapter {
         elapsed > *ready_time
     }
 
-    /// Builds a transaction for the given payloads.
+    /// Builds transactions for the given payloads.
     ///
     /// If there is only one payload, it builds a transaction without batching.
     /// If there are multiple payloads, it batches them into a single transaction.
     /// The order of individual calls in the batched transaction is determined
     /// by the order of payloads.
     /// The order should not change since the simulation and estimation of the batched transaction
     /// depend on the order of payloads.
+    /// If batching fails, it will fallback to building multiple transactions with a
+    /// single payload for each of them.
     async fn build_transactions(&self, payloads: &[FullPayload]) -> Vec<TxBuildingResult> {
         use super::transaction::TransactionFactory;
 
@@ -445,12 +450,48 @@ impl AdaptsChain for EthereumAdapter {
             return results;
         }
 
-        // Batched transaction
-        let results = self
-            .build_batched_transaction(precursors, payload_details)
-            .await;
+        match self
+            .build_batched_transaction(precursors.clone(), payload_details.clone())
+            .await
+        {
+            Ok(results) => {
+                self.metrics().increment_batched_transactions(
+                    LABEL_BATCHED_TRANSACTION_SUCCESS,
+                    payloads.len() as u64,
+                );
+                info!(
+                    ?payloads,
+                    ?results,
+                    "built batched transaction for payloads"
+                );
+                // Batched transaction
+                return results;
+            }
+            Err(err) => {
+                self.metrics().increment_batched_transactions(
+                    LABEL_BATCHED_TRANSACTION_FAILED,
+                    payloads.len() as u64,
+                );
+                warn!(
+                    domain = self.domain.name(),
+                    ?err,
+                    "Failed to build batch transaction. Fallback to single tx submission"
+                );
+            }
+        }
 
-        info!(?payloads, ?results, "built transaction for payloads");
+        let results: Vec<_> = payloads
+            .iter()
+            .map(|payload| {
+                let precursor = EthereumTxPrecursor::from_payload(payload, self.signer);
+                self.build_single_transaction(precursor, vec![payload.details.clone()])
+            })
+            .collect();
+        info!(
+            ?payloads,
+            ?results,
+            "built multiple transactions for multiple payloads"
+        );
         results
     }