Limit maintenance time (#3951)

# Description
Recently there was an issue where indexing 1 particular settlement took
a very long time. This caused the whole protocol to slow down which in
turn reduced the throughput and increased the time to happy moo.
This PR introduces a timeout for the maintenance logic. Whenever we hit
this timeout we prefer running a not fully updated auction over stalling
the protocol further (e.g. some order might be included which has
already been settled).

Author: MartinquaXD

crates/autopilot/src/arguments.rs

@@ -267,6 +267,13 @@ pub struct Arguments {
     /// and not cut any auctions.
     #[clap(long, env, default_value = "false", action = clap::ArgAction::Set)]
     pub enable_leader_lock: bool,
+
+    /// Limits the amount of time the autopilot may spend running the
+    /// maintenance logic between 2 auctions. When this times out we prefer
+    /// running a not fully updated auction over stalling the protocol any
+    /// further.
+    #[clap(long, env, default_value = "5s", value_parser = humantime::parse_duration)]
+    pub max_maintenance_timeout: Duration,
 }
 
 #[derive(Debug, clap::Parser)]
@@ -399,6 +406,7 @@ impl std::fmt::Display for Arguments {
             disable_1271_order_balance_filter,
             disable_1271_order_sig_filter,
             enable_leader_lock,
+            max_maintenance_timeout,
         } = self;
 
         write!(f, "{shared}")?;
@@ -481,6 +489,7 @@ impl std::fmt::Display for Arguments {
             "disable_1271_order_sig_filter: {disable_1271_order_sig_filter}"
         )?;
         writeln!(f, "enable_leader_lock: {enable_leader_lock}")?;
+        writeln!(f, "max_maintenance_timeout: {max_maintenance_timeout:?}")?;
         Ok(())
     }
 }

crates/autopilot/src/maintenance.rs

@@ -20,7 +20,11 @@ use {
         core::{AtomicU64, GenericGauge},
     },
     shared::{event_handling::AlloyEventRetriever, maintenance::Maintaining},
-    std::{future::Future, sync::Arc, time::Instant},
+    std::{
+        future::Future,
+        sync::Arc,
+        time::{Duration, Instant},
+    },
     tokio::sync::Mutex,
 };
 
@@ -36,18 +40,25 @@ pub struct Maintenance {
     cow_amm_indexer: Vec<Arc<dyn Maintaining>>,
     /// On which block we last ran an update successfully.
     last_processed: Mutex<BlockInfo>,
+    /// Limits the amount of time the autopilot may spend running the
+    /// maintenance logic between 2 auctions. When this times out we prefer
+    /// running a not fully updated auction over stalling the protocol any
+    /// further.
+    timeout: Duration,
 }
 
 impl Maintenance {
     pub fn new(
         settlement_indexer: EventUpdater<Indexer, AlloyEventRetriever<GPv2SettlementContract>>,
         db_cleanup: Postgres,
+        timeout: Duration,
     ) -> Self {
         Self {
             settlement_indexer,
             db_cleanup,
             cow_amm_indexer: Default::default(),
             last_processed: Default::default(),
+            timeout,
         }
     }
 
@@ -62,7 +73,8 @@ impl Maintenance {
         }
 
         let start = Instant::now();
-        if let Err(err) = self.update_inner().await {
+
+        if let Err(err) = tokio::time::timeout(self.timeout, self.update_inner()).await {
             tracing::warn!(?err, block = new_block.number, "failed to run maintenance");
             metrics().updates.with_label_values(&["error"]).inc();
             return;

crates/autopilot/src/run.rs

@@ -569,7 +569,11 @@ pub async fn run(args: Arguments, shutdown_controller: ShutdownController) {
     let trusted_tokens =
         AutoUpdatingTokenList::from_configuration(market_makable_token_list_configuration).await;
 
-    let mut maintenance = Maintenance::new(settlement_event_indexer, db_write.clone());
+    let mut maintenance = Maintenance::new(
+        settlement_event_indexer,
+        db_write.clone(),
+        args.max_maintenance_timeout,
+    );
     maintenance.with_cow_amms(&cow_amm_registry);
 
     if !args.ethflow_contracts.is_empty() {