feat(coprocessor): make drift auto revert off by default

But keep it running on E2E.

Author: dartdart26

charts/coprocessor/templates/coprocessor-gw-listener-deployment.yaml

@@ -70,7 +70,8 @@ spec:
             - --get-logs-poll-interval={{ .Values.gwListener.args.getLogsPollInterval }}
             - --get-logs-block-batch-size={{ .Values.gwListener.args.getLogsBlockBatchSize }}
             - --log-last-processed-every-number-of-updates={{ .Values.gwListener.args.logLastProcessedEveryNumberOfUpdates }}
-            - --drift-revert-grace-period={{ .Values.gwListener.args.driftRevertGracePeriod }}
+            - --drift-auto-revert-grace-period={{ .Values.gwListener.args.driftAutoRevertGracePeriod }}
+            - --drift-auto-revert-enabled={{ .Values.gwListener.args.driftAutoRevertEnabled }}
             {{- with .Values.gwListener.extraArgs }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

charts/coprocessor/values.yaml

@@ -593,7 +593,8 @@ gwListener:
     getLogsPollInterval: 500ms
     getLogsBlockBatchSize: 100
     logLastProcessedEveryNumberOfUpdates: 50
-    driftRevertGracePeriod: 60s
+    driftAutoRevertGracePeriod: 60s
+    driftAutoRevertEnabled: false
 
   # Additional args appended after all template-injected and structured args.
   # Use for replay parameters, e.g.:

coprocessor/fhevm-engine/gw-listener/src/bin/gw_listener.rs

@@ -118,7 +118,13 @@ struct Conf {
     /// running the revert SQL. Gives other services time to see the signal and
     /// re-exec before the DB state changes.
     #[arg(long, default_value = "60s", value_parser = parse_duration)]
-    drift_revert_grace_period: Duration,
+    drift_auto_revert_grace_period: Duration,
+
+    /// Enable automatic drift recovery. When false (default), the drift
+    /// detector still runs and logs drift, but no revert signal is created
+    /// and no automatic recovery kicks in. Opt-in while the feature rolls out.
+    #[arg(long, default_value_t = false)]
+    drift_auto_revert_enabled: bool,
 }
 
 fn install_signal_handlers(cancel_token: CancellationToken) -> anyhow::Result<()> {
@@ -196,7 +202,8 @@ async fn main() -> anyhow::Result<()> {
         gateway_config_address: conf.gateway_config_address,
         drift_no_consensus_timeout: conf.drift_no_consensus_timeout,
         drift_post_consensus_grace: conf.drift_post_consensus_grace,
-        drift_revert_grace_period: conf.drift_revert_grace_period,
+        drift_auto_revert_grace_period: conf.drift_auto_revert_grace_period,
+        drift_auto_revert_enabled: conf.drift_auto_revert_enabled,
     };
 
     let gw_listener = std::sync::Arc::new(GatewayListener::new(
@@ -229,7 +236,7 @@ async fn main() -> anyhow::Result<()> {
         config.database_url.as_str(),
         cancel_token.clone(),
         Some(RevertRunnerConfig {
-            grace_period: config.drift_revert_grace_period,
+            grace_period: config.drift_auto_revert_grace_period,
         }),
     )
     .await?;

coprocessor/fhevm-engine/gw-listener/src/drift_detector.rs

@@ -91,6 +91,7 @@ pub(crate) struct DriftDetector {
     local_node_id: String,
     drift_no_consensus_timeout: Duration,
     drift_post_consensus_grace: Duration,
+    auto_revert_enabled: bool,
     deferred_drift_detected: u64,
     deferred_consensus_timeout: u64,
     deferred_missing_submission: u64,
@@ -102,13 +103,15 @@ impl DriftDetector {
         expected_senders: Vec<Address>,
         drift_no_consensus_timeout: Duration,
         drift_post_consensus_grace: Duration,
+        auto_revert_enabled: bool,
     ) -> Self {
         Self {
             current_expected_senders: expected_senders,
             open_handles: HashMap::new(),
             local_node_id: std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown".to_owned()),
             drift_no_consensus_timeout,
             drift_post_consensus_grace,
+            auto_revert_enabled,
             deferred_drift_detected: 0,
             deferred_consensus_timeout: 0,
             deferred_missing_submission: 0,
@@ -339,13 +342,14 @@ impl DriftDetector {
             );
             self.deferred_drift_detected += 1;
 
-            // Auto drift recovery: signal that a revert is needed.
-            fhevm_engine_common::drift_revert::on_drift_detected(
-                db_pool,
-                handle.as_slice(),
-                chain_id_from_handle(handle),
-            )
-            .await;
+            if self.auto_revert_enabled {
+                fhevm_engine_common::drift_revert::on_drift_detected(
+                    db_pool,
+                    handle.as_slice(),
+                    chain_id_from_handle(handle),
+                )
+                .await;
+            }
         }
 
         let Some(state) = self.open_handles.get_mut(&handle) else {
@@ -698,10 +702,12 @@ mod tests {
         let digest_b = FixedBytes::from([0x66; 32]);
         let digest_128 = FixedBytes::from([0x77; 32]);
         let base = Instant::now();
+        let auto_revert_enabled = false;
         let mut detector = DriftDetector::new(
             vec![sender_a, sender_b, sender_c],
             Duration::from_secs(50),
             Duration::from_secs(10),
+            auto_revert_enabled,
         );
 
         detector.set_replaying(true);
@@ -829,7 +835,23 @@ mod tests {
     }
 
     fn detector() -> DriftDetector {
-        DriftDetector::new(senders(), Duration::from_secs(5), Duration::from_secs(2))
+        let auto_revert_enabled = false;
+        DriftDetector::new(
+            senders(),
+            Duration::from_secs(5),
+            Duration::from_secs(2),
+            auto_revert_enabled,
+        )
+    }
+
+    fn detector_with_auto_revert() -> DriftDetector {
+        let auto_revert_enabled = true;
+        DriftDetector::new(
+            senders(),
+            Duration::from_secs(5),
+            Duration::from_secs(2),
+            auto_revert_enabled,
+        )
     }
 
     fn make_consensus_state(
@@ -1648,7 +1670,7 @@ mod tests {
         .await
         .unwrap();
 
-        let mut detector = detector();
+        let mut detector = detector_with_auto_revert();
         detector
             .handle_consensus(
                 make_consensus_event(
@@ -1683,7 +1705,7 @@ mod tests {
             .await
             .unwrap();
 
-        let mut detector = detector();
+        let mut detector = detector_with_auto_revert();
         detector
             .handle_consensus(
                 make_consensus_event(

coprocessor/fhevm-engine/gw-listener/src/gw_listener.rs

@@ -173,6 +173,7 @@ impl<P: Provider<Ethereum> + Clone + 'static, A: AwsS3Interface + Clone + 'stati
             expected_senders,
             self.conf.drift_no_consensus_timeout,
             self.conf.drift_post_consensus_grace,
+            self.conf.drift_auto_revert_enabled,
         );
         if replay_from_block.is_none() {
             if let Err(e) = self

coprocessor/fhevm-engine/gw-listener/src/lib.rs

@@ -60,11 +60,13 @@ pub struct ConfigSettings {
     /// How long to wait after detecting a pending drift-revert signal before
     /// running the revert SQL. Gives other services time to see the signal and
     /// drop their in-flight work.
-    pub drift_revert_grace_period: Duration,
+    pub drift_auto_revert_grace_period: Duration,
+    /// If true, the drift detector creates drift-revert signals when it sees
+    /// a consensus mismatch. If false, drift is still detected and logged,
+    /// but no signal is created.
+    pub drift_auto_revert_enabled: bool,
 }
 
-/// Default is used by unit tests only. Production defaults come from
-/// the CLI arg definitions in `bin/gw_listener.rs` (e.g. `--drift-no-consensus-timeout 5m`).
 impl Default for ConfigSettings {
     fn default() -> Self {
         Self {
@@ -85,7 +87,8 @@ impl Default for ConfigSettings {
             gateway_config_address: None,
             drift_no_consensus_timeout: Duration::from_secs(5),
             drift_post_consensus_grace: Duration::from_secs(2),
-            drift_revert_grace_period: Duration::from_secs(60),
+            drift_auto_revert_grace_period: Duration::from_secs(60),
+            drift_auto_revert_enabled: false,
         }
     }
 }

test-suite/fhevm/docker-compose/coprocessor-docker-compose.yml

@@ -79,6 +79,7 @@ services:
       - --kms-generation-address=${KMS_GENERATION_ADDRESS}
       - --error-sleep-initial-secs=1
       - --error-sleep-max-secs=10
+      - --drift-auto-revert-enabled=true
     depends_on:
       coprocessor-db-migration:
         condition: service_completed_successfully