refactor(tx-sender): check transient errors before terminal config errors

Transient transport errors (BackendGone, retry-eligible, KMS signer)
should be classified before attempting to decode gateway config errors,
since transport-level failures may lack a valid RPC error payload.

Author: obatirou

coprocessor/fhevm-engine/transaction-sender/src/ops/add_ciphertext.rs

@@ -75,20 +75,6 @@ where
                 return Ok(());
             }
             Err(e) => {
-                if let Some(terminal_config_error) = try_extract_terminal_config_error(&e) {
-                    ADD_CIPHERTEXT_MATERIAL_FAIL_COUNTER.inc();
-                    error!(
-                        error = %terminal_config_error,
-                        handle = h,
-                        "Detected non-retryable gateway coprocessor config error while adding ciphertext"
-                    );
-                    self.mark_add_ciphertext_terminal_config_error(
-                        handle,
-                        &terminal_config_error.to_string(),
-                    )
-                    .await?;
-                    return Ok(());
-                }
                 // Consider transport retryable errors, BackendGone and local usage errors as something that must be retried infinitely.
                 // Local usage are included as they might be transient due to external AWS KMS signers.
                 if matches!(&e, RpcError::Transport(inner) if inner.is_retry_err() || matches!(inner, TransportErrorKind::BackendGone))
@@ -108,6 +94,20 @@ where
                     .await?;
                     bail!(e);
                 }
+                if let Some(terminal_config_error) = try_extract_terminal_config_error(&e) {
+                    ADD_CIPHERTEXT_MATERIAL_FAIL_COUNTER.inc();
+                    error!(
+                        error = %terminal_config_error,
+                        handle = h,
+                        "Detected non-retryable gateway coprocessor config error while adding ciphertext"
+                    );
+                    self.mark_add_ciphertext_terminal_config_error(
+                        handle,
+                        &terminal_config_error.to_string(),
+                    )
+                    .await?;
+                    return Ok(());
+                }
                 ADD_CIPHERTEXT_MATERIAL_FAIL_COUNTER.inc();
                 warn!(
                     error = %e,

coprocessor/fhevm-engine/transaction-sender/src/ops/allow_handle.rs

@@ -100,20 +100,6 @@ where
                 return Ok(());
             }
             Err(e) => {
-                if let Some(terminal_config_error) = try_extract_terminal_config_error(&e) {
-                    ALLOW_HANDLE_FAIL_COUNTER.inc();
-                    error!(
-                        error = %terminal_config_error,
-                        key = %key,
-                        "Detected non-retryable gateway coprocessor config error while allowing handle"
-                    );
-                    self.mark_allow_handle_terminal_config_error(
-                        key,
-                        &terminal_config_error.to_string(),
-                    )
-                    .await?;
-                    return Ok(());
-                }
                 // Consider transport retryable errors, BackendGone and local usage errors as something that must be retried infinitely.
                 // Local usage are included as they might be transient due to external AWS KMS signers.
                 if matches!(&e, RpcError::Transport(inner) if inner.is_retry_err() || matches!(inner, TransportErrorKind::BackendGone))
@@ -133,6 +119,20 @@ where
                     .await?;
                     bail!(e);
                 }
+                if let Some(terminal_config_error) = try_extract_terminal_config_error(&e) {
+                    ALLOW_HANDLE_FAIL_COUNTER.inc();
+                    error!(
+                        error = %terminal_config_error,
+                        key = %key,
+                        "Detected non-retryable gateway coprocessor config error while allowing handle"
+                    );
+                    self.mark_allow_handle_terminal_config_error(
+                        key,
+                        &terminal_config_error.to_string(),
+                    )
+                    .await?;
+                    return Ok(());
+                }
                 ALLOW_HANDLE_FAIL_COUNTER.inc();
                 warn!(
                     error = %e,

coprocessor/fhevm-engine/transaction-sender/src/ops/delegate_user_decrypt.rs

@@ -150,14 +150,6 @@ impl<P: Provider<Ethereum> + Clone + 'static> DelegateUserDecryptOperation<P> {
                 return TxResult::IdemPotentError;
             }
             Err(error) => {
-                if let Some(terminal_config_error) = try_extract_terminal_config_error(&error) {
-                    error!(
-                        error = %terminal_config_error,
-                        ?delegation,
-                        "{operation} failed with non-retryable gateway coprocessor config error"
-                    );
-                    return TxResult::NonRetryableConfigError(terminal_config_error);
-                }
                 if is_transient_error(&error) {
                     warn!(
                         %error,
@@ -166,6 +158,14 @@ impl<P: Provider<Ethereum> + Clone + 'static> DelegateUserDecryptOperation<P> {
                     );
                     return TxResult::TransientError;
                 }
+                if let Some(terminal_config_error) = try_extract_terminal_config_error(&error) {
+                    error!(
+                        error = %terminal_config_error,
+                        ?delegation,
+                        "{operation} failed with non-retryable gateway coprocessor config error"
+                    );
+                    return TxResult::NonRetryableConfigError(terminal_config_error);
+                }
                 warn!(
                     %error,
                     ?delegation,