fix: add explicit timeouts to all reqwest HTTP clients + add lots of prover logs (#4772)

Several HTTP clients across the codebase were created with
`reqwest::Client::new()` which has no timeout configured. This can cause
indefinite hangs when a remote server is slow or unresponsive, with no
error logged — the task silently freezes.

This was the root cause of batch 58064 being stuck for 5 hours: the
ProofGenDataSubmitter's HTTP POST to the prover gateway hung
indefinitely, leaving the batch as `picked_by_prover` until the
`proof_generation_timeout` (18000s) reclaim caught it.

Affected clients:
- proof_data_handler HTTP client (60s) — root cause of the incident
- Avail DA bridge API client (60s)
- Contract verifier GitHub resolver (300s — downloads large binaries)
- Metadata calculator tree API client (60s)
- ZK OS tree manager API client (60s)
- Prover autoscaler HTTP client (60s)

## What ❔

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- The `Why` has to be clear to non-Matter Labs entities running their
own ZK Chain -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Is this a breaking change?
- [ ] Yes
- [ ] No

## Operational changes
<!-- Any config changes? Any new flags? Any changes to any scripts? -->
<!-- Please add anything that non-Matter Labs entities running their own
ZK Chain may need to know -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zkstack dev fmt` and `zkstack dev
lint`.

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: tomg10

core/bin/contract-verifier/src/main.rs

@@ -94,6 +94,7 @@ async fn main() -> anyhow::Result<()> {
         verifier_config.etherscan_api_url.is_some() && etherscan_api_key.is_some();
     let contract_verifier = ContractVerifier::new(
         verifier_config.compilation_timeout,
+        verifier_config.compiler_download_timeout,
         pool.clone(),
         etherscan_verifier_enabled,
     )

core/bin/external_node/src/config/tests/config.yaml

@@ -79,6 +79,7 @@ api:
     latest_values_cache_size_mb: 200
     request_timeout_sec: 20
     tree_api_url: http://tree/
+    tree_api_request_timeout_sec: 45
     max_batch_request_size: 50
     websocket_requests_per_minute_limit: 1000
     mempool_cache_size: 1000

core/bin/external_node/src/config/tests/mod.rs

@@ -304,6 +304,7 @@ fn parsing_from_full_env() {
 
         # API component config
         EN_API_TREE_API_REMOTE_URL=http://tree/
+        EN_API_WEB3_JSON_RPC_TREE_API_REQUEST_TIMEOUT_SEC=45
         # Tree component config
         EN_TREE_API_PORT=2955
 
@@ -563,6 +564,7 @@ fn avail_da_client_from_env() {
         EN_DA_AVAIL_CLIENT_TYPE="FullClient"
         EN_DA_BRIDGE_API_URL="localhost:54321"
         EN_DA_TIMEOUT_MS="2000"
+        EN_DA_API_CLIENT_TIMEOUT_SEC="90"
         EN_DA_API_NODE_URL="localhost:12345"
         EN_DA_APP_ID="1"
         EN_DA_FINALITY_STATE="inBlock"

core/bin/external_node/src/node_builder.rs

@@ -432,6 +432,7 @@ impl ExternalNodeBuilder {
     fn add_tree_api_client_layer(mut self) -> anyhow::Result<Self> {
         self.node.add_layer(TreeApiClientLayer::http(
             self.config.local.api.web3_json_rpc.tree_api_url.clone(),
+            self.config.local.api.web3_json_rpc.tree_api_request_timeout,
         ));
         Ok(self)
     }

core/bin/zksync_server/src/node_builder.rs

@@ -460,8 +460,10 @@ impl MainNodeBuilder {
 
     fn add_tree_api_client_layer(mut self) -> anyhow::Result<Self> {
         let rpc_config = try_load_config!(self.configs.api_config).web3_json_rpc;
-        self.node
-            .add_layer(TreeApiClientLayer::http(rpc_config.tree_api_url));
+        self.node.add_layer(TreeApiClientLayer::http(
+            rpc_config.tree_api_url,
+            rpc_config.tree_api_request_timeout,
+        ));
         Ok(self)
     }
 

core/deny.toml

@@ -31,6 +31,13 @@ ignore = [
   "RUSTSEC-2026-0047",
   "RUSTSEC-2026-0048",
   "RUSTSEC-2026-0049", # `rustls-webpki` CRL vulnerability, old 0.101.7/0.102.8 pinned by rustls 0.21/0.22 (transitive deps of aws-smithy, hyper-rustls)
+  # `rand` 0.8.x unsoundness in `ThreadRng` reseed under a custom logger calling rand from
+  # log macros. Patch only exists for >=0.9.3 / >=0.10.1; no fix backported to 0.8.x. We
+  # already updated 0.9.x to 0.9.3; rand 0.8.5 remains via many transitive deps
+  # (alloy-primitives, sqlx-postgres, jsonrpsee, boojum, secp256k1, foundry-compilers, ...)
+  # that haven't migrated to rand 0.9. The unsoundness requires a custom logger that calls
+  # rand from logging callbacks, which we don't do.
+  "RUSTSEC-2026-0097",
 ]
 
 [licenses]

core/lib/config/src/configs/api.rs

@@ -385,6 +385,9 @@ pub struct Web3JsonRpcConfig {
     /// since the server can communicate with the tree in-process.
     #[config(alias = "tree_api_remote_url")]
     pub tree_api_url: Option<String>,
+    /// Total request timeout for the Tree API HTTP client. Only used when [`Self::tree_api_url`] is set.
+    #[config(default_t = Duration::from_secs(60))]
+    pub tree_api_request_timeout: Duration,
     /// Polling period for mempool cache update - how often the mempool cache is updated from the database.
     #[config(default_t = Duration::from_millis(50), with = Fallback(TimeUnit::Millis))]
     pub mempool_cache_update_interval: Duration,
@@ -542,6 +545,7 @@ mod tests {
                 websocket_requests_per_minute_limit: NonZeroU32::new(10).unwrap(),
                 request_timeout: Some(Duration::from_secs(20)),
                 tree_api_url: Some("http://tree/".into()),
+                tree_api_request_timeout: Duration::from_secs(45),
                 mempool_cache_update_interval: Duration::from_millis(50),
                 mempool_cache_size: 10000,
                 whitelisted_tokens_for_aa: vec![
@@ -605,6 +609,7 @@ mod tests {
             API_CONTRACT_VERIFICATION_PORT="3070"
             API_CONTRACT_VERIFICATION_URL="http://127.0.0.1:3070"
             API_WEB3_JSON_RPC_TREE_API_URL="http://tree/"
+            API_WEB3_JSON_RPC_TREE_API_REQUEST_TIMEOUT_SEC=45
             API_WEB3_JSON_RPC_MAX_RESPONSE_BODY_SIZE_MB=15
             API_WEB3_JSON_RPC_MAX_RESPONSE_BODY_SIZE_OVERRIDES_MB="eth_call=1, eth_getTransactionReceipt=None, zks_getProof=32"
             API_PROMETHEUS_LISTENER_PORT="3312"
@@ -668,6 +673,7 @@ mod tests {
             eth_call_gas_cap: null
             request_timeout_sec: 20
             tree_api_url: "http://tree/"
+            tree_api_request_timeout_sec: 45
             send_raw_tx_sync_max_timeout_ms: 10000
             send_raw_tx_sync_default_timeout_ms: 2000
           prometheus:
@@ -733,6 +739,7 @@ mod tests {
             eth_call_gas_cap: null
             request_timeout: 20s
             tree_api_url: "http://tree/"
+            tree_api_request_timeout: 45s
             send_raw_tx_sync_max_timeout_ms: 10000
             send_raw_tx_sync_default_timeout_ms: 2000
           prometheus:

core/lib/config/src/configs/contract_verifier.rs

@@ -13,6 +13,10 @@ pub struct ContractVerifierConfig {
     /// Max time of a single compilation.
     #[config(default_t = 4 * TimeUnit::Minutes, with = Fallback(TimeUnit::Seconds))]
     pub compilation_timeout: Duration,
+    /// Total request timeout for the GitHub compiler resolver HTTP client used to download
+    /// compiler binaries. Defaults to 5 minutes since binaries can be large.
+    #[config(default_t = 5 * TimeUnit::Minutes, with = Fallback(TimeUnit::Seconds))]
+    pub compiler_download_timeout: Duration,
     /// Port to which the Prometheus exporter server is listening.
     #[config(default_t = 3_318)]
     pub prometheus_port: u16,
@@ -39,6 +43,7 @@ mod tests {
     fn expected_config() -> ContractVerifierConfig {
         ContractVerifierConfig {
             compilation_timeout: Duration::from_secs(30),
+            compiler_download_timeout: Duration::from_secs(600),
             prometheus_port: 3314,
             port: 3070,
             etherscan_api_url: Some("https://api.etherscan.io/".to_owned()),
@@ -49,6 +54,7 @@ mod tests {
     fn parsing_from_env() {
         let env = r#"
             CONTRACT_VERIFIER_COMPILATION_TIMEOUT=30
+            CONTRACT_VERIFIER_COMPILER_DOWNLOAD_TIMEOUT=600
             CONTRACT_VERIFIER_PROMETHEUS_PORT=3314
             CONTRACT_VERIFIER_PORT=3070
             CONTRACT_VERIFIER_ETHERSCAN_API_URL="https://api.etherscan.io/"
@@ -66,6 +72,7 @@ mod tests {
         let yaml = r#"
           port: 3070
           compilation_timeout: 30
+          compiler_download_timeout: 600
           prometheus_port: 3314
           etherscan_api_url: https://api.etherscan.io/
         "#;
@@ -79,6 +86,7 @@ mod tests {
         let yaml = r#"
           port: 3070
           compilation_timeout: 30s
+          compiler_download_timeout: 10 min
           prometheus_port: 3314
           etherscan_api_url: https://api.etherscan.io/
         "#;

core/lib/config/src/configs/da_client/avail.rs

@@ -19,6 +19,11 @@ pub struct AvailConfig {
     pub bridge_api_url: String,
     #[config(default_t = Duration::from_secs(30))]
     pub timeout: Duration,
+    /// Total request timeout for the bridge API HTTP client.
+    /// Applies to every request made by the client; individual requests can still
+    /// further restrict via [`Self::timeout`].
+    #[config(default_t = Duration::from_secs(60))]
+    pub api_client_timeout: Duration,
     #[config(flatten)]
     pub config: AvailClientConfig,
 }

core/lib/config/src/configs/da_client/mod.rs

@@ -153,6 +153,7 @@ mod tests {
           client: Avail
           bridge_api_url: https://bridge-api.avail.so
           timeout_ms: 20000
+          api_client_timeout: 90s
           avail_client_type: GasRelay
           gas_relay_api_url: https://lens-turbo-api.availproject.org
           max_retries: 4
@@ -252,6 +253,7 @@ mod tests {
           client: Avail
           bridge_api_url: https://turing-bridge-api.avail.so
           timeout: 20s
+          api_client_timeout: 90s
           dispatch_timeout: 5s
           avail_client_type: FullClient
           api_node_url: wss://turing-rpc.avail.so/ws