feat: improve download speed with the in-built downloader (#6632)

Author: LesnyRumcajs

.config/forest.dic

@@ -1,7 +1,8 @@
-75
+78
 Algorand/M
 API/M
 APIs
+aria2c
 args
 arities
 arity
@@ -81,6 +82,7 @@ libp2p
 liveness
 localhost
 mainnet
+MD5
 MDNS
 mempool
 Merkle
@@ -131,6 +133,7 @@ TOML
 trie
 truthy
 TTY
+UI
 uncompress
 unrepresentable
 untrusted

CHANGELOG.md

@@ -29,6 +29,8 @@
 
 ### Added
 
+- [#3715](https://github.com/ChainSafe/forest/issues/3715): Implemented parallel HTTP downloads for snapshots with 5 concurrent connections by default (configurable via `FOREST_DOWNLOAD_CONNECTIONS`), bringing significant performance improvements for snapshot downloads (on par with a manual `aria2c -x5`).
+
 ### Changed
 
 ### Removed

docs/docs/users/knowledge_base/network_upgrades_state_migrations.md

@@ -37,10 +37,10 @@ Sample output:
 location: /archive/mainnet/latest/forest_snapshot_mainnet_2024-08-06_height_415650.forest.car.zst
 ```
 
-You see that the snapshot is past the upgrade epoch by ten epochs. You download the snapshot with `aria2c` because it's significantly faster than a raw `curl`.
+You see that the snapshot is past the upgrade epoch by ten epochs. You download the snapshot with the in-built tool which is faster than raw `cURL`.
 
 ```bash
-aria2c -x5 https://forest-archive.chainsafe.dev/latest/mainnet/
+forest-tool snapshot fetch --chain mainnet
 ```
 
 You start your node with `--import-snapshot <snapshot-path>` and enjoy the new, fancy NV23 features. Hooray!

docs/docs/users/reference/env_variables.md

@@ -57,6 +57,7 @@ process.
 | `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK`            | 1 or true                        | false                                          | 1                                                             | Whether or not to backfill full tipsets from the p2p network                                                          |
 | `FOREST_STRICT_JSON`                                      | 1 or true                        | false                                          | 1                                                             | Enable strict JSON validation to detect duplicate keys in RPC requests                                                |
 | `FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH`                      | URL or file path                 | empty                                          | `/var/tmp/forest_snapshot_calibnet.forest.car.zst`            | Override snapshot path for `--auto-download-snapshot`                                                                 |
+| `FOREST_DOWNLOAD_CONNECTIONS`                             | positive integer                 | 5                                              | 10                                                            | Number of parallel HTTP connections for downloading snapshots                                                         |
 
 ### `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT`
 

src/cli_shared/snapshot.rs

@@ -67,6 +67,8 @@ pub async fn fetch(
         .date_and_height_and_forest();
     let filename = filename(vendor, chain, date, height, forest_format);
 
+    tracing::info!("Downloading snapshot: {filename}");
+
     download_file_with_retry(
         &url,
         directory,

src/daemon/mod.rs

@@ -718,14 +718,19 @@ async fn maybe_set_snapshot_path(
                     config.client.snapshot_path = Some(path.into());
                 }
                 _ => {
-                    let url = crate::cli_shared::snapshot::stable_url(vendor, chain)?;
-                    config.client.snapshot_path = Some(url.to_string().into());
+                    // Resolve the redirect URL to get the actual snapshot URL
+                    // This ensures all chunks download from the same snapshot even if
+                    // a new snapshot is published during the download
+                    let (resolved_url, _num_bytes, filename) =
+                        crate::cli_shared::snapshot::peek(vendor, chain).await?;
+                    tracing::info!("Downloading snapshot: {filename}");
+                    config.client.snapshot_path = Some(resolved_url.to_string().into());
                 }
             }
         }
         (true, false, false) => {
             // we need a snapshot, don't have one, and don't have permission to download one, so ask the user
-            let (url, num_bytes, _path) = crate::cli_shared::snapshot::peek(vendor, chain)
+            let (url, num_bytes, filename) = crate::cli_shared::snapshot::peek(vendor, chain)
                 .await
                 .context("couldn't get snapshot size")?;
             // dialoguer will double-print long lines, so manually print the first clause ourselves,
@@ -751,6 +756,7 @@ async fn maybe_set_snapshot_path(
                     "Forest requires a snapshot to sync with the network, but automatic fetching is disabled."
                 )
             }
+            tracing::info!("Downloading snapshot: {filename}");
             config.client.snapshot_path = Some(url.to_string().into());
         }
     };

src/lib.rs

@@ -94,6 +94,7 @@ pub mod doctest_private {
             version::NetworkVersion,
         },
         utils::io::progress_log::WithProgress,
+        utils::net::{DownloadFileOption, download_to},
         utils::{encoding::blake2b_256, encoding::keccak_256, io::read_toml},
     };
 }

src/utils/net.rs

@@ -105,7 +105,10 @@ pub async fn reader(
         }
     };
 
-    Ok(tokio::io::BufReader::new(
+    // Use a larger buffer (512KB) for better throughput on large files
+    const DOWNLOAD_BUFFER_SIZE: usize = 512 * 1024;
+    Ok(tokio::io::BufReader::with_capacity(
+        DOWNLOAD_BUFFER_SIZE,
         WithProgress::wrap_sync_read_with_callback("Loading", stream, content_length, callback)
             .bytes(),
     ))