Switch from reqwest to bitreq HTTP client

Replace the `reqwest` dependency with `bitreq`, a lighter HTTP(s) client
that reduces code bloat and dependency overhead.

Key changes:
- Use `bitreq::Client` for connection pooling and reuse
- Update all HTTP request handling to use bitreq's API
- Remove `reqwest::header::HeaderMap` in favor of `HashMap<String, String>`
- Simplify `LnurlAuthToJwtProvider::new()` to no longer return a `Result`
- Use `serde_json::from_slice()` directly for JSON parsing
- Build script uses bitreq's blocking `send()` method

Co-Authored-By: HAL 9000
Signed-off-by: Elias Rohrer <dev@tnull.de>

Author: tnull

Cargo.toml

@@ -15,28 +15,28 @@ build = "build.rs"
 
 [features]
 default = ["lnurl-auth", "sigs-auth"]
-lnurl-auth = ["dep:bitcoin", "dep:url", "dep:serde", "dep:serde_json", "reqwest/json"]
+lnurl-auth = ["dep:bitcoin", "dep:url", "dep:serde", "dep:serde_json"]
 sigs-auth = ["dep:bitcoin"]
 
 [dependencies]
 prost = "0.11.6"
-reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
+bitreq = { version = "0.3", default-features = false, features = ["async-https"] }
 tokio = { version = "1", default-features = false, features = ["time"] }
 rand = "0.8.5"
 async-trait = "0.1.77"
 bitcoin = { version = "0.32.2", default-features = false, features = ["std", "rand-std"], optional = true }
-url = { version = "2.5.0", default-features = false, optional = true }
-base64 = { version = "0.22", default-features = false}
+url = { version = "2.5.0", default-features = false, optional = true, features = ["std"] }
+base64 = { version = "0.22", default-features = false, features = ["std"]}
 serde = { version = "1.0.196", default-features = false, features = ["serde_derive"], optional = true }
-serde_json = { version = "1.0.113", default-features = false, optional = true }
+serde_json = { version = "1.0.113", default-features = false, features = ["std"], optional = true }
 
 bitcoin_hashes = "0.14.0"
 chacha20-poly1305 = "0.1.2"
 log = { version = "0.4.29", default-features = false, features = ["std"]}
 
 [target.'cfg(genproto)'.build-dependencies]
 prost-build = { version = "0.11.3" }
-reqwest =  { version = "0.12", default-features = false, features = ["rustls-tls", "blocking"] }
+bitreq = { version = "0.3", default-features = false, features = ["std", "https"] }
 
 [dev-dependencies]
 mockito = "0.28.0"

build.rs

@@ -1,6 +1,8 @@
 #[cfg(genproto)]
 extern crate prost_build;
 #[cfg(genproto)]
+use std::io::Write;
+#[cfg(genproto)]
 use std::{env, fs, fs::File, path::Path};
 
 /// To generate updated proto objects:
@@ -25,9 +27,9 @@ fn generate_protos() {
 
 #[cfg(genproto)]
 fn download_file(url: &str, save_to: &str) -> Result<(), Box<dyn std::error::Error>> {
-	let mut response = reqwest::blocking::get(url)?;
+	let response = bitreq::get(url).send()?;
 	fs::create_dir_all(Path::new(save_to).parent().unwrap())?;
 	let mut out_file = File::create(save_to)?;
-	response.copy_to(&mut out_file)?;
+	out_file.write_all(&response.into_bytes())?;
 	Ok(())
 }

src/client.rs

@@ -1,14 +1,13 @@
+use bitreq::Client;
 use prost::Message;
-use reqwest::header::CONTENT_TYPE;
-use reqwest::Client;
 use std::collections::HashMap;
 use std::default::Default;
 use std::sync::Arc;
 
 use log::trace;
 
 use crate::error::VssError;
-use crate::headers::{get_headermap, FixedHeaders, VssHeaderProvider};
+use crate::headers::{FixedHeaders, VssHeaderProvider};
 use crate::types::{
 	DeleteObjectRequest, DeleteObjectResponse, GetObjectRequest, GetObjectResponse,
 	ListKeyVersionsRequest, ListKeyVersionsResponse, PutObjectRequest, PutObjectResponse,
@@ -17,7 +16,10 @@ use crate::util::retry::{retry, RetryPolicy};
 use crate::util::KeyValueVecKeyPrinter;
 
 const APPLICATION_OCTET_STREAM: &str = "application/octet-stream";
-const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
+const CONTENT_TYPE: &str = "content-type";
+const DEFAULT_TIMEOUT_SECS: u64 = 10;
+const MAX_RESPONSE_BODY_SIZE: usize = 1024 * 1024 * 1024; // 1GB
+const DEFAULT_CLIENT_CAPACITY: usize = 10;
 
 /// Thin-client to access a hosted instance of Versioned Storage Service (VSS).
 /// The provided [`VssClient`] API is minimalistic and is congruent to the VSS server-side API.
@@ -35,11 +37,11 @@ where
 impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	/// Constructs a [`VssClient`] using `base_url` as the VSS server endpoint.
 	pub fn new(base_url: String, retry_policy: R) -> Self {
-		let client = build_client();
+		let client = Client::new(DEFAULT_CLIENT_CAPACITY);
 		Self::from_client(base_url, client, retry_policy)
 	}
 
-	/// Constructs a [`VssClient`] from a given [`reqwest::Client`], using `base_url` as the VSS server endpoint.
+	/// Constructs a [`VssClient`] from a given [`bitreq::Client`], using `base_url` as the VSS server endpoint.
 	pub fn from_client(base_url: String, client: Client, retry_policy: R) -> Self {
 		Self {
 			base_url,
@@ -49,7 +51,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		}
 	}
 
-	/// Constructs a [`VssClient`] from a given [`reqwest::Client`], using `base_url` as the VSS server endpoint.
+	/// Constructs a [`VssClient`] from a given [`bitreq::Client`], using `base_url` as the VSS server endpoint.
 	///
 	/// HTTP headers will be provided by the given `header_provider`.
 	pub fn from_client_and_headers(
@@ -65,7 +67,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	pub fn new_with_headers(
 		base_url: String, retry_policy: R, header_provider: Arc<dyn VssHeaderProvider>,
 	) -> Self {
-		let client = build_client();
+		let client = Client::new(DEFAULT_CLIENT_CAPACITY);
 		Self { base_url, client, retry_policy, header_provider }
 	}
 
@@ -85,15 +87,17 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/getObject", self.base_url);
-				self.post_request(request, &url).await.and_then(|response: GetObjectResponse| {
-					if response.value.is_none() {
-						Err(VssError::InternalServerError(
-							"VSS Server API Violation, expected value in GetObjectResponse but found none".to_string(),
-						))
-					} else {
-						Ok(response)
-					}
-				})
+				self.post_request(request, &url, true).await.and_then(
+					|response: GetObjectResponse| {
+						if response.value.is_none() {
+							Err(VssError::InternalServerError(
+								"VSS Server API Violation, expected value in GetObjectResponse but found none".to_string(),
+							))
+						} else {
+							Ok(response)
+						}
+					},
+				)
 			},
 			&self.retry_policy,
 		)
@@ -121,7 +125,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/putObjects", self.base_url);
-				self.post_request(request, &url).await
+				self.post_request(request, &url, false).await
 			},
 			&self.retry_policy,
 		)
@@ -147,7 +151,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/deleteObject", self.base_url);
-				self.post_request(request, &url).await
+				self.post_request(request, &url, true).await
 			},
 			&self.retry_policy,
 		)
@@ -175,7 +179,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/listKeyVersions", self.base_url);
-				self.post_request(request, &url).await
+				self.post_request(request, &url, true).await
 			},
 			&self.retry_policy,
 		)
@@ -187,40 +191,36 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	}
 
 	async fn post_request<Rq: Message, Rs: Message + Default>(
-		&self, request: &Rq, url: &str,
+		&self, request: &Rq, url: &str, enable_pipelining: bool,
 	) -> Result<Rs, VssError> {
 		let request_body = request.encode_to_vec();
-		let headermap = self
+		let headers = self
 			.header_provider
 			.get_headers(&request_body)
 			.await
-			.and_then(|h| get_headermap(&h))
 			.map_err(|e| VssError::AuthError(e.to_string()))?;
-		let response_raw = self
-			.client
-			.post(url)
-			.header(CONTENT_TYPE, APPLICATION_OCTET_STREAM)
-			.headers(headermap)
-			.body(request_body)
-			.send()
-			.await?;
-		let status = response_raw.status();
-		let payload = response_raw.bytes().await?;
-
-		if status.is_success() {
+
+		let mut http_request = bitreq::post(url)
+			.with_header(CONTENT_TYPE, APPLICATION_OCTET_STREAM)
+			.with_headers(headers)
+			.with_body(request_body)
+			.with_timeout(DEFAULT_TIMEOUT_SECS)
+			.with_max_body_size(Some(MAX_RESPONSE_BODY_SIZE));
+
+		if enable_pipelining {
+			http_request = http_request.with_pipelining();
+		}
+
+		let response = self.client.send_async(http_request).await?;
+
+		let status_code = response.status_code;
+		let payload = response.into_bytes();
+
+		if (200..300).contains(&status_code) {
 			let response = Rs::decode(&payload[..])?;
 			Ok(response)
 		} else {
-			Err(VssError::new(status, payload))
+			Err(VssError::new(status_code, payload))
 		}
 	}
 }
-
-fn build_client() -> Client {
-	Client::builder()
-		.timeout(DEFAULT_TIMEOUT)
-		.connect_timeout(DEFAULT_TIMEOUT)
-		.read_timeout(DEFAULT_TIMEOUT)
-		.build()
-		.unwrap()
-}

src/error.rs

@@ -1,7 +1,5 @@
 use crate::types::{ErrorCode, ErrorResponse};
-use prost::bytes::Bytes;
 use prost::{DecodeError, Message};
-use reqwest::StatusCode;
 use std::error::Error;
 use std::fmt::{Display, Formatter};
 
@@ -32,13 +30,13 @@ pub enum VssError {
 
 impl VssError {
 	/// Create new instance of `VssError`
-	pub fn new(status: StatusCode, payload: Bytes) -> VssError {
+	pub fn new(status_code: i32, payload: Vec<u8>) -> VssError {
 		match ErrorResponse::decode(&payload[..]) {
 			Ok(error_response) => VssError::from(error_response),
 			Err(e) => {
 				let message = format!(
 					"Unable to decode ErrorResponse from server, HttpStatusCode: {}, DecodeErr: {}",
-					status, e
+					status_code, e
 				);
 				VssError::InternalError(message)
 			},
@@ -99,8 +97,8 @@ impl From<DecodeError> for VssError {
 	}
 }
 
-impl From<reqwest::Error> for VssError {
-	fn from(err: reqwest::Error) -> Self {
+impl From<bitreq::Error> for VssError {
+	fn from(err: bitreq::Error) -> Self {
 		VssError::InternalError(err.to_string())
 	}
 }

src/headers/lnurl_auth_jwt.rs

@@ -1,4 +1,4 @@
-use crate::headers::{get_headermap, VssHeaderProvider, VssHeaderProviderError};
+use crate::headers::{VssHeaderProvider, VssHeaderProviderError};
 use async_trait::async_trait;
 use base64::engine::general_purpose::URL_SAFE_NO_PAD;
 use base64::Engine;
@@ -26,6 +26,8 @@ const SIG_QUERY_PARAM: &str = "sig";
 const KEY_QUERY_PARAM: &str = "key";
 // The authorization header name.
 const AUTHORIZATION: &str = "Authorization";
+// The maximum body size we allow for requests.
+const MAX_RESPONSE_BODY_SIZE: usize = 16 * 1024 * 1024; // 16 KB
 
 #[derive(Debug, Clone)]
 struct JwtToken {
@@ -45,13 +47,14 @@ impl JwtToken {
 	}
 }
 
+const DEFAULT_TIMEOUT_SECS: u64 = 10;
+
 /// Provides a JWT token based on LNURL Auth.
 pub struct LnurlAuthToJwtProvider {
 	engine: Secp256k1<SignOnly>,
 	parent_key: Xpriv,
 	url: String,
 	default_headers: HashMap<String, String>,
-	client: reqwest::Client,
 	cached_jwt_token: RwLock<Option<JwtToken>>,
 }
 
@@ -70,47 +73,46 @@ impl LnurlAuthToJwtProvider {
 	/// with the JWT authorization header for VSS requests.
 	pub fn new(
 		parent_key: Xpriv, url: String, default_headers: HashMap<String, String>,
-	) -> Result<LnurlAuthToJwtProvider, VssHeaderProviderError> {
+	) -> LnurlAuthToJwtProvider {
 		let engine = Secp256k1::signing_only();
-		let default_headermap = get_headermap(&default_headers)?;
-		let client = reqwest::Client::builder()
-			.default_headers(default_headermap)
-			.build()
-			.map_err(VssHeaderProviderError::from)?;
 
-		Ok(LnurlAuthToJwtProvider {
+		LnurlAuthToJwtProvider {
 			engine,
 			parent_key,
 			url,
 			default_headers,
-			client,
 			cached_jwt_token: RwLock::new(None),
-		})
+		}
 	}
 
 	async fn fetch_jwt_token(&self) -> Result<JwtToken, VssHeaderProviderError> {
 		// Fetch the LNURL.
-		let lnurl_str = self
-			.client
-			.get(&self.url)
-			.send()
-			.await
-			.map_err(VssHeaderProviderError::from)?
-			.text()
-			.await
-			.map_err(VssHeaderProviderError::from)?;
+		let lnurl_request = bitreq::get(&self.url)
+			.with_headers(self.default_headers.clone())
+			.with_timeout(DEFAULT_TIMEOUT_SECS)
+			.with_max_body_size(Some(MAX_RESPONSE_BODY_SIZE));
+		let lnurl_response =
+			lnurl_request.send_async().await.map_err(VssHeaderProviderError::from)?;
+		let lnurl_str = String::from_utf8(lnurl_response.into_bytes()).map_err(|e| {
+			VssHeaderProviderError::InvalidData {
+				error: format!("LNURL response is not valid UTF-8: {}", e),
+			}
+		})?;
 
 		// Sign the LNURL and perform the request.
 		let signed_lnurl = sign_lnurl(&self.engine, &self.parent_key, &lnurl_str)?;
-		let lnurl_auth_response: LnurlAuthResponse = self
-			.client
-			.get(&signed_lnurl)
-			.send()
-			.await
-			.map_err(VssHeaderProviderError::from)?
-			.json()
-			.await
-			.map_err(VssHeaderProviderError::from)?;
+		let auth_request = bitreq::get(&signed_lnurl)
+			.with_headers(self.default_headers.clone())
+			.with_timeout(DEFAULT_TIMEOUT_SECS)
+			.with_max_body_size(Some(MAX_RESPONSE_BODY_SIZE));
+		let auth_response =
+			auth_request.send_async().await.map_err(VssHeaderProviderError::from)?;
+		let lnurl_auth_response: LnurlAuthResponse =
+			serde_json::from_slice(&auth_response.into_bytes()).map_err(|e| {
+				VssHeaderProviderError::InvalidData {
+					error: format!("Failed to parse LNURL Auth response as JSON: {}", e),
+				}
+			})?;
 
 		let untrusted_token = match lnurl_auth_response {
 			LnurlAuthResponse { token: Some(token), .. } => token,
@@ -256,8 +258,8 @@ impl From<bitcoin::bip32::Error> for VssHeaderProviderError {
 	}
 }
 
-impl From<reqwest::Error> for VssHeaderProviderError {
-	fn from(e: reqwest::Error) -> VssHeaderProviderError {
+impl From<bitreq::Error> for VssHeaderProviderError {
+	fn from(e: bitreq::Error) -> VssHeaderProviderError {
 		VssHeaderProviderError::RequestError { error: e.to_string() }
 	}
 }