diff --git a/Cargo.lock b/Cargo.lock index 5bf0d823ff5..3d08dd207fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -883,6 +883,8 @@ dependencies = [ "compression-core", "flate2", "memchr", + "zstd", + "zstd-safe", ] [[package]] @@ -7404,6 +7406,7 @@ dependencies = [ "dashmap", "derive_more", "env_logger", + "flate2", "futures", "getrandom 0.2.17", "heapless 0.9.2", @@ -7473,6 +7476,7 @@ dependencies = [ "weezl", "windows-sys 0.61.2", "xxhash-rust", + "zstd", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9750ebbc5fc..a53c63851f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,6 +156,7 @@ env_logger = { version = "0.11.5", default-features = false } field-offset = "0.3.3" filetime = "0.2.18" flate2 = "1.0.34" +zstd = "0.13.2" fnv = "1.0.3" fs_extra = { version = "1.3.0" } futures = "0.3.30" diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 6c81cac071f..5600c2a801a 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -252,6 +252,7 @@ reqwest = { workspace = true, default-features = false, features = [ "json", "multipart", "gzip", + "zstd", ] } [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] @@ -259,6 +260,7 @@ reqwest = { workspace = true, default-features = false, features = [ "native-tls", "json", "multipart", + "zstd", ] } diff --git a/lib/cli/src/commands/package/download.rs b/lib/cli/src/commands/package/download.rs index 2feca668276..22c560e5092 100644 --- a/lib/cli/src/commands/package/download.rs +++ b/lib/cli/src/commands/package/download.rs @@ -204,7 +204,10 @@ impl PackageDownload { let b = client .get(download_url) - .header(http::header::ACCEPT, "application/webc"); + .header(http::header::ACCEPT, "application/webc") + // NOTE: reqwest handles gzip/zstd decoding the respone body + // automatically when the relevant features are enabled. + .header(http::header::ACCEPT_ENCODING, "gzip, zstd"); pb.println(format!( "{} {DOWNLOADING_PACKAGE_EMOJI}Downloading package {ident} ...", diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 99b6b095ee4..cb7924403e6 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -100,6 +100,7 @@ bytecheck.workspace = true blake3.workspace = true petgraph.workspace = true lz4_flex.workspace = true +flate2.workspace = true rayon = { workspace = true, optional = true } wasm-bindgen = { workspace = true, optional = true } js-sys = { workspace = true, optional = true } @@ -152,6 +153,7 @@ windows-sys = { workspace = true, features = [ [target.'cfg(not(target_arch = "wasm32"))'.dependencies] terminal_size.workspace = true +zstd.workspace = true [dev-dependencies] wasmer = { path = "../api", version = "=7.0.0-alpha.2", default-features = false, features = [ diff --git a/lib/wasix/src/runtime/package_loader/builtin_loader.rs b/lib/wasix/src/runtime/package_loader/builtin_loader.rs index 942750b2dd6..b5246c6d0ea 100644 --- a/lib/wasix/src/runtime/package_loader/builtin_loader.rs +++ b/lib/wasix/src/runtime/package_loader/builtin_loader.rs @@ -1,12 +1,12 @@ use std::{ collections::HashMap, fmt::Write as _, - io::{ErrorKind, Write as _}, + io::{ErrorKind, Read, Write as _}, path::PathBuf, sync::{Arc, RwLock}, }; -use anyhow::{Context, Error}; +use anyhow::{Context, Error, bail}; use bytes::Bytes; use http::{HeaderMap, Method}; use tempfile::NamedTempFile; @@ -284,6 +284,8 @@ impl BuiltinPackageLoader { } let body = response.body.context("package download failed")?; + let body = Self::decode_response_body(&response.headers, body) + .context("package download failed: could not decode response body")?; tracing::debug!(%url, "package_download_succeeded"); let body = bytes::Bytes::from(body); @@ -298,6 +300,12 @@ impl BuiltinPackageLoader { headers.insert("Accept", "application/webc".parse().unwrap()); headers.insert("User-Agent", USER_AGENT.parse().unwrap()); + // Accept compressed responses. + // NOTE: gzip and zstd decoding is available on native platforms. + // In browser platforms, the fetch implementation should automatically + // handle decoding of gzip/zstd responses transparently. + headers.insert(http::header::ACCEPT_ENCODING, "gzip, zstd".parse().unwrap()); + if url.has_authority() && let Some(token) = self.tokens.get(url.authority()) { @@ -317,6 +325,53 @@ impl BuiltinPackageLoader { headers } + + fn decode_response_body(headers: &HeaderMap, body: Vec) -> Result, anyhow::Error> { + let encodings = match headers.get(http::header::CONTENT_ENCODING) { + Some(header) => header + .to_str() + .context("invalid content-encoding header")? + .split(',') + .map(|encoding| encoding.trim().to_ascii_lowercase()) + .filter(|encoding| !encoding.is_empty()) + .collect::>(), + None => Vec::new(), + }; + + if encodings.is_empty() || (encodings.len() == 1 && encodings[0] == "identity") { + return Ok(body); + } + + let mut reader: Box = Box::new(std::io::Cursor::new(body)); + for encoding in encodings.iter().rev() { + match encoding.as_str() { + "gzip" => { + reader = Box::new(flate2::read::GzDecoder::new(reader)); + } + "zstd" => { + #[cfg(not(target_arch = "wasm32"))] + { + reader = Box::new( + zstd::stream::read::Decoder::new(reader) + .context("failed to initialize zstd decoder")?, + ); + } + #[cfg(target_arch = "wasm32")] + { + bail!("zstd content-encoding is not supported on wasm32"); + } + } + "identity" => {} + other => bail!("unsupported content-encoding: {other}"), + } + } + + let mut decoded = Vec::new(); + reader + .read_to_end(&mut decoded) + .context("failed to decode response body")?; + Ok(decoded) + } } impl Default for BuiltinPackageLoader {