Support compressed package downloads

Cargo.toml

@@ -156,6 +156,7 @@ env_logger = { version = "0.11.5", default-features = false }
 field-offset = "0.3.3"
 filetime = "0.2.18"
 flate2 = "1.0.34"
+zstd = "0.13.2"
 fnv = "1.0.3"
 fs_extra = { version = "1.3.0" }
 futures = "0.3.30"

lib/cli/Cargo.toml

@@ -252,13 +252,15 @@ reqwest = { workspace = true, default-features = false, features = [
 	"json",
 	"multipart",
 	"gzip",
+	"zstd",
 ] }
 
 [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies]
 reqwest = { workspace = true, default-features = false, features = [
 	"native-tls",
 	"json",
 	"multipart",
+	"zstd",
 ] }
 
 

lib/cli/src/commands/package/download.rs

@@ -204,7 +204,10 @@ impl PackageDownload {
 
         let b = client
             .get(download_url)
-            .header(http::header::ACCEPT, "application/webc");
+            .header(http::header::ACCEPT, "application/webc")
+            // NOTE: reqwest handles gzip/zstd decoding  the respone body
+            // automatically when the relevant features are enabled.
+            .header(http::header::ACCEPT_ENCODING, "gzip, zstd");
 
         pb.println(format!(
             "{} {DOWNLOADING_PACKAGE_EMOJI}Downloading package {ident} ...",

lib/wasix/Cargo.toml

@@ -100,6 +100,7 @@ bytecheck.workspace = true
 blake3.workspace = true
 petgraph.workspace = true
 lz4_flex.workspace = true
+flate2.workspace = true
 rayon = { workspace = true, optional = true }
 wasm-bindgen = { workspace = true, optional = true }
 js-sys = { workspace = true, optional = true }
@@ -152,6 +153,7 @@ windows-sys = { workspace = true, features = [
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 terminal_size.workspace = true
+zstd.workspace = true
 
 [dev-dependencies]
 wasmer = { path = "../api", version = "=7.0.0-alpha.2", default-features = false, features = [

lib/wasix/src/runtime/package_loader/builtin_loader.rs

@@ -1,12 +1,12 @@
 use std::{
     collections::HashMap,
     fmt::Write as _,
-    io::{ErrorKind, Write as _},
+    io::{ErrorKind, Read, Write as _},
     path::PathBuf,
     sync::{Arc, RwLock},
 };
 
-use anyhow::{Context, Error};
+use anyhow::{Context, Error, bail};
 use bytes::Bytes;
 use http::{HeaderMap, Method};
 use tempfile::NamedTempFile;
@@ -284,6 +284,8 @@ impl BuiltinPackageLoader {
         }
 
         let body = response.body.context("package download failed")?;
+        let body = Self::decode_response_body(&response.headers, body)
+            .context("package download failed: could not decode response body")?;
         tracing::debug!(%url, "package_download_succeeded");
 
         let body = bytes::Bytes::from(body);
@@ -298,6 +300,12 @@ impl BuiltinPackageLoader {
         headers.insert("Accept", "application/webc".parse().unwrap());
         headers.insert("User-Agent", USER_AGENT.parse().unwrap());
 
+        // Accept compressed responses.
+        // NOTE: gzip and zstd decoding is available on native platforms.
+        // In browser platforms, the fetch implementation should automatically
+        // handle decoding of gzip/zstd responses transparently.
+        headers.insert(http::header::ACCEPT_ENCODING, "gzip, zstd".parse().unwrap());
+
         if url.has_authority()
             && let Some(token) = self.tokens.get(url.authority())
         {
@@ -317,6 +325,53 @@ impl BuiltinPackageLoader {
 
         headers
     }
+
+    fn decode_response_body(headers: &HeaderMap, body: Vec<u8>) -> Result<Vec<u8>, anyhow::Error> {
+        let encodings = match headers.get(http::header::CONTENT_ENCODING) {
+            Some(header) => header
+                .to_str()
+                .context("invalid content-encoding header")?
+                .split(',')
+                .map(|encoding| encoding.trim().to_ascii_lowercase())
+                .filter(|encoding| !encoding.is_empty())
+                .collect::<Vec<_>>(),
+            None => Vec::new(),
+        };
+
+        if encodings.is_empty() || (encodings.len() == 1 && encodings[0] == "identity") {
+            return Ok(body);
+        }
+
+        let mut reader: Box<dyn Read> = Box::new(std::io::Cursor::new(body));
+        for encoding in encodings.iter().rev() {
+            match encoding.as_str() {
+                "gzip" => {
+                    reader = Box::new(flate2::read::GzDecoder::new(reader));
+                }
+                "zstd" => {
+                    #[cfg(not(target_arch = "wasm32"))]
+                    {
+                        reader = Box::new(
+                            zstd::stream::read::Decoder::new(reader)
+                                .context("failed to initialize zstd decoder")?,
+                        );
+                    }
+                    #[cfg(target_arch = "wasm32")]
+                    {
+                        bail!("zstd content-encoding is not supported on wasm32");
+                    }
+                }
+                "identity" => {}
+                other => bail!("unsupported content-encoding: {other}"),
+            }
+        }
+
+        let mut decoded = Vec::new();
+        reader
+            .read_to_end(&mut decoded)
+            .context("failed to decode response body")?;
+        Ok(decoded)
+    }
 }
 
 impl Default for BuiltinPackageLoader {