Add PMTiles compression support to the Rust MVT-to-MLT converter (#1498)

I'm looking into switching my project [wogibt.es](https://wogibt.es/) to
MLT.

When I first ran `mlt convert`, I was surprised that the output PMTiles
archive was much larger than the input. The MVT tiles were
gzip-compressed, while the generated MLT tiles were not. This PR adds
compression support and preserves compatible input compression by
default.

## Changes

- Add `--tile-compression auto|gzip|none`.
- Use `auto` by default.
- Preserve uncompressed, gzip, Brotli, and Zstandard input compression
in auto mode.
- Require an explicit `gzip` or `none` override for zlib input because
PMTiles v3 cannot represent zlib compression.
- Write the selected compression to the PMTiles header and metadata.
- Reject explicit compression options for unsupported input/output
combinations.
- Report raw payload and complete archive sizes separately.
- Add documentation and tests for compression handling and round trips.

Brotli and Zstandard are intentionally only preserved through
auto-detection and are not exposed as explicit CLI options. Common web
PMTiles clients require custom decompression support for these codecs,
so exposing them directly
could make it easy to produce archives that clients cannot read.

If exposing every PMTiles-supported compression mode is preferred, I can
add explicit `brotli` and `zstd` options as well.

Example output:

```text
input.mvt.pmtiles -> output.mlt.pmtiles (pmtiles):
converted 5 tiles (5 unique encoded, 0 cache hits) in 740.9ms
size raw/archive: MVT(gzip) 813.7kB/459.8kB -> MLT(gzip) 460.3kB/357.0kB
```

## AI Assistance

This pull request was developed with assistance from OpenAI Codex. The
changes were reviewed and tested by me.

---------

Co-authored-by: Frank Elsinga <frank@elsinga.de>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: 4 people

rust/mlt/README.md

@@ -6,8 +6,25 @@ The `mlt` binary provides several commands for working with MLT files:
 
 * **`dump`** - Parse an MLT file and dump raw layer data without decoding
 * **`decode`** - Parse an MLT file, decode all layers, and dump the result (supports text and `GeoJSON` output)
+* **`convert`** - Convert MVT or MLT tile files and MVT `.mbtiles`/`.pmtiles` archives to MLT
 * **`ui`** - Interactive terminal visualizer for MLT files
 
+### Format conversion
+
+Convert an MVT archive or files to MLT:
+
+```bash
+mlt convert input.mvt.pmtiles output.mlt.pmtiles
+```
+
+The conversion summary reports unique, decompressed tile payloads as such:
+
+```text
+input.mvt.pmtiles -> output.mlt.pmtiles (pmtiles):
+  converted 5 tiles (5 unique encoded, 0 cache hits) in 740.9ms
+  size raw/archive: MVT(gzip) 813.7kB/459.8kB -> MLT(gzip) 460.3kB/357.0kB
+```
+
 ### Visualizer
 
 The visualizer command provides an interactive terminal-based UI for exploring MLT files:

rust/mlt/src/convert/common.rs

@@ -6,7 +6,7 @@ use indicatif::{ProgressBar, ProgressStyle};
 use martin_tile_utils::Encoding;
 use mlt_core::encoder::EncoderConfig;
 use moka::sync::Cache;
-use pmtiles::{PmTilesWriter, TileCoord};
+use pmtiles::{Compression, PmTilesWriter, TileCoord};
 use size_format::SizeFormatterSI;
 use xxhash_rust::xxh3::Xxh3Builder;
 
@@ -50,7 +50,7 @@ pub const MAX_TILE_CACHE_TRACK_SIZE_BYTES: usize = 1024;
 const PROGRESS_BAR_TEMPLATE: &str = "  {bar:40.cyan/blue} {pos}/{len} tiles [{rate}, eta {eta}]";
 
 /// The encode dedup cache, keyed on the raw (small) tile bytes.
-pub type EncodeCache = Cache<Vec<u8>, Bytes, Xxh3Builder>;
+pub type EncodeCache = Cache<Vec<u8>, (Bytes, u64), Xxh3Builder>;
 
 pub fn make_progress_bar(total: u64) -> ProgressBar {
     let bar = ProgressBar::new(total);
@@ -68,25 +68,26 @@ pub fn make_progress_bar(total: u64) -> ProgressBar {
 pub fn make_encode_cache() -> EncodeCache {
     Cache::builder()
         .max_capacity(ENCODE_CACHE_BYTES)
-        .weigher(|_, v: &Bytes| u32::try_from(v.len()).unwrap_or(u32::MAX))
+        .weigher(|_, v: &(Bytes, u64)| u32::try_from(v.0.len()).unwrap_or(u32::MAX))
         .build_with_hasher(Xxh3Builder::default())
 }
 
-/// Encode one source tile MVT → MLT, deduplicating through `cache`.
+/// Encode one source tile MVT -> MLT, deduplicating through `cache`.
 ///
-/// Only small tiles (ocean, empty land, ...) actually repeat across a tileset;
-/// big city tiles are essentially unique, so tiles over
-/// [`MAX_TILE_CACHE_TRACK_SIZE_BYTES`] skip the cache (any rare repeat still
-/// dedups when the container is written). Returns the encoded bytes and whether
-/// they came from the cache. Mirrors `files.rs` and the mbtiles transcoder.
+/// Returns the encoded bytes, the raw MVT size, and whether the result came from the cache.
+///
+/// Only small tiles (ocean, empty land, ...) actually repeat across a tileset.
+/// Big city tiles are essentially unique, so tiles over [`MAX_TILE_CACHE_TRACK_SIZE_BYTES`] skip the cache.
+/// Any rare repeat still dedups when the container is written.
 pub fn encode_tile(
     cache: &EncodeCache,
     data: &[u8],
     encoding: Encoding,
     cfg: EncoderConfig,
-) -> AnyResult<(Bytes, bool)> {
+) -> AnyResult<(Bytes, u64, bool)> {
     if data.len() > MAX_TILE_CACHE_TRACK_SIZE_BYTES {
-        return Ok((encode_one(data.to_vec(), encoding, cfg)?, false));
+        let (encoded, raw_mvt_size) = encode_one(data.to_vec(), encoding, cfg)?;
+        return Ok((encoded, raw_mvt_size, false));
     }
     let mut hit = true;
     let encoded = cache
@@ -95,7 +96,7 @@ pub fn encode_tile(
             encode_one(data.to_vec(), encoding, cfg)
         })
         .map_err(|e| anyhow!("{e}"))?;
-    Ok((encoded, hit))
+    Ok((encoded.0, encoded.1, hit))
 }
 
 /// Running totals for a container-to-container conversion, matching the
@@ -105,32 +106,46 @@ pub struct TileStats {
     written: u64,
     cache_hits: u64,
     cache_encoded: u64,
-    bytes_in: u64,
-    bytes_out: u64,
+    raw_mvt_bytes: u64,
+    raw_mlt_bytes: u64,
 }
 
 impl TileStats {
-    pub fn record(&mut self, encoded_len: u64, bytes_in: u64, hit: bool) {
+    pub fn record(&mut self, encoded_len: u64, raw_mvt_size: u64, hit: bool) {
         self.written += 1;
         if hit {
             self.cache_hits += 1;
         } else {
             self.cache_encoded += 1;
-            self.bytes_in += bytes_in;
-            self.bytes_out += encoded_len;
+            self.raw_mvt_bytes += raw_mvt_size;
+            self.raw_mlt_bytes += encoded_len;
         }
     }
 
-    pub fn print_summary(&self, start: Instant) {
+    pub fn print_summary(
+        &self,
+        start: Instant,
+        input_archive_size: u64,
+        output_archive_size: u64,
+        source_encoding: Encoding,
+        tile_compression: Compression,
+    ) {
+        let source_encoding = source_encoding.compression().unwrap_or("none");
+        let tile_compression = tile_compression.content_encoding().unwrap_or("none");
         eprintln!(
-            "  converted {} tiles ({} unique encoded, {} cache hits, {:.1}B -> {:.1}B) in {:.1?}",
+            "  converted {} tiles ({} unique encoded, {} cache hits) in {:.1?}",
             self.written,
             self.cache_encoded,
             self.cache_hits,
-            SizeFormatterSI::new(self.bytes_in),
-            SizeFormatterSI::new(self.bytes_out),
             start.elapsed(),
         );
+        eprintln!(
+            "  size raw/archive: MVT({source_encoding}) {:.1}B/{:.1}B -> MLT({tile_compression}) {:.1}B/{:.1}B",
+            SizeFormatterSI::new(self.raw_mvt_bytes),
+            SizeFormatterSI::new(input_archive_size),
+            SizeFormatterSI::new(self.raw_mlt_bytes),
+            SizeFormatterSI::new(output_archive_size),
+        );
     }
 }
 
@@ -139,6 +154,6 @@ impl TileStats {
 pub struct EncodedTile {
     pub coord: TileCoord,
     pub data: Bytes,
-    pub bytes_in: u64,
+    pub raw_mvt_size: u64,
     pub hit: bool,
 }

rust/mlt/src/convert/from_mbtiles.rs

@@ -9,15 +9,15 @@ use futures::StreamExt;
 use martin_tile_utils::{Encoding, Format};
 use mbtiles::{MbtType, Mbtiles, MbtilesTranscoder, Metadata};
 use mlt_core::encoder::EncoderConfig;
-use pmtiles::{PmTilesWriter, TileCoord, TileType};
+use pmtiles::{Compression, PmTilesWriter, TileCoord, TileType};
 use size_format::SizeFormatterSI;
 use usize_cast::FromUsize as _;
 
 use super::common::{
     ENCODE_CACHE_BYTES, EncodedTile, MAX_TILE_CACHE_TRACK_SIZE_BYTES, PmTilesGeography, TileStats,
     encode_tile, make_encode_cache, make_progress_bar,
 };
-use super::{ContainerFormat, MbtFormat, encode_one};
+use super::{ContainerFormat, MbtFormat, encode_one, update_mlt_pmtiles_metadata};
 
 fn geography_from_metadata(metadata: &Metadata) -> PmTilesGeography {
     let tilejson = &metadata.tilejson;
@@ -35,12 +35,15 @@ pub async fn convert(
     output: (&Path, ContainerFormat),
     cfg: EncoderConfig,
     mbtiles_format: Option<MbtFormat>,
+    tile_compression: Compression,
 ) -> AnyResult<()> {
     match output {
         (output, ContainerFormat::Mbtiles) => {
             convert_mbtiles_to_mbtiles(input, output, mbtiles_format, cfg).await
         }
-        (output, ContainerFormat::Pmtiles) => convert_mbtiles_to_pmtiles(input, output, cfg).await,
+        (output, ContainerFormat::Pmtiles) => {
+            convert_mbtiles_to_pmtiles(input, output, cfg, tile_compression).await
+        }
         (output, ContainerFormat::Files) => bail!(
             "Output must be either an .mbtiles or a .pmtiles file when input is an .mbtiles file, got: {}",
             output.display()
@@ -108,6 +111,7 @@ async fn convert_mbtiles_to_mbtiles(
             .bytes_in
             .fetch_add(u64::from_usize(data.len()), Ordering::Relaxed);
         let result = encode_one(data, encoding, cfg)
+            .map(|(data, _raw_mvt_size)| data)
             .map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { e.to_string().into() });
         if let Ok(ref encoded) = result {
             sizes_ref
@@ -155,24 +159,28 @@ async fn convert_mbtiles_to_pmtiles(
     input: &Path,
     output: &Path,
     cfg: EncoderConfig,
+    tile_compression: Compression,
 ) -> AnyResult<()> {
     // FIXME: add a fastpath for normalised schemas. We don't need to cache them
-    let (encoding, _, mut metadata, total) = get_metadata(input).await?;
+    let (encoding, _, metadata, total) = get_metadata(input).await?;
+    let input_archive_size = std::fs::metadata(input)?.len();
 
     eprintln!("{} -> {} (pmtiles):", input.display(), output.display());
 
     let start = Instant::now();
     let bar = make_progress_bar(total);
 
     let geography = geography_from_metadata(&metadata);
-    metadata.tilejson.other.insert(
-        "format".into(),
-        serde_json::Value::String(Format::Mlt.metadata_format_value().into()),
-    );
     let file = std::fs::File::create(output)?;
-    let metadata_str = serde_json::to_string(&metadata.tilejson)?;
+    let mut metadata_json = serde_json::to_value(&metadata.tilejson)?;
+    let metadata_obj = metadata_json
+        .as_object_mut()
+        .ok_or_else(|| anyhow!("MBTiles metadata must serialize to a JSON object"))?;
+    update_mlt_pmtiles_metadata(metadata_obj, tile_compression);
+    let metadata_str = serde_json::to_string(&metadata_json)?;
     let mut stream_writer = geography
         .apply(PmTilesWriter::new(TileType::Mlt))
+        .tile_compression(tile_compression)
         .metadata(&metadata_str)
         .create(file)?;
 
@@ -199,12 +207,11 @@ async fn convert_mbtiles_to_pmtiles(
         .map(|(coord, data)| {
             let cache = cache.clone();
             tokio::task::spawn_blocking(move || -> AnyResult<EncodedTile> {
-                let bytes_in = data.len() as u64;
-                let (data, hit) = encode_tile(&cache, &data, encoding, cfg)?;
+                let (data, raw_mvt_size, hit) = encode_tile(&cache, &data, encoding, cfg)?;
                 Ok(EncodedTile {
                     coord,
                     data,
-                    bytes_in,
+                    raw_mvt_size,
                     hit,
                 })
             })
@@ -217,17 +224,24 @@ async fn convert_mbtiles_to_pmtiles(
         let EncodedTile {
             coord,
             data,
-            bytes_in,
+            raw_mvt_size,
             hit,
         } = joined??;
         stream_writer.add_tile(coord, &data)?;
-        stats.record(data.len() as u64, bytes_in, hit);
+        stats.record(data.len() as u64, raw_mvt_size, hit);
         bar.inc(1);
     }
 
     stream_writer.finalize()?;
+    let output_archive_size = std::fs::metadata(output)?.len();
     bar.finish_and_clear();
-    stats.print_summary(start);
+    stats.print_summary(
+        start,
+        input_archive_size,
+        output_archive_size,
+        encoding,
+        tile_compression,
+    );
 
     Ok(())
 }