Merge pull request #82 from roteiro-gis/ian/dev

Deflate backend/levels and validation

Author: i-norden

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- add `deflate_level(0..=9)` to `ImageBuilder` and `GeoTiffBuilder` for controlling Deflate output size/speed, plus `compress_with_level` in `tiff-writer::compress`; `BlockEncodingOptions` gains a `deflate_level` field
+- speed up decode and write paths: switch the Deflate backend to the pure-Rust `zlib-rs` (~20-40% faster on predictor-compressed rasters), stream overview resampling by source row spans (~6x faster streaming COG average overviews), copy raster regions by row slice instead of per-element indexing (~4.5x faster plain multiband writes), resolve per-block decode metadata once per read, encode writer tag values once, and reuse the floating-point predictor scratch buffer across rows
 - pad one-shot COG edge tiles with the configured nodata fill value instead of zero, matching the streaming tile writer and overview generation paths
 - reject writer configurations that pair the horizontal predictor with float samples, the floating-point predictor with integer samples, or float sample formats below 32 bits, instead of emitting files other readers refuse
 - decode `BitsPerSample`/`SampleFormat` tags stored with nonstandard BYTE or LONG encodings instead of silently falling back to 1-bit defaults, and reject other unexpected tag types via new `Ifd::checked_bits_per_sample` / `Ifd::checked_sample_format` used by all decode paths

Cargo.lock

@@ -55,6 +55,7 @@ pub struct GeoTiffBuilder {
     pub(crate) predictor: Predictor,
     pub(crate) lerc_options: Option<tiff_writer::LercOptions>,
     pub(crate) jpeg_options: Option<JpegOptions>,
+    pub(crate) deflate_level: Option<u32>,
     pub(crate) extra_samples: Vec<ExtraSample>,
     pub(crate) color_map: Option<ColorMap>,
     pub(crate) ink_set: Option<InkSet>,
@@ -85,6 +86,7 @@ impl GeoTiffBuilder {
             predictor: Predictor::None,
             lerc_options: None,
             jpeg_options: None,
+            deflate_level: None,
             extra_samples: Vec::new(),
             color_map: None,
             ink_set: None,
@@ -297,6 +299,15 @@ impl GeoTiffBuilder {
         self
     }
 
+    /// Set the Deflate compression level (0-9).
+    ///
+    /// Applies to `Compression::Deflate` output. The additional Deflate layer
+    /// of `LERC+Deflate` always uses the codec default level.
+    pub fn deflate_level(mut self, level: u32) -> Self {
+        self.deflate_level = Some(level);
+        self
+    }
+
     /// Set predictor (requires compression != None).
     pub fn predictor(mut self, predictor: Predictor) -> Self {
         // LERC and JPEG do not use TIFF predictors; ignore the request.
@@ -580,6 +591,9 @@ impl GeoTiffBuilder {
         if let Some(opts) = self.jpeg_options {
             ib = ib.jpeg_options(opts);
         }
+        if let Some(level) = self.deflate_level {
+            ib = ib.deflate_level(level);
+        }
 
         if let (Some(tw), Some(th)) = (self.tile_width, self.tile_height) {
             ib = ib.tiles(tw, th);

fuzz/Cargo.lock

@@ -97,6 +97,7 @@ struct CogBlockEncoding {
     block_height: u32,
     lerc_options: Option<LercOptions>,
     jpeg_options: Option<JpegOptions>,
+    deflate_level: Option<u32>,
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -108,6 +109,7 @@ struct TileWritePlan {
     predictor: Predictor,
     lerc_options: Option<LercOptions>,
     jpeg_options: Option<JpegOptions>,
+    deflate_level: Option<u32>,
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -525,6 +527,7 @@ fn compress_cog_block<T: NumericSample>(
                 samples_per_pixel: encoding.samples_per_pixel,
                 row_width_pixels: encoding.row_width_pixels,
                 jpeg_options: encoding.jpeg_options.as_ref(),
+                deflate_level: encoding.deflate_level,
             },
             block_index,
         )
@@ -1034,6 +1037,7 @@ impl CogBuilder {
             predictor: self.inner.predictor,
             lerc_options: self.inner.lerc_options,
             jpeg_options: self.inner.jpeg_options,
+            deflate_level: self.inner.deflate_level,
         };
         let grid = RawTileGrid {
             tile_width: tw,
@@ -1115,6 +1119,7 @@ pub struct CogTileWriter<T: NumericSample, W: Write + Seek> {
     predictor: Predictor,
     lerc_options: Option<LercOptions>,
     jpeg_options: Option<JpegOptions>,
+    deflate_level: Option<u32>,
     overview_levels: Vec<u32>,
     resampling: Resampling,
     fill_value: T,
@@ -1159,6 +1164,7 @@ impl<T: NumericSample, W: Write + Seek> CogTileWriter<T, W> {
             predictor: cog.inner.predictor,
             lerc_options: cog.inner.lerc_options,
             jpeg_options: cog.inner.jpeg_options,
+            deflate_level: cog.inner.deflate_level,
             overview_levels,
             resampling: cog.resampling,
             fill_value,
@@ -1347,6 +1353,7 @@ impl<T: NumericSample, W: Write + Seek> CogTileWriter<T, W> {
             predictor: self.predictor,
             lerc_options: self.lerc_options,
             jpeg_options: self.jpeg_options,
+            deflate_level: self.deflate_level,
         };
         let grid = RawTileGrid {
             tile_width: tw,
@@ -1614,6 +1621,7 @@ where
                             block_height: plan.tile_height as u32,
                             lerc_options: plan.lerc_options,
                             jpeg_options: plan.jpeg_options,
+                            deflate_level: plan.deflate_level,
                         },
                     )?;
                 }
@@ -1637,6 +1645,7 @@ where
                         block_height: plan.tile_height as u32,
                         lerc_options: plan.lerc_options,
                         jpeg_options: plan.jpeg_options,
+                        deflate_level: plan.deflate_level,
                     },
                 )?;
             }
@@ -1698,6 +1707,7 @@ fn spool_base_blocks_from_store<T: NumericSample>(
                             block_height: plan.tile_height as u32,
                             lerc_options: plan.lerc_options,
                             jpeg_options: plan.jpeg_options,
+                            deflate_level: plan.deflate_level,
                         },
                     )?;
                 }
@@ -1725,6 +1735,7 @@ fn spool_base_blocks_from_store<T: NumericSample>(
                         block_height: plan.tile_height as u32,
                         lerc_options: plan.lerc_options,
                         jpeg_options: plan.jpeg_options,
+                        deflate_level: plan.deflate_level,
                     },
                 )?;
             }
@@ -1800,6 +1811,7 @@ fn spool_tiled_data_3d<T: NumericSample>(
                             block_height: th as u32,
                             lerc_options: plan.lerc_options,
                             jpeg_options: plan.jpeg_options,
+                            deflate_level: plan.deflate_level,
                         },
                     )?;
                 }
@@ -1835,6 +1847,7 @@ fn spool_tiled_data_3d<T: NumericSample>(
                         block_height: th as u32,
                         lerc_options: plan.lerc_options,
                         jpeg_options: plan.jpeg_options,
+                        deflate_level: plan.deflate_level,
                     },
                 )?;
             }

geotiff-writer/src/builder.rs

@@ -673,3 +673,54 @@ fn explicit_bigtiff_roundtrips_small_images() {
     assert_eq!(offset, Some(0));
     assert_eq!(values, vec![1, 2, 3, 4]);
 }
+
+#[test]
+fn deflate_level_controls_output_size_and_roundtrips() {
+    use std::io::Cursor;
+    use tiff_writer::{TiffWriter, WriteOptions};
+
+    let samples: Vec<u16> = (0..64 * 64)
+        .map(|index| ((index / 7) % 500) as u16)
+        .collect();
+
+    let mut sizes = Vec::new();
+    for level in [1u32, 9] {
+        let mut writer = TiffWriter::new(Cursor::new(Vec::new()), WriteOptions::default()).unwrap();
+        let handle = writer
+            .add_image(
+                ImageBuilder::new(64, 64)
+                    .sample_type::<u16>()
+                    .compression(Compression::Deflate)
+                    .deflate_level(level)
+                    .strips(64),
+            )
+            .unwrap();
+        writer.write_block(&handle, 0, &samples).unwrap();
+        let bytes = writer.finish().unwrap().into_inner();
+
+        let file = tiff_reader::TiffFile::from_bytes(bytes.clone()).unwrap();
+        let decoded = file.read_image::<u16>(0).unwrap();
+        assert_eq!(decoded.into_raw_vec_and_offset().0, samples);
+        sizes.push(bytes.len());
+    }
+    assert!(
+        sizes[1] <= sizes[0],
+        "level 9 output ({}) should not exceed level 1 output ({})",
+        sizes[1],
+        sizes[0]
+    );
+
+    let err = ImageBuilder::new(16, 16)
+        .compression(Compression::Deflate)
+        .deflate_level(12)
+        .validate()
+        .unwrap_err();
+    assert!(err.to_string().contains("deflate_level"), "{err}");
+
+    let err = ImageBuilder::new(16, 16)
+        .compression(Compression::Lzw)
+        .deflate_level(6)
+        .validate()
+        .unwrap_err();
+    assert!(err.to_string().contains("Deflate compression"), "{err}");
+}

geotiff-writer/src/cog.rs

@@ -19,7 +19,7 @@ lerc-reader = { workspace = true }
 ndarray = { workspace = true }
 thiserror = { workspace = true }
 memmap2 = { workspace = true }
-flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
+flate2 = { version = "1.0.35", default-features = false, features = ["zlib-rs"] }
 weezl = "0.1"
 lru = "0.16.3"
 parking_lot = "0.12"

integration-tests/tiff-integration/tests/tiff_writer_roundtrip.rs

@@ -287,8 +287,8 @@ fn read_bounded_to_end<R: Read>(
 ) -> Result<Vec<u8>> {
     let probe_limit = decoded_len_probe_limit(index, codec, decoded_len_limit)?;
     let mut reader = reader.take(probe_limit as u64);
-    let mut out = Vec::with_capacity(decoded_len_limit.min(8192));
-    let mut scratch = [0u8; 8192];
+    let mut out = Vec::with_capacity(decoded_len_limit.min(64 * 1024));
+    let mut scratch = vec![0u8; probe_limit.min(64 * 1024)];
 
     loop {
         let remaining = probe_limit.saturating_sub(out.len());

tiff-reader/Cargo.toml

@@ -17,7 +17,7 @@ tiff-core = { version = "0.7.0", path = "../tiff-core" }
 lerc-core = { workspace = true }
 lerc-writer = { workspace = true }
 thiserror = { workspace = true }
-flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
+flate2 = { version = "1.0.35", default-features = false, features = ["zlib-rs"] }
 weezl = "0.1"
 
 [features]

tiff-reader/src/filters.rs

@@ -70,6 +70,7 @@ pub struct ImageBuilder {
     pub(crate) subfile_type: u32,
     pub(crate) lerc_options: Option<LercOptions>,
     pub(crate) jpeg_options: Option<JpegOptions>,
+    pub(crate) deflate_level: Option<u32>,
 }
 
 impl ImageBuilder {
@@ -97,6 +98,7 @@ impl ImageBuilder {
             subfile_type: 0,
             lerc_options: None,
             jpeg_options: None,
+            deflate_level: None,
         }
     }
 
@@ -145,6 +147,16 @@ impl ImageBuilder {
         self
     }
 
+    /// Set the Deflate compression level (0-9).
+    ///
+    /// Applies to `Compression::Deflate`/`Compression::DeflateOld` blocks.
+    /// The additional Deflate layer of `LERC+Deflate` always uses the codec
+    /// default level.
+    pub fn deflate_level(mut self, level: u32) -> Self {
+        self.deflate_level = Some(level);
+        self
+    }
+
     pub fn photometric(mut self, p: PhotometricInterpretation) -> Self {
         self.photometric = p;
         self
@@ -756,6 +768,21 @@ impl ImageBuilder {
                     .into(),
             ));
         }
+        if let Some(level) = self.deflate_level {
+            if level > 9 {
+                return Err(crate::error::Error::InvalidConfig(format!(
+                    "deflate_level must be 0-9, got {level}"
+                )));
+            }
+            if !matches!(
+                self.compression,
+                Compression::Deflate | Compression::DeflateOld
+            ) {
+                return Err(crate::error::Error::InvalidConfig(
+                    "deflate_level requires Deflate compression".into(),
+                ));
+            }
+        }
         self.validate_color_model()?;
         if matches!(self.compression, Compression::Jpeg) {
             self.validate_jpeg_config()?;