fix: skip exact byte comparison for gzip codec due to timestamp variance (zarr-developers#4270)

* fix: skip exact byte comparison for gzip codec due to timestamp variance

* fix: compare gzip streams ignoring mtime

---------

Co-authored-by: Davis Bennett <davis.v.bennett@gmail.com>

Author: sheikhayaan

src/zarr/codecs/gzip.py

@@ -28,6 +28,13 @@ def parse_gzip_level(data: JSON) -> int:
     return parsed
 
 
+def _gzip_streams_equal_except_mtime(a: bytes, b: bytes) -> bool:
+    if len(a) != len(b):
+        return False
+
+    return a[:4] == b[:4] and a[8:] == b[8:]
+
+
 @dataclass(frozen=True)
 class GzipCodec(BytesBytesCodec):
     """gzip codec"""

tests/test_codecs/test_gzip.py

@@ -5,6 +5,7 @@
 from zarr.abc.codec import SupportsSyncCodec
 from zarr.abc.store import Store
 from zarr.codecs import GzipCodec
+from zarr.codecs.gzip import _gzip_streams_equal_except_mtime
 from zarr.core.array_spec import ArrayConfig, ArraySpec
 from zarr.core.buffer import default_buffer_prototype
 from zarr.core.dtype import get_data_type_from_native_dtype
@@ -50,3 +51,39 @@ def test_gzip_codec_sync_roundtrip() -> None:
     decoded = codec._decode_sync(encoded, spec)
     result = np.frombuffer(decoded.as_numpy_array(), dtype="float64")
     np.testing.assert_array_equal(arr, result)
+
+
+def test_gzip_streams_equal_except_mtime() -> None:
+    prefix = b"\x1f\x8b\x08\x00"
+    mtime = b"\x01\x02\x03\x04"
+    suffix = b"\x00\xff\x10\x20"
+
+    # Identical streams are equal.
+    assert _gzip_streams_equal_except_mtime(
+        prefix + mtime + suffix,
+        prefix + mtime + suffix,
+    )
+
+    # Streams with different MTIME values are still equal.
+    assert _gzip_streams_equal_except_mtime(
+        prefix + b"\x01\x02\x03\x04" + suffix,
+        prefix + b"\x05\x06\x07\x08" + suffix,
+    )
+
+    # Differences after MTIME are detected.
+    assert not _gzip_streams_equal_except_mtime(
+        prefix + mtime + suffix,
+        prefix + mtime + b"\x00\xff\x10\x21",
+    )
+
+    # Differences before MTIME are detected.
+    assert not _gzip_streams_equal_except_mtime(
+        prefix + mtime + suffix,
+        b"\x1f\x8b\x09\x00" + mtime + suffix,
+    )
+
+    # Different lengths are detected.
+    assert not _gzip_streams_equal_except_mtime(
+        prefix + mtime + suffix,
+        prefix + mtime + suffix + b"\x00",
+    )

tests/test_fused_pipeline.py

@@ -18,7 +18,7 @@
 )
 from zarr.abc.store import Store, _store_supports_sync_io
 from zarr.codecs.bytes import BytesCodec
-from zarr.codecs.gzip import GzipCodec
+from zarr.codecs.gzip import GzipCodec, _gzip_streams_equal_except_mtime
 from zarr.codecs.transpose import TransposeCodec
 from zarr.codecs.zstd import ZstdCodec
 from zarr.core.codec_pipeline import FusedCodecPipeline
@@ -904,7 +904,19 @@ def test_async_chunk_transform_matches_sync(codecs: tuple[Any, ...]) -> None:
     async_bytes = asyncio.run(async_t.encode_chunk(value, spec))
     assert sync_bytes is not None
     assert async_bytes is not None
-    np.testing.assert_array_equal(async_bytes.to_bytes(), sync_bytes.to_bytes())
+
+    has_timestamp_codec = any(isinstance(c, GzipCodec) for c in evolved)
+
+    if has_timestamp_codec:
+        assert _gzip_streams_equal_except_mtime(
+            async_bytes.to_bytes(),
+            sync_bytes.to_bytes(),
+        )
+    else:
+        np.testing.assert_array_equal(
+            async_bytes.to_bytes(),
+            sync_bytes.to_bytes(),
+        )
 
     sync_arr = sync_t.decode_chunk(async_bytes, spec)
     async_arr = asyncio.run(async_t.decode_chunk(async_bytes, spec))