Skip to content

Commit eaeeae6

Browse files
committed
docs: clarify gzip MTIME handling in comparison helper and fused-pipeline test docs
Follow-up to zarr-developers#4270: document that _gzip_streams_equal_except_mtime skips bytes 4-8 (the RFC 1952 MTIME field of the standard 10-byte gzip header), and update the AsyncChunkTransform section comment and test docstring to note that byte-identity is expected except for gzip's MTIME header field. Assisted-by: ClaudeCode:claude-fable-5
1 parent 608c919 commit eaeeae6

2 files changed

Lines changed: 14 additions & 3 deletions

File tree

src/zarr/codecs/gzip.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ def parse_gzip_level(data: JSON) -> int:
2929

3030

3131
def _gzip_streams_equal_except_mtime(a: bytes, b: bytes) -> bool:
32+
"""Compare two gzip streams, ignoring the MTIME field of the header.
33+
34+
Per RFC 1952 the gzip header is [magic(2)][CM(1)][FLG(1)][MTIME(4)][XFL(1)][OS(1)],
35+
so bytes 4-8 are MTIME. The fixed offsets assume the standard 10-byte header
36+
with no FNAME/FEXTRA/FCOMMENT flags set, which holds here because numcodecs'
37+
``GZip.encode`` wraps ``gzip.GzipFile`` without a filename.
38+
"""
3239
if len(a) != len(b):
3340
return False
3441

tests/test_fused_pipeline.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -849,7 +849,9 @@ def test_sharded_roundtrip_with_async_only_inner_codec() -> None:
849849
# ---------------------------------------------------------------------------
850850
# AsyncChunkTransform: the async per-chunk codec chain used on the async
851851
# fallback path. It is the async mirror of ChunkTransform, so it must produce
852-
# identical bytes/arrays. The default (Fused, sync-store) path never uses it;
852+
# identical bytes/arrays — except for gzip's embedded 4-byte MTIME header
853+
# field, which the assertions deliberately ignore when a GzipCodec is present.
854+
# The default (Fused, sync-store) path never uses it;
853855
# these tests drive it directly over multi-codec chains so the aa/bb loops and
854856
# the all-fill drop branch are exercised.
855857
# ---------------------------------------------------------------------------
@@ -867,8 +869,10 @@ def test_sharded_roundtrip_with_async_only_inner_codec() -> None:
867869
)
868870
def test_async_chunk_transform_matches_sync(codecs: tuple[Any, ...]) -> None:
869871
"""`AsyncChunkTransform.decode_chunk`/`encode_chunk` must round-trip and
870-
produce exactly what the synchronous `ChunkTransform` produces, across
871-
array->array, array->bytes, and bytes->bytes codec combinations.
872+
produce what the synchronous `ChunkTransform` produces, across
873+
array->array, array->bytes, and bytes->bytes codec combinations. Output is
874+
byte-identical except for gzip's embedded 4-byte MTIME header field, which
875+
the assertion deliberately ignores when a GzipCodec is present.
872876
873877
This is the async mirror of the codecs the default pipeline runs
874878
synchronously; a divergence here corrupts data only on the async fallback

0 commit comments

Comments
 (0)