Fix align_chunks xarray backcompat + add min version tests (#1299)

* CI: min versions test on ci

* CI: use matrix

* CI: simplify

* include zarr in min

* xarray backcompat

* warnigns and mypy

* try higher zarr

* back to 3.0.8

* zarr versison for backend + less verbose logging

* skip problematic testss

* warnings that shouldn't fail us

* always use correct xarray tag

* asssume instead of skip

* zero padding in xarray version

* clarify naming

Author: ianhi

.github/workflows/python-check.yaml

@@ -16,7 +16,7 @@ permissions:
   contents: read
 
 env:
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: "3.11"
 
 defaults:
   run:
@@ -37,6 +37,8 @@ jobs:
           working-directory: icechunk-python
           args: --release --out dist --find-interpreter
           sccache: true
+        env:
+          CARGO_TERM_QUIET: true
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4
@@ -47,13 +49,25 @@ jobs:
   test:
     runs-on: ubuntu-latest
     needs: [build-wheels]
+    strategy:
+      matrix:
+        deps-version:
+          - name: minimum
+            xarray: "2025.01.2"
+            dask: "2024.11.0"
+            distributed: "2024.11.0"
+            zarr: "3.0.8"
+          - name: latest
+            xarray: "latest-release"
+            dask: "latest-release"
+            distributed: "latest-release"
+            zarr: "latest-release"
     steps:
       - uses: actions/checkout@v5
       - name: Stand up MinIO
         run: |
           docker compose up -d minio
 
-
       - name: Download wheels
         uses: actions/download-artifact@v5
         with:
@@ -75,7 +89,7 @@ jobs:
           enable-cache: true
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Install dependencies
+      - name: Install dependencies - ${{ matrix.deps-version.name }}
         shell: bash
         working-directory: icechunk-python
         run: |
@@ -85,6 +99,15 @@ jobs:
           python --version
           WHEEL=$(ls dist/*-cp311-*.whl)
           uv pip install "${WHEEL}[test]"
+          # Install specific versions based on matrix
+          if [ "${{ matrix.deps-version.name }}" = "minimum" ]; then
+            echo "Installing minimum versions:"
+            uv pip install --upgrade 'xarray==${{ matrix.deps-version.xarray }}' 'dask==${{ matrix.deps-version.dask }}' 'distributed==${{ matrix.deps-version.distributed }}' 'zarr==${{ matrix.deps-version.zarr }}'
+          else
+            echo "Using latest versions (already installed)"
+          fi
+          echo "Installed package versions:"
+          uv pip list | grep -E "(xarray|dask|distributed|zarr)"
 
       # this is here instead of earlier to give minio more time to get up and running
       # checking and waiting only right before we need it
@@ -114,11 +137,20 @@ jobs:
         uses: actions/cache/save@v4
         with:
           path: icechunk-python/.hypothesis/
-          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
+          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}-${{ matrix.deps-version.name }}
 
   xarray-backends:
     runs-on: ubuntu-latest
     needs: [build-wheels]
+    strategy:
+      matrix:
+        xarray-version:
+          - name: minimum
+            version: "2025.01.2"
+            zarr: "3.0.8"
+          - name: latest-release
+            version: "latest-release"
+            zarr: "latest-release"
     steps:
       - uses: actions/checkout@v5
         with:
@@ -129,13 +161,6 @@ jobs:
         run: |
           docker compose up -d minio
 
-
-      - uses: actions/checkout@v5
-        with:
-          repository: "pydata/xarray"
-          path: "xarray"
-          fetch-depth: 0 # Fetch all history for all branches and tags.
-
       - name: Download wheels
         uses: actions/download-artifact@v5
         with:
@@ -161,9 +186,39 @@ jobs:
           WHEEL=$(ls dist/*-cp311-*.whl)
           uv pip install "${WHEEL}[test]"
           uv pip install pytest-mypy-plugins
+          # Install specific xarray version based on matrix
+          if [ "${{ matrix.xarray-version.version }}" != "latest-release" ]; then
+            echo "Installing xarray ${{ matrix.xarray-version.version }}"
+            uv pip install --upgrade 'xarray==${{ matrix.xarray-version.version }}'
+          fi
+          # Install specific zarr version based on matrix
+          if [ "${{ matrix.xarray-version.zarr }}" != "latest-release" ]; then
+            echo "Installing zarr ${{ matrix.xarray-version.zarr }}"
+            uv pip install --upgrade 'zarr==${{ matrix.xarray-version.zarr }}'
+          fi
+          echo "Installed package versions:"
+          python -c "import xarray; import zarr; print(f'xarray: {xarray.__version__}'); print(f'zarr: {zarr.__version__}')"
+
+      - name: Checkout xarray at installed version
+        shell: bash
+        working-directory: icechunk/icechunk-python
+        run: |
+          set -e
+          source .venv/bin/activate
+          XARRAY_VERSION=$(python -c "import xarray; print(xarray.__version__)")
+          # Convert version to tag format (e.g., 2025.1.2 -> v2025.01.2)
+          XARRAY_TAG=$(python -c "
+          import xarray
+          v = xarray.__version__
+          parts = v.split('.')
+          # Zero-pad month to 2 digits
+          tag = f'v{parts[0]}.{parts[1].zfill(2)}.{parts[2]}'
+          print(tag)
+          ")
+          echo "Checking out xarray ${XARRAY_TAG} (version ${XARRAY_VERSION})"
+          cd ../../
+          git clone --depth 1 --branch "${XARRAY_TAG}" https://github.com/pydata/xarray.git xarray
 
-      # this is here instead of earlier to give minio more time to get up and running
-      # checking and waiting only right before we need it
       - name: Wait for MinIO to be ready
         working-directory: icechunk
         run: |
@@ -179,7 +234,6 @@ jobs:
         working-directory: icechunk/icechunk-python
         env:
           ICECHUNK_XARRAY_BACKENDS_TESTS: 1
-          ICECHUNK_LOG: trace
         run: |
           set -e
           # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture

icechunk-python/pyproject.toml

@@ -144,9 +144,12 @@ filterwarnings = [
   "error",
   "ignore:Jupyter is migrating its paths to use:DeprecationWarning:",
   "ignore:Port 8787 is already in use:UserWarning:",
-  # TODO: this is raised for vlen-utf8, consolidated metadata, U1 dtype
+  # TODO: this is raised for vlen-utf8, vlen-bytes, consolidated metadata, U1 dtype, S1 dtype
   "ignore:The codec `vlen-utf8` is currently not part in the Zarr format 3 specification.",
+  "ignore:The codec `vlen-bytes` is currently not part in the Zarr format 3 specification.",
   "ignore:The dtype `<U1` is currently not part in the Zarr format 3 specification.",
+  "ignore:The dtype `\\|S1` is currently not part in the Zarr format 3 specification.",
+  "ignore:Consolidated metadata is currently not part in the Zarr format 3 specification.",
   "ignore::ResourceWarning",
   "ignore:Unused async fixture loop scope:pytest.PytestWarning",
   "ignore:.*does not have a Zarr V3 specification.*",

icechunk-python/python/icechunk/xarray.py

@@ -117,20 +117,26 @@ def _open_group(
             mode=mode, append_dim=append_dim, region=region
         )
 
-        self.xarray_store = ZarrStore.open_group(
-            store=self.store,
-            group=group,
-            mode=concrete_mode,
-            zarr_format=3,
-            append_dim=append_dim,
-            write_region=region,
-            safe_chunks=self.safe_chunks,
-            align_chunks=self.align_chunks,
-            synchronizer=None,
-            consolidated=False,
-            consolidate_on_close=False,
-            zarr_version=None,
-        )
+        # align_chunks was added in xarray 2025.06.0
+        # For backwards compatibility, only pass it if supported
+        kwargs: dict[str, Any] = {
+            "store": self.store,
+            "group": group,
+            "mode": concrete_mode,
+            "zarr_format": 3,
+            "append_dim": append_dim,
+            "write_region": region,
+            "safe_chunks": self.safe_chunks,
+            "synchronizer": None,
+            "consolidated": False,
+            "consolidate_on_close": False,
+            "zarr_version": None,
+        }
+
+        if Version(xr.__version__) >= Version("2025.06.0"):
+            kwargs["align_chunks"] = self.align_chunks
+
+        self.xarray_store = ZarrStore.open_group(**kwargs)
         self.dataset = self.xarray_store._validate_and_autodetect_region(self.dataset)
 
     def write_metadata(self, encoding: Mapping[Any, Any] | None = None) -> None:

icechunk-python/tests/test_zarr/test_properties.py

@@ -2,14 +2,15 @@
 
 import pytest
 from numpy.testing import assert_array_equal
+from packaging.version import Version
 
 from icechunk import IcechunkStore, Repository, in_memory_storage
 
 pytest.importorskip("hypothesis")
-
 import hypothesis.strategies as st
 from hypothesis import assume, given, settings
 
+import zarr
 from zarr.testing.strategies import arrays, numpy_arrays
 
 
@@ -27,6 +28,11 @@ def test_roundtrip(data: st.DataObject, nparray: Any) -> None:
     # TODO: support size-0 arrays GH392
     assume(nparray.size > 0)
 
+    # Skip bytes, unicode string, and datetime dtypes with zarr < 3.1.0
+    # These have codec/dtype issues that were fixed in 3.1.0's dtype refactor
+    if Version(zarr.__version__) < Version("3.1.0"):
+        assume(nparray.dtype.kind not in ("S", "U", "M", "m"))
+
     zarray = data.draw(
         arrays(
             stores=icechunk_stores,

icechunk-python/tests/test_zarr/test_stateful.py

@@ -15,6 +15,7 @@
     rule,
     run_state_machine_as_test,
 )
+from packaging.version import Version
 
 import icechunk as ic
 import zarr
@@ -205,6 +206,10 @@ def add_array(
         array, _ = array_and_chunks
         # TODO: support size-0 arrays GH392
         assume(array.size > 0)
+        # Skip bytes, unicode string, and datetime dtypes with zarr < 3.1.0
+        # These have codec/dtype issues that were fixed in 3.1.0's dtype refactor
+        if Version(zarr.__version__) < Version("3.1.0"):
+            assume(array.dtype.kind not in ("S", "U", "M", "m"))
         super().add_array(data, name, array_and_chunks)
 
     @precondition(lambda self: bool(self.all_groups))