python-check.yaml

name: Python Check

on:
  push:
    branches:
      - main
      - 'support/**'
  pull_request:
    types: [opened, reopened, synchronize, labeled]
  merge_group:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  PYTHON_VERSION: "3.12"

defaults:
  run:
    working-directory: ./icechunk-python

jobs:
  build-wheels:
    runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Build wheels
        uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1
        with:
          working-directory: icechunk-python
          args: --release --out dist -i python${{ env.PYTHON_VERSION }}
          sccache: true
          manylinux: auto

      - name: Upload wheels
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-wheels
          path: icechunk-python/dist

  test:
    runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64
    needs: [build-wheels]
    strategy:
      matrix:
        deps-version:
          - name: minimum
            xarray: "2025.07.1"
            dask: "2025.2.0"
            distributed: "2025.2.0"
            zarr: "3.1.0"
          - name: latest
            xarray: "latest-release"
            dask: "latest-release"
            distributed: "latest-release"
            zarr: "latest-release"
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      - name: Stand up RustFS
        run: |
          docker compose up -d rustfs_init

      - name: Download wheels
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: test-wheels
          path: icechunk-python/dist

      - name: Restore cached hypothesis directory
        id: restore-hypothesis-cache
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
        with:
          path: icechunk-python/.hypothesis/
          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
          restore-keys: |
            cache-hypothesis-

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          enable-cache: true
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies - ${{ matrix.deps-version.name }}
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          uv venv
          source .venv/bin/activate
          python --version
          PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')"
          WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl)
          uv pip install "$WHEEL" --group test
          # Install specific versions based on matrix
          if [ "${{ matrix.deps-version.name }}" = "minimum" ]; then
            echo "Installing minimum versions:"
            uv pip install --upgrade 'xarray==${{ matrix.deps-version.xarray }}' 'dask==${{ matrix.deps-version.dask }}' 'distributed==${{ matrix.deps-version.distributed }}' 'zarr==${{ matrix.deps-version.zarr }}'
          else
            echo "Using latest versions (already installed)"
          fi
          echo "Installed package versions:"
          uv pip list | grep -E "(xarray|dask|distributed|zarr)"

      # this is here instead of earlier to give RustFS more time to get up and running
      # checking and waiting only right before we need it
      - name: Wait for RustFS to be ready
        run: |
          for _ in {1..30}; do
              if docker compose ps --status exited --filter status==0 | grep rustfs ; then
              break
              fi
              sleep 1
          done

      - name: Install flatc
        run: |
          sudo apt-get update && sudo apt-get install -y flatbuffers-compiler

      - name: Run pytest (non-hypothesis)
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          source .venv/bin/activate
          python -m pytest -n 4 -m "not hypothesis"

  build-wheels-free-threaded:
    runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.13t"

      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          working-directory: icechunk-python
          args: --release --out dist -i python3.13t
          sccache: true
          manylinux: auto

      - name: Upload wheels
        uses: actions/upload-artifact@v7
        with:
          name: test-wheels-free-threaded
          path: icechunk-python/dist

  test-free-threaded:
    runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64
    needs: [build-wheels-free-threaded]
    steps:
      - uses: actions/checkout@v6
      - name: Stand up RustFS
        run: |
          docker compose up -d rustfs_init

      - name: Download wheels
        uses: actions/download-artifact@v8
        with:
          name: test-wheels-free-threaded
          path: icechunk-python/dist

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          python-version: "3.13t"

      - name: Install dependencies
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          uv venv
          source .venv/bin/activate
          python --version
          WHEEL=$(ls dist/*-cp313t-*.whl)
          uv pip install "$WHEEL" --group test

      - name: Wait for RustFS to be ready
        run: |
          for _ in {1..30}; do
              if docker compose ps --status exited --filter status==0 | grep rustfs ; then
              break
              fi
              sleep 1
          done

      - name: Install flatc
        run: |
          sudo apt-get update && sudo apt-get install -y flatbuffers-compiler

      - name: Run pytest (non-hypothesis)
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          source .venv/bin/activate
          python -m pytest -n 4 -m "not hypothesis"

  test-hypothesis-shard:
    runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64
    needs: [build-wheels]
    strategy:
      fail-fast: false
      matrix:
        shard-id: [0, 1, 2]
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Download wheels
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: test-wheels
          path: icechunk-python/dist

      - name: Restore cached hypothesis directory
        id: restore-hypothesis-cache
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
        with:
          path: icechunk-python/.hypothesis/
          key: cache-hypothesis-${{ runner.os }}-shard${{ matrix.shard-id }}-${{ github.run_id }}
          restore-keys: |
            cache-hypothesis-${{ runner.os }}-shard${{ matrix.shard-id }}-

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          enable-cache: true
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          uv venv
          source .venv/bin/activate
          python --version
          PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')"
          WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl)
          uv pip install "$WHEEL" --group test
          uv pip install pytest-shard

      - name: Run hypothesis tests (shard ${{ matrix.shard-id }}/3)
        shell: bash
        working-directory: icechunk-python
        env:
          HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY: "1"
          NOCOLOR: 1
        run: |
          set -e
          source .venv/bin/activate
          python -m pytest -m hypothesis --num-shards=3 --shard-id=${{ matrix.shard-id }}

      - name: Save cached hypothesis directory
        id: save-hypothesis-cache
        if: always()
        uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
        with:
          path: icechunk-python/.hypothesis/
          key: cache-hypothesis-${{ runner.os }}-shard${{ matrix.shard-id }}-${{ github.run_id }}

  # Aggregation job preserves the required "test-hypothesis" check name
  test-hypothesis:
    if: always()
    needs: [test-hypothesis-shard]
    runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64
    defaults:
      run:
        working-directory: .
    steps:
      - name: Verify all shards passed
        run: |
          if [ "${{ needs.test-hypothesis-shard.result }}" != "success" ]; then
            echo "::error::One or more hypothesis test shards failed"
            exit 1
          fi

  compat-stateful:
    runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64
    needs: [build-wheels]
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Download wheels
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: test-wheels
          path: icechunk-python/dist

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          enable-cache: true
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install icechunk + deps + icechunk_v1
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          uv venv
          source .venv/bin/activate
          PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')"
          WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl)
          uv pip install "$WHEEL" --group test
          uv run third-wheel sync -v

      - name: Run cross-version stateful tests
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          source .venv/bin/activate
          python -m pytest tests/test_stateful_compat.py -v

  xarray-backends:
    runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64
    needs: [build-wheels]
    strategy:
      matrix:
        xarray-version:
          - name: minimum
            version: "2025.07.1"
            zarr: "3.1.0"
          - name: latest-release
            version: "latest-release"
            zarr: "latest-release"
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          path: "icechunk"

      - name: Stand up RustFS
        working-directory: icechunk
        run: |
          docker compose up -d rustfs_init

      - name: Download wheels
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: test-wheels
          path: icechunk/icechunk-python/dist

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          enable-cache: true
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        shell: bash
        working-directory: icechunk/icechunk-python
        env:
          ICECHUNK_XARRAY_BACKENDS_TESTS: 1
        run: |
          set -e
          uv venv
          source .venv/bin/activate
          python --version
          PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')"
          WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl)
          uv pip install "$WHEEL" --group test
          uv pip install "pytest-mypy-plugins<4"
          # Install specific xarray version based on matrix
          if [ "${{ matrix.xarray-version.version }}" != "latest-release" ]; then
            echo "Installing xarray ${{ matrix.xarray-version.version }}"
            uv pip install --upgrade 'xarray==${{ matrix.xarray-version.version }}'
          fi
          # Install specific zarr version based on matrix
          if [ "${{ matrix.xarray-version.zarr }}" != "latest-release" ]; then
            echo "Installing zarr ${{ matrix.xarray-version.zarr }}"
            uv pip install --upgrade 'zarr==${{ matrix.xarray-version.zarr }}'
          fi
          echo "Installed package versions:"
          python -c "import xarray; import zarr; print(f'xarray: {xarray.__version__}'); print(f'zarr: {zarr.__version__}')"

      - name: Checkout xarray at installed version
        shell: bash
        working-directory: icechunk/icechunk-python
        run: |
          set -e
          source .venv/bin/activate
          XARRAY_VERSION=$(python -c "import xarray; print(xarray.__version__)")
          # Convert version to tag format (e.g., 2025.1.2 -> v2025.01.2)
          XARRAY_TAG=$(python -c "
          import xarray
          v = xarray.__version__
          parts = v.split('.')
          # Zero-pad month to 2 digits
          tag = f'v{parts[0]}.{parts[1].zfill(2)}.{parts[2]}'
          print(tag)
          ")
          echo "Checking out xarray ${XARRAY_TAG} (version ${XARRAY_VERSION})"
          cd ../../
          git clone --depth 1 --branch "${XARRAY_TAG}" https://github.com/pydata/xarray.git xarray

      - name: Wait for RustFS to be ready
        working-directory: icechunk
        run: |
          for _ in {1..30}; do
              if docker compose ps --status exited --filter status==0 | grep rustfs ; then
              break
              fi
              sleep 1
          done
      - name: Run xarray backends tests
        shell: bash
        working-directory: icechunk/icechunk-python
        env:
          ICECHUNK_XARRAY_BACKENDS_TESTS: 1
        run: |
          set -e
          # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture
          source .venv/bin/activate
          python -m pytest -c=../../xarray/pyproject.toml -W ignore --override-ini="strict_markers=false" tests/run_xarray_backends_tests.py

  stubtest:
    runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64
    needs: [build-wheels]
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Download wheels
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: test-wheels
          path: icechunk-python/dist

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          enable-cache: true
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Run stubtest
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          uv venv
          source .venv/bin/activate
          PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')"
          WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl)
          uv pip install "$WHEEL" --group dev
          python -m mypy.stubtest --ignore-disjoint-bases icechunk._icechunk_python --allowlist stubtest_allowlist.txt

  check-xarray-docs:
    runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          path: "icechunk"

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: "pydata/xarray"
          path: "xarray"
          fetch-depth: 0

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          enable-cache: true
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Check xarray documentation consistency
        shell: bash
        working-directory: icechunk/icechunk-python
        env:
          XARRAY_DIR: ../../xarray
        run: |
          set -e
          uv run scripts/check_xarray_docs_sync.py