Set up pixi for development #4458
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Check | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'support/**' | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| PYTHON_VERSION: "3.11" | |
| defaults: | |
| run: | |
| working-directory: ./icechunk-python | |
| jobs: | |
| build-wheels: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| working-directory: icechunk-python | |
| args: --release --out dist -i python${{ env.PYTHON_VERSION }} | |
| sccache: true | |
| manylinux: auto | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels] | |
| strategy: | |
| matrix: | |
| deps-version: | |
| - name: minimum | |
| xarray: "2025.07.1" | |
| dask: "2025.2.0" | |
| distributed: "2025.2.0" | |
| zarr: "3.1.0" | |
| - name: latest | |
| xarray: "latest-release" | |
| dask: "latest-release" | |
| distributed: "latest-release" | |
| zarr: "latest-release" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Stand up RustFS | |
| run: | | |
| docker compose up -d rustfs | |
| - name: Download wheels | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| - name: Restore cached hypothesis directory | |
| id: restore-hypothesis-cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| restore-keys: | | |
| cache-hypothesis- | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies - ${{ matrix.deps-version.name }} | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| python --version | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| # Install specific versions based on matrix | |
| if [ "${{ matrix.deps-version.name }}" = "minimum" ]; then | |
| echo "Installing minimum versions:" | |
| uv pip install --upgrade 'xarray==${{ matrix.deps-version.xarray }}' 'dask==${{ matrix.deps-version.dask }}' 'distributed==${{ matrix.deps-version.distributed }}' 'zarr==${{ matrix.deps-version.zarr }}' | |
| else | |
| echo "Using latest versions (already installed)" | |
| fi | |
| echo "Installed package versions:" | |
| uv pip list | grep -E "(xarray|dask|distributed|zarr)" | |
| # this is here instead of earlier to give RustFS more time to get up and running | |
| # checking and waiting only right before we need it | |
| - name: Wait for RustFS to be ready | |
| run: | | |
| for _ in {1..30}; do | |
| if curl --silent --fail http://rustfs:9000/health; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Run pytest | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| # Run tests | |
| python -m pytest -n 4 | |
| # explicitly save the cache so it gets updated, also do this even if it fails. | |
| - name: Save cached hypothesis directory | |
| id: save-hypothesis-cache | |
| if: always() | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}-${{ matrix.deps-version.name }} | |
| compat-stateful: | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download wheels | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install icechunk + deps + icechunk_v1 | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| uvx --from "git+https://github.com/earth-mover/third-wheel" \ | |
| third-wheel download icechunk --version ">=1,<2" \ | |
| --rename icechunk_v1 \ | |
| --python-version ${{ env.PYTHON_VERSION }} \ | |
| -o ./dist-v1/ | |
| uv pip install dist-v1/icechunk_v1-*.whl | |
| - name: Run cross-version stateful tests | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| python -m pytest tests/test_stateful_compat.py -v | |
| xarray-backends: | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels] | |
| strategy: | |
| matrix: | |
| xarray-version: | |
| - name: minimum | |
| version: "2025.07.1" | |
| zarr: "3.1.0" | |
| - name: latest-release | |
| version: "latest-release" | |
| zarr: "latest-release" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| path: "icechunk" | |
| - name: Stand up RustFS | |
| working-directory: icechunk | |
| run: | | |
| docker compose up -d rustfs | |
| - name: Download wheels | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: test-wheels | |
| path: icechunk/icechunk-python/dist | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| ICECHUNK_XARRAY_BACKENDS_TESTS: 1 | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| python --version | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| uv pip install pytest-mypy-plugins | |
| # Install specific xarray version based on matrix | |
| if [ "${{ matrix.xarray-version.version }}" != "latest-release" ]; then | |
| echo "Installing xarray ${{ matrix.xarray-version.version }}" | |
| uv pip install --upgrade 'xarray==${{ matrix.xarray-version.version }}' | |
| fi | |
| # Install specific zarr version based on matrix | |
| if [ "${{ matrix.xarray-version.zarr }}" != "latest-release" ]; then | |
| echo "Installing zarr ${{ matrix.xarray-version.zarr }}" | |
| uv pip install --upgrade 'zarr==${{ matrix.xarray-version.zarr }}' | |
| fi | |
| echo "Installed package versions:" | |
| python -c "import xarray; import zarr; print(f'xarray: {xarray.__version__}'); print(f'zarr: {zarr.__version__}')" | |
| - name: Checkout xarray at installed version | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| XARRAY_VERSION=$(python -c "import xarray; print(xarray.__version__)") | |
| # Convert version to tag format (e.g., 2025.1.2 -> v2025.01.2) | |
| XARRAY_TAG=$(python -c " | |
| import xarray | |
| v = xarray.__version__ | |
| parts = v.split('.') | |
| # Zero-pad month to 2 digits | |
| tag = f'v{parts[0]}.{parts[1].zfill(2)}.{parts[2]}' | |
| print(tag) | |
| ") | |
| echo "Checking out xarray ${XARRAY_TAG} (version ${XARRAY_VERSION})" | |
| cd ../../ | |
| git clone --depth 1 --branch "${XARRAY_TAG}" https://github.com/pydata/xarray.git xarray | |
| - name: Wait for RustFS to be ready | |
| working-directory: icechunk | |
| run: | | |
| for _ in {1..30}; do | |
| if curl --silent --fail http://rustfs:9000/health; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Run xarray backends tests | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| ICECHUNK_XARRAY_BACKENDS_TESTS: 1 | |
| run: | | |
| set -e | |
| # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture | |
| source .venv/bin/activate | |
| python -m pytest -c=../../xarray/pyproject.toml -W ignore --override-ini="strict_markers=false" tests/run_xarray_backends_tests.py | |
| stubtest: | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download wheels | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Run stubtest | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group dev | |
| python -m mypy.stubtest icechunk._icechunk_python --allowlist stubtest_allowlist.txt | |
| check-xarray-docs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| path: "icechunk" | |
| - uses: actions/checkout@v6 | |
| with: | |
| repository: "pydata/xarray" | |
| path: "xarray" | |
| fetch-depth: 0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Check xarray documentation consistency | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| XARRAY_DIR: ../../xarray | |
| run: | | |
| set -e | |
| uv run scripts/check_xarray_docs_sync.py |