chore(ci): migrate Linux CI to runs-on self-hosted runners #5965
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Check | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'support/**' | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| merge_group: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| defaults: | |
| run: | |
| working-directory: ./icechunk-python | |
| jobs: | |
| build-wheels: | |
| runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 | |
| with: | |
| working-directory: icechunk-python | |
| args: --release --out dist -i python${{ env.PYTHON_VERSION }} | |
| sccache: true | |
| manylinux: auto | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| test: | |
| runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64 | |
| needs: [build-wheels] | |
| strategy: | |
| matrix: | |
| deps-version: | |
| - name: minimum | |
| xarray: "2025.07.1" | |
| dask: "2025.2.0" | |
| distributed: "2025.2.0" | |
| zarr: "3.1.0" | |
| - name: latest | |
| xarray: "latest-release" | |
| dask: "latest-release" | |
| distributed: "latest-release" | |
| zarr: "latest-release" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Stand up RustFS | |
| run: | | |
| docker compose up -d rustfs_init | |
| - name: Download wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| - name: Restore cached hypothesis directory | |
| id: restore-hypothesis-cache | |
| uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| restore-keys: | | |
| cache-hypothesis- | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies - ${{ matrix.deps-version.name }} | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| python --version | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| # Install specific versions based on matrix | |
| if [ "${{ matrix.deps-version.name }}" = "minimum" ]; then | |
| echo "Installing minimum versions:" | |
| uv pip install --upgrade 'xarray==${{ matrix.deps-version.xarray }}' 'dask==${{ matrix.deps-version.dask }}' 'distributed==${{ matrix.deps-version.distributed }}' 'zarr==${{ matrix.deps-version.zarr }}' | |
| else | |
| echo "Using latest versions (already installed)" | |
| fi | |
| echo "Installed package versions:" | |
| uv pip list | grep -E "(xarray|dask|distributed|zarr)" | |
| # this is here instead of earlier to give RustFS more time to get up and running | |
| # checking and waiting only right before we need it | |
| - name: Wait for RustFS to be ready | |
| run: | | |
| for _ in {1..30}; do | |
| if docker compose ps --status exited --filter status==0 | grep rustfs ; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Install flatc | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y flatbuffers-compiler | |
| - name: Run pytest (non-hypothesis) | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| python -m pytest -n 4 -m "not hypothesis" | |
| build-wheels-free-threaded: | |
| runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13t" | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| working-directory: icechunk-python | |
| args: --release --out dist -i python3.13t | |
| sccache: true | |
| manylinux: auto | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-wheels-free-threaded | |
| path: icechunk-python/dist | |
| test-free-threaded: | |
| runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64 | |
| needs: [build-wheels-free-threaded] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Stand up RustFS | |
| run: | | |
| docker compose up -d rustfs_init | |
| - name: Download wheels | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: test-wheels-free-threaded | |
| path: icechunk-python/dist | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: "3.13t" | |
| - name: Install dependencies | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| python --version | |
| WHEEL=$(ls dist/*-cp313t-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| - name: Wait for RustFS to be ready | |
| run: | | |
| for _ in {1..30}; do | |
| if docker compose ps --status exited --filter status==0 | grep rustfs ; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Install flatc | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y flatbuffers-compiler | |
| - name: Run pytest (non-hypothesis) | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| python -m pytest -n 4 -m "not hypothesis" | |
| test-hypothesis-shard: | |
| runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64 | |
| needs: [build-wheels] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard-id: [0, 1, 2] | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Download wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| - name: Restore cached hypothesis directory | |
| id: restore-hypothesis-cache | |
| uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-shard${{ matrix.shard-id }}-${{ github.run_id }} | |
| restore-keys: | | |
| cache-hypothesis-${{ runner.os }}-shard${{ matrix.shard-id }}- | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| python --version | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| uv pip install pytest-shard | |
| - name: Run hypothesis tests (shard ${{ matrix.shard-id }}/3) | |
| shell: bash | |
| working-directory: icechunk-python | |
| env: | |
| HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY: "1" | |
| NOCOLOR: 1 | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| python -m pytest -m hypothesis --num-shards=3 --shard-id=${{ matrix.shard-id }} | |
| - name: Save cached hypothesis directory | |
| id: save-hypothesis-cache | |
| if: always() | |
| uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-shard${{ matrix.shard-id }}-${{ github.run_id }} | |
| # Aggregation job preserves the required "test-hypothesis" check name | |
| test-hypothesis: | |
| if: always() | |
| needs: [test-hypothesis-shard] | |
| runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64 | |
| defaults: | |
| run: | |
| working-directory: . | |
| steps: | |
| - name: Verify all shards passed | |
| run: | | |
| if [ "${{ needs.test-hypothesis-shard.result }}" != "success" ]; then | |
| echo "::error::One or more hypothesis test shards failed" | |
| exit 1 | |
| fi | |
| compat-stateful: | |
| runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64 | |
| needs: [build-wheels] | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Download wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install icechunk + deps + icechunk_v1 | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| uv run third-wheel sync -v | |
| - name: Run cross-version stateful tests | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| python -m pytest tests/test_stateful_compat.py -v | |
| xarray-backends: | |
| runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64 | |
| needs: [build-wheels] | |
| strategy: | |
| matrix: | |
| xarray-version: | |
| - name: minimum | |
| version: "2025.07.1" | |
| zarr: "3.1.0" | |
| - name: latest-release | |
| version: "latest-release" | |
| zarr: "latest-release" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| path: "icechunk" | |
| - name: Stand up RustFS | |
| working-directory: icechunk | |
| run: | | |
| docker compose up -d rustfs_init | |
| - name: Download wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: test-wheels | |
| path: icechunk/icechunk-python/dist | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| ICECHUNK_XARRAY_BACKENDS_TESTS: 1 | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| python --version | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group test | |
| uv pip install "pytest-mypy-plugins<4" | |
| # Install specific xarray version based on matrix | |
| if [ "${{ matrix.xarray-version.version }}" != "latest-release" ]; then | |
| echo "Installing xarray ${{ matrix.xarray-version.version }}" | |
| uv pip install --upgrade 'xarray==${{ matrix.xarray-version.version }}' | |
| fi | |
| # Install specific zarr version based on matrix | |
| if [ "${{ matrix.xarray-version.zarr }}" != "latest-release" ]; then | |
| echo "Installing zarr ${{ matrix.xarray-version.zarr }}" | |
| uv pip install --upgrade 'zarr==${{ matrix.xarray-version.zarr }}' | |
| fi | |
| echo "Installed package versions:" | |
| python -c "import xarray; import zarr; print(f'xarray: {xarray.__version__}'); print(f'zarr: {zarr.__version__}')" | |
| - name: Checkout xarray at installed version | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| run: | | |
| set -e | |
| source .venv/bin/activate | |
| XARRAY_VERSION=$(python -c "import xarray; print(xarray.__version__)") | |
| # Convert version to tag format (e.g., 2025.1.2 -> v2025.01.2) | |
| XARRAY_TAG=$(python -c " | |
| import xarray | |
| v = xarray.__version__ | |
| parts = v.split('.') | |
| # Zero-pad month to 2 digits | |
| tag = f'v{parts[0]}.{parts[1].zfill(2)}.{parts[2]}' | |
| print(tag) | |
| ") | |
| echo "Checking out xarray ${XARRAY_TAG} (version ${XARRAY_VERSION})" | |
| cd ../../ | |
| git clone --depth 1 --branch "${XARRAY_TAG}" https://github.com/pydata/xarray.git xarray | |
| - name: Wait for RustFS to be ready | |
| working-directory: icechunk | |
| run: | | |
| for _ in {1..30}; do | |
| if docker compose ps --status exited --filter status==0 | grep rustfs ; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Run xarray backends tests | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| ICECHUNK_XARRAY_BACKENDS_TESTS: 1 | |
| run: | | |
| set -e | |
| # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture | |
| source .venv/bin/activate | |
| python -m pytest -c=../../xarray/pyproject.toml -W ignore --override-ini="strict_markers=false" tests/run_xarray_backends_tests.py | |
| stubtest: | |
| runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64 | |
| needs: [build-wheels] | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Download wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: test-wheels | |
| path: icechunk-python/dist | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Run stubtest | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| uv venv | |
| source .venv/bin/activate | |
| PYTHON_TAG="cp$(echo ${{ env.PYTHON_VERSION }} | tr -d '.')" | |
| WHEEL=$(ls dist/*-${PYTHON_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group dev | |
| python -m mypy.stubtest --ignore-disjoint-bases icechunk._icechunk_python --allowlist stubtest_allowlist.txt | |
| check-xarray-docs: | |
| runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| path: "icechunk" | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| repository: "pydata/xarray" | |
| path: "xarray" | |
| fetch-depth: 0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| enable-cache: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Check xarray documentation consistency | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| XARRAY_DIR: ../../xarray | |
| run: | | |
| set -e | |
| uv run scripts/check_xarray_docs_sync.py |