chore(ci): migrate Linux CI to runs-on self-hosted runners #4664
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Check - Upstream | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - "support/**" | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| merge_group: | |
| schedule: | |
| - cron: "0 0 * * *" # Daily “At 00:00” UTC | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| issues: write | |
| env: | |
| PYTHON_VERSION: "3.14" | |
| defaults: | |
| run: | |
| working-directory: ./icechunk-python | |
| jobs: | |
| build: | |
| name: upstream-dev | |
| runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache | |
| continue-on-error: true | |
| if: ${{ | |
| (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') | |
| || github.event_name == 'workflow_dispatch' | |
| || github.event_name == 'schedule' | |
| }} | |
| steps: | |
| - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Install Just | |
| uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # v2 | |
| with: | |
| tool: just | |
| - name: Stand up RustFS | |
| run: | | |
| docker compose up -d rustfs_init | |
| - name: Wait for RustFS to be ready | |
| run: | | |
| for _ in {1..10}; do | |
| if docker compose ps --status exited --filter status==0 | grep rustfs ; then | |
| break | |
| fi | |
| sleep 3 | |
| done | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 | |
| with: | |
| working-directory: icechunk-python | |
| # target: ${{ matrix.platform.target }} | |
| args: --release --out dist --find-interpreter | |
| sccache: true | |
| # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245 | |
| - name: setup | |
| id: setup | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python --version | |
| PY_TAG="cp${PYTHON_VERSION//./}" | |
| WHEEL=$(ls dist/*-${PY_TAG}-*.whl) | |
| # Install upstream dependencies from nightly wheels | |
| export UV_INDEX="https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" | |
| export UV_PRERELEASE=allow | |
| uv pip install "$WHEEL" --group dev \ | |
| --resolution highest \ | |
| --index-strategy unsafe-best-match 2>&1 | tee setup-output.log | |
| uv pip install "hypothesis @ git+https://github.com/ianhi/hypothesis.git@flaky-feedback#subdirectory=hypothesis-python" | |
| uv pip list | |
| - name: Create or update setup failure issue | |
| if: | | |
| always() | |
| && steps.setup.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| shell: bash | |
| working-directory: . | |
| run: | | |
| # Read the template and setup output | |
| template=$(cat .github/setup-failure-template.md) | |
| setup_output="No setup output captured" | |
| if [ -f "icechunk-python/setup-output.log" ]; then | |
| setup_output=$(cat icechunk-python/setup-output.log) | |
| fi | |
| # Replace placeholders | |
| issue_body="${template//\{\{WORKFLOW\}\}/${{ github.workflow }}}" | |
| issue_body="${issue_body//\{\{RUN_ID\}\}/${{ github.run_id }}}" | |
| issue_body="${issue_body//\{\{RUN_URL\}\}/${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}}" | |
| issue_body="${issue_body//\{\{DATE\}\}/$(date -u)}" | |
| issue_body="${issue_body//\{\{SETUP_OUTPUT\}\}/$setup_output}" | |
| # Check for existing open issue with same title | |
| issue_title="Nightly upstream dependency installation failed" | |
| existing_issue=$(gh issue list --state open --label "CI" --search "\"$issue_title\" in:title" --json number --jq '.[0].number // empty') | |
| if [ -n "$existing_issue" ]; then | |
| echo "Found existing open issue #$existing_issue, updating it..." | |
| echo "$issue_body" | gh issue edit "$existing_issue" --body-file - | |
| echo "Updated existing issue #$existing_issue" | |
| else | |
| echo "No existing open issue found, creating new one..." | |
| echo "$issue_body" | gh issue create \ | |
| --title "$issue_title" \ | |
| --body-file - \ | |
| --label "CI" | |
| echo "Created new issue" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: mypy | |
| id: mypy | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| mypy --python-version ${{ env.PYTHON_VERSION }} python 2>&1 | tee mypy-output.log | |
| - name: Create or update mypy failure issue | |
| if: | | |
| always() | |
| && steps.mypy.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| shell: bash | |
| working-directory: . | |
| run: | | |
| # Read the template and mypy output | |
| template=$(cat .github/mypy-failure-template.md) | |
| mypy_output="No mypy output captured" | |
| if [ -f "icechunk-python/mypy-output.log" ]; then | |
| mypy_output=$(cat icechunk-python/mypy-output.log) | |
| fi | |
| # Replace placeholders | |
| issue_body="${template//\{\{WORKFLOW\}\}/${{ github.workflow }}}" | |
| issue_body="${issue_body//\{\{RUN_ID\}\}/${{ github.run_id }}}" | |
| issue_body="${issue_body//\{\{RUN_URL\}\}/${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}}" | |
| issue_body="${issue_body//\{\{DATE\}\}/$(date -u)}" | |
| issue_body="${issue_body//\{\{MYPY_OUTPUT\}\}/$mypy_output}" | |
| # Check for existing open issue with same title | |
| issue_title="Nightly MyPy type checking failed with upstream dependencies" | |
| existing_issue=$(gh issue list --state open --label "CI" --search "\"$issue_title\" in:title" --json number --jq '.[0].number // empty') | |
| if [ -n "$existing_issue" ]; then | |
| echo "Found existing open issue #$existing_issue, updating it..." | |
| echo "$issue_body" | gh issue edit "$existing_issue" --body-file - | |
| echo "Updated existing issue #$existing_issue with latest mypy output" | |
| else | |
| echo "No existing open issue found, creating new one..." | |
| echo "$issue_body" | gh issue create \ | |
| --title "$issue_title" \ | |
| --body-file - \ | |
| --label "CI" | |
| echo "Created new issue" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Restore cached hypothesis directory | |
| id: restore-hypothesis-cache | |
| if: always() | |
| uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| restore-keys: | | |
| cache-hypothesis- | |
| - name: describe environment | |
| if: steps.setup.outcome == 'success' | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip list | |
| - name: Install flatc | |
| if: steps.setup.outcome == 'success' | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y flatbuffers-compiler | |
| - name: pytest | |
| id: pytest-icechunk | |
| if: steps.setup.outcome == 'success' | |
| shell: bash | |
| working-directory: icechunk-python | |
| env: | |
| HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY: "1" | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pytest -n 4 --hypothesis-profile=nightly --report-log output-pytest-log.jsonl | |
| # explicitly save the cache so it gets updated, also do this even if it fails. | |
| - name: Save cached hypothesis directory | |
| id: save-hypothesis-cache | |
| if: always() | |
| uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| - name: Generate and publish the pytest report | |
| if: | | |
| always() | |
| && steps.pytest-icechunk.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1 | |
| with: | |
| log-path: icechunk-python/output-pytest-log.jsonl | |
| xarray-backends: | |
| name: xarray-tests-upstream | |
| runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache | |
| if: ${{ | |
| (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') | |
| || github.event_name == 'workflow_dispatch' | |
| || github.event_name == 'schedule' | |
| }} | |
| steps: | |
| - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| path: "icechunk" | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| repository: "pydata/xarray" | |
| path: "xarray" | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 | |
| with: | |
| working-directory: icechunk/icechunk-python | |
| args: --release --out dist --find-interpreter | |
| - name: setup | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python --version | |
| PY_TAG="cp${PYTHON_VERSION//./}" | |
| WHEEL=$(ls dist/*-${PY_TAG}-*.whl) | |
| # Install upstream dependencies from nightly wheels | |
| export UV_INDEX="https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" | |
| export UV_PRERELEASE=allow | |
| uv pip install "$WHEEL" --group test pytest-mypy-plugins \ | |
| --resolution highest \ | |
| --index-strategy unsafe-best-match | |
| uv pip list | |
| - name: Stand up RustFS | |
| working-directory: icechunk | |
| run: | | |
| docker compose up -d rustfs_init | |
| - name: Wait for RustFS to be ready | |
| working-directory: icechunk | |
| run: | | |
| for _ in {1..10}; do | |
| if docker compose ps --status exited --filter status==0 | grep rustfs ; then | |
| break | |
| fi | |
| sleep 3 | |
| done | |
| - name: pytest | |
| id: pytest-xarray-backends | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| ICECHUNK_XARRAY_BACKENDS_TESTS: 1 | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture | |
| pytest -c=../../xarray/pyproject.toml -W ignore tests/run_xarray_backends_tests.py --report-log output-pytest-log.jsonl | |
| - name: Generate and publish the xarray backends report | |
| if: | | |
| failure() | |
| && steps.pytest-xarray-backends.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1 | |
| with: | |
| log-path: icechunk/icechunk-python/output-pytest-log.jsonl | |
| issue-title: "Nightly Xarray backends tests failed" |