python-upstream.yaml

name: Python Check - Upstream

on:
  push:
    branches:
      - main
      - "support/**"
  pull_request:
    types: [opened, reopened, synchronize, labeled]
  merge_group:
  schedule:
    - cron: "0 0 * * *" # Daily “At 00:00” UTC
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  issues: write

env:
  PYTHON_VERSION: "3.14"

defaults:
  run:
    working-directory: ./icechunk-python

jobs:
  build:
    name: upstream-dev
    runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache
    continue-on-error: true
    if: ${{
      (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request')
      || github.event_name == 'workflow_dispatch'
      || github.event_name == 'schedule'
      }}
    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2
        with:
          sccache: s3
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      - name: Install Just
        uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # v2
        with:
          tool: just
      - name: Stand up RustFS
        run: |
          docker compose up -d rustfs_init

      - name: Wait for RustFS to be ready
        run: |
          for _ in {1..10}; do
              if docker compose ps --status exited --filter status==0 | grep rustfs ; then
              break
              fi
              sleep 3
          done
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Build wheels
        uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1
        with:
          working-directory: icechunk-python
          # target: ${{ matrix.platform.target }}
          args: --release --out dist --find-interpreter
          sccache: true
          # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245

      - name: setup
        id: setup
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          python --version
          PY_TAG="cp${PYTHON_VERSION//./}"
          WHEEL=$(ls dist/*-${PY_TAG}-*.whl)
          # Install upstream dependencies from nightly wheels
          export UV_INDEX="https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/"
          export UV_PRERELEASE=allow
          uv pip install "$WHEEL" --group dev \
            --resolution highest \
            --index-strategy unsafe-best-match 2>&1 | tee setup-output.log
          uv pip install "hypothesis @ git+https://github.com/ianhi/hypothesis.git@flaky-feedback#subdirectory=hypothesis-python"
          uv pip list

      - name: Create or update setup failure issue
        if: |
          always()
          && steps.setup.outcome == 'failure'
          && github.event_name == 'schedule'
          && github.repository_owner == 'earth-mover'
        shell: bash
        working-directory: .
        run: |
          # Read the template and setup output
          template=$(cat .github/setup-failure-template.md)

          setup_output="No setup output captured"
          if [ -f "icechunk-python/setup-output.log" ]; then
            setup_output=$(cat icechunk-python/setup-output.log)
          fi

          # Replace placeholders
          issue_body="${template//\{\{WORKFLOW\}\}/${{ github.workflow }}}"
          issue_body="${issue_body//\{\{RUN_ID\}\}/${{ github.run_id }}}"
          issue_body="${issue_body//\{\{RUN_URL\}\}/${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}}"
          issue_body="${issue_body//\{\{DATE\}\}/$(date -u)}"
          issue_body="${issue_body//\{\{SETUP_OUTPUT\}\}/$setup_output}"

          # Check for existing open issue with same title
          issue_title="Nightly upstream dependency installation failed"
          existing_issue=$(gh issue list --state open --label "CI" --search "\"$issue_title\" in:title" --json number --jq '.[0].number // empty')

          if [ -n "$existing_issue" ]; then
            echo "Found existing open issue #$existing_issue, updating it..."
            echo "$issue_body" | gh issue edit "$existing_issue" --body-file -
            echo "Updated existing issue #$existing_issue"
          else
            echo "No existing open issue found, creating new one..."
            echo "$issue_body" | gh issue create \
              --title "$issue_title" \
              --body-file - \
              --label "CI"
            echo "Created new issue"
          fi
        env:
          GH_TOKEN: ${{ github.token }}

      - name: mypy
        id: mypy
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          mypy --python-version ${{ env.PYTHON_VERSION }} python 2>&1 | tee mypy-output.log

      - name: Create or update mypy failure issue
        if: |
          always()
          && steps.mypy.outcome == 'failure'
          && github.event_name == 'schedule'
          && github.repository_owner == 'earth-mover'
        shell: bash
        working-directory: .
        run: |
          # Read the template and mypy output
          template=$(cat .github/mypy-failure-template.md)

          mypy_output="No mypy output captured"
          if [ -f "icechunk-python/mypy-output.log" ]; then
            mypy_output=$(cat icechunk-python/mypy-output.log)
          fi

          # Replace placeholders
          issue_body="${template//\{\{WORKFLOW\}\}/${{ github.workflow }}}"
          issue_body="${issue_body//\{\{RUN_ID\}\}/${{ github.run_id }}}"
          issue_body="${issue_body//\{\{RUN_URL\}\}/${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}}"
          issue_body="${issue_body//\{\{DATE\}\}/$(date -u)}"
          issue_body="${issue_body//\{\{MYPY_OUTPUT\}\}/$mypy_output}"

          # Check for existing open issue with same title
          issue_title="Nightly MyPy type checking failed with upstream dependencies"
          existing_issue=$(gh issue list --state open --label "CI" --search "\"$issue_title\" in:title" --json number --jq '.[0].number // empty')

          if [ -n "$existing_issue" ]; then
            echo "Found existing open issue #$existing_issue, updating it..."
            echo "$issue_body" | gh issue edit "$existing_issue" --body-file -
            echo "Updated existing issue #$existing_issue with latest mypy output"
          else
            echo "No existing open issue found, creating new one..."
            echo "$issue_body" | gh issue create \
              --title "$issue_title" \
              --body-file - \
              --label "CI"
            echo "Created new issue"
          fi
        env:
          GH_TOKEN: ${{ github.token }}

      - name: Restore cached hypothesis directory
        id: restore-hypothesis-cache
        if: always()
        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
        with:
          path: icechunk-python/.hypothesis/
          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
          restore-keys: |
            cache-hypothesis-

      - name: describe environment
        if: steps.setup.outcome == 'success'
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          pip list

      - name: Install flatc
        if: steps.setup.outcome == 'success'
        run: |
          sudo apt-get update && sudo apt-get install -y flatbuffers-compiler

      - name: pytest
        id: pytest-icechunk
        if: steps.setup.outcome == 'success'
        shell: bash
        working-directory: icechunk-python
        env:
          HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY: "1"
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          pytest -n 4 --hypothesis-profile=nightly --report-log output-pytest-log.jsonl

      # explicitly save the cache so it gets updated, also do this even if it fails.
      - name: Save cached hypothesis directory
        id: save-hypothesis-cache
        if: always()
        uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
        with:
          path: icechunk-python/.hypothesis/
          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}

      - name: Generate and publish the pytest report
        if: |
          always()
          && steps.pytest-icechunk.outcome == 'failure'
          && github.event_name == 'schedule'
          && github.repository_owner == 'earth-mover'
        uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1
        with:
          log-path: icechunk-python/output-pytest-log.jsonl

  xarray-backends:
    name: xarray-tests-upstream
    runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache
    if: ${{
      (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request')
      || github.event_name == 'workflow_dispatch'
      || github.event_name == 'schedule'
      }}
    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2
        with:
          sccache: s3
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          path: "icechunk"

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: "pydata/xarray"
          path: "xarray"
          fetch-depth: 0 # Fetch all history for all branches and tags.

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install uv
        uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Build wheels
        uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1
        with:
          working-directory: icechunk/icechunk-python
          args: --release --out dist --find-interpreter

      - name: setup
        shell: bash
        working-directory: icechunk/icechunk-python
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          python --version
          PY_TAG="cp${PYTHON_VERSION//./}"
          WHEEL=$(ls dist/*-${PY_TAG}-*.whl)
          # Install upstream dependencies from nightly wheels
          export UV_INDEX="https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/"
          export UV_PRERELEASE=allow
          uv pip install "$WHEEL" --group test pytest-mypy-plugins \
            --resolution highest \
            --index-strategy unsafe-best-match
          uv pip list

      - name: Stand up RustFS
        working-directory: icechunk
        run: |
          docker compose up -d rustfs_init

      - name: Wait for RustFS to be ready
        working-directory: icechunk
        run: |
          for _ in {1..10}; do
              if docker compose ps --status exited --filter status==0 | grep rustfs ; then
              break
              fi
              sleep 3
          done
      - name: pytest
        id: pytest-xarray-backends
        shell: bash
        working-directory: icechunk/icechunk-python
        env:
          ICECHUNK_XARRAY_BACKENDS_TESTS: 1
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture
          pytest -c=../../xarray/pyproject.toml -W ignore tests/run_xarray_backends_tests.py --report-log output-pytest-log.jsonl

      - name: Generate and publish the xarray backends report
        if: |
          failure()
          && steps.pytest-xarray-backends.outcome == 'failure'
          && github.event_name == 'schedule'
          && github.repository_owner == 'earth-mover'
        uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1
        with:
          log-path: icechunk/icechunk-python/output-pytest-log.jsonl
          issue-title: "Nightly Xarray backends tests failed"