python.yml

name: python

on: pull_request

env:
  APP_API_KEY: ${{ secrets.APP_API_KEY }}
  ENABLE_HOPSWORKS_USAGE: "false"

jobs:
  lint_stylecheck:
    name: Lint and Stylecheck
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: Get all changed files
        id: get-changed-files
        uses: tj-actions/changed-files@v47
        with:
          files_yaml: |
            src:
              - 'python/**/*.py'
              - '!python/tests/**/*.py'
            test:
              - 'python/tests/**/*.py'
              - '!python/tests/test_helpers/transformation_test_helper.py'

      - name: install deps
        run: pip install ruff==0.15.6

      - name: ruff on python files
        if: steps.get-changed-files.outputs.src_any_changed == 'true'
        env:
          SRC_ALL_CHANGED_FILES:
            ${{ steps.get-changed-files.outputs.src_all_changed_files }}
        run: ruff check --config python/pyproject.toml --output-format=github $SRC_ALL_CHANGED_FILES

      - name: ruff on test files
        if: steps.get-changed-files.outputs.test_any_changed == 'true'
        env:
          TEST_ALL_CHANGED_FILES:
            ${{ steps.get-changed-files.outputs.test_all_changed_files }}
        run: ruff check --config python/pyproject.toml --output-format=github $TEST_ALL_CHANGED_FILES

      - name: ruff format --check $ALL_CHANGED_FILES
        env:
          ALL_CHANGED_FILES:
            ${{ steps.get-changed-files.outputs.all_changed_files }}
        run: ruff format --config python/pyproject.toml --check $ALL_CHANGED_FILES

      - name: install docsig
        run: pip install docsig==0.79.0

      - name: docsig
        if: steps.get-changed-files.outputs.src_any_changed == 'true'
        working-directory: python
        run: docsig hopsworks_common hopsworks hsfs hsml

  unit_tests:
    name: Unit Tests
    needs: lint_stylecheck
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13"]

    steps:
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: "8"
          distribution: "adopt"

      - name: Set Timezone
        run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone

      - uses: actions/checkout@v4
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v5
        name: Setup Python
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev --project python

      - name: Display Python version
        run: python --version

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV

      - name: Run Pytest suite
        run: pytest python/tests

  unit_tests_no_opt:
    name: Unit Tests (No Optional Dependencies)
    needs: lint_stylecheck
    runs-on: ubuntu-latest

    steps:
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: "8"
          distribution: "adopt"

      - name: Set Timezone
        run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone

      - uses: actions/checkout@v4
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v5
        name: Setup Python
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev-no-opt --project python

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV

      - name: Run Pytest suite
        run: pytest python/tests

  unit_tests_pandas1:
    name: Unit Tests (Pandas 1.x)
    needs: lint_stylecheck
    runs-on: ubuntu-latest

    steps:
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: "8"
          distribution: "adopt"

      - name: Set Timezone
        run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone

      - uses: actions/checkout@v4
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v4
        name: Setup Python
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev-pandas1 --project python

      - name: Display Python version
        run: python --version

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV

      - name: Run Pytest suite
        run: pytest python/tests

  unit_tests_legacy_optional:
    name: Unit Tests (Legacy Optional Deps)
    needs: lint_stylecheck
    runs-on: ubuntu-latest

    steps:
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: "8"
          distribution: "adopt"

      - name: Set Timezone
        run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone

      - uses: actions/checkout@v4
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v5
        name: Setup Python
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev --project python

      - name: Pin Great Expectations and Polars to lower bounds
        run: uv pip install "great-expectations==0.18.12" "polars==0.20.31"

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV

      - name: Run Pytest suite
        run: pytest python/tests

  unit_tests_local_tz:
    name: Unit Tests (Local TZ)
    needs: lint_stylecheck
    runs-on: ubuntu-latest

    steps:
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: "8"
          distribution: "adopt"

      - name: Set Timezone
        run: sudo timedatectl set-timezone Europe/Amsterdam && echo Europe/Amsterdam | sudo tee /etc/timezone

      - uses: actions/checkout@v4
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v5
        name: Setup Python
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev --project python

      - name: Display Python version
        run: python --version

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV

      - name: Run Pytest suite
        run: pytest python/tests

  unit_tests_typechecked:
    name: Unit Tests (Typechecked)
    needs: lint_stylecheck
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v4
        name: Setup Python
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev --project python

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV

      - name: Run Pytest suite
        env:
          HOPSWORKS_RUN_WITH_TYPECHECK: "true"
        run: pytest python/tests
        continue-on-error: true

  unit_tests_windows:
    name: Unit Tests (Windows)
    needs: lint_stylecheck
    runs-on: windows-latest

    steps:
      - name: Set Timezone
        run: tzutil /s "UTC"

      - uses: actions/checkout@v4
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v5
        name: Setup Python
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev --project python

      - name: Display Python version
        run: python --version

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        shell: cmd
        run: |
          echo PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell >> %GITHUB_ENV%

      - name: Run Pytest suite
        run: pytest python/tests

  unit_tests_windows_local_tz:
    name: Unit Tests (Windows) (Local TZ)
    needs: lint_stylecheck
    runs-on: windows-latest

    steps:
      - name: Set Timezone
        run: tzutil /s "W. Europe Standard Time"

      - uses: actions/checkout@v4
      - name: Copy README
        run: cp README.md python/

      - uses: actions/setup-python@v5
        name: Setup Python
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          activate-environment: true
          working-directory: python

      - name: Install dependencies
        run: uv sync --extra dev --project python

      - name: Display Python version
        run: python --version

      - name: Display pip freeze
        run: uv pip freeze

      - name: Set PYSPARK_SUBMIT_ARGS for Avro support
        shell: cmd
        run: |
          echo PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell >> %GITHUB_ENV%

      - name: Run Pytest suite
        run: pytest python/tests