Skip to content
python

[FSTORE-2034] Fix managed FG #3657

Sign in to view logs

[FSTORE-2034] Fix managed FG

[FSTORE-2034] Fix managed FG #3657

Workflow file for this run

.github/workflows/python.yml at 20a1665
name: python
on: pull_request
env:
APP_API_KEY: ${{ secrets.APP_API_KEY }}
ENABLE_HOPSWORKS_USAGE: "false"
jobs:
lint_stylecheck:
name: Lint and Stylecheck
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- name: Get all changed files
id: get-changed-files
uses: tj-actions/changed-files@v47
with:
files_yaml: |
src:
- 'python/**/*.py'
- '!python/tests/**/*.py'
test:
- 'python/tests/**/*.py'
- '!python/tests/test_helpers/transformation_test_helper.py'
- name: install deps
run: pip install ruff==0.15.6
- name: ruff on python files
if: steps.get-changed-files.outputs.src_any_changed == 'true'
env:
SRC_ALL_CHANGED_FILES:
${{ steps.get-changed-files.outputs.src_all_changed_files }}
run: ruff check --config python/pyproject.toml --output-format=github $SRC_ALL_CHANGED_FILES
- name: ruff on test files
if: steps.get-changed-files.outputs.test_any_changed == 'true'
env:
TEST_ALL_CHANGED_FILES:
${{ steps.get-changed-files.outputs.test_all_changed_files }}
run: ruff check --config python/pyproject.toml --output-format=github $TEST_ALL_CHANGED_FILES
- name: ruff format --check $ALL_CHANGED_FILES
env:
ALL_CHANGED_FILES:
${{ steps.get-changed-files.outputs.all_changed_files }}
run: ruff format --config python/pyproject.toml --check $ALL_CHANGED_FILES
- name: install docsig
run: pip install docsig==0.79.0
- name: docsig
if: steps.get-changed-files.outputs.src_any_changed == 'true'
working-directory: python
run: docsig hopsworks_common hopsworks hsfs hsml
unit_tests:
name: Unit Tests
needs: lint_stylecheck
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
steps:
- name: Set up JDK 8
uses: actions/setup-java@v3
with:
java-version: "8"
distribution: "adopt"
- name: Set Timezone
run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone
- uses: actions/checkout@v4
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v5
name: Setup Python
with:
python-version: ${{ matrix.python-version }}
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev --project python
- name: Display Python version
run: python --version
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV
- name: Run Pytest suite
run: pytest python/tests
unit_tests_no_opt:
name: Unit Tests (No Optional Dependencies)
needs: lint_stylecheck
runs-on: ubuntu-latest
steps:
- name: Set up JDK 8
uses: actions/setup-java@v3
with:
java-version: "8"
distribution: "adopt"
- name: Set Timezone
run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone
- uses: actions/checkout@v4
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v5
name: Setup Python
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev-no-opt --project python
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV
- name: Run Pytest suite
run: pytest python/tests
unit_tests_pandas1:
name: Unit Tests (Pandas 1.x)
needs: lint_stylecheck
runs-on: ubuntu-latest
steps:
- name: Set up JDK 8
uses: actions/setup-java@v3
with:
java-version: "8"
distribution: "adopt"
- name: Set Timezone
run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone
- uses: actions/checkout@v4
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v4
name: Setup Python
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev-pandas1 --project python
- name: Display Python version
run: python --version
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV
- name: Run Pytest suite
run: pytest python/tests
unit_tests_legacy_optional:
name: Unit Tests (Legacy Optional Deps)
needs: lint_stylecheck
runs-on: ubuntu-latest
steps:
- name: Set up JDK 8
uses: actions/setup-java@v3
with:
java-version: "8"
distribution: "adopt"
- name: Set Timezone
run: sudo timedatectl set-timezone UTC && echo UTC | sudo tee /etc/timezone
- uses: actions/checkout@v4
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v5
name: Setup Python
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev --project python
- name: Pin Great Expectations and Polars to lower bounds
run: uv pip install "great-expectations==0.18.12" "polars==0.20.31"
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV
- name: Run Pytest suite
run: pytest python/tests
unit_tests_local_tz:
name: Unit Tests (Local TZ)
needs: lint_stylecheck
runs-on: ubuntu-latest
steps:
- name: Set up JDK 8
uses: actions/setup-java@v3
with:
java-version: "8"
distribution: "adopt"
- name: Set Timezone
run: sudo timedatectl set-timezone Europe/Amsterdam && echo Europe/Amsterdam | sudo tee /etc/timezone
- uses: actions/checkout@v4
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v5
name: Setup Python
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev --project python
- name: Display Python version
run: python --version
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV
- name: Run Pytest suite
run: pytest python/tests
unit_tests_typechecked:
name: Unit Tests (Typechecked)
needs: lint_stylecheck
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v4
name: Setup Python
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev --project python
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
run: echo "PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell" >> $GITHUB_ENV
- name: Run Pytest suite
env:
HOPSWORKS_RUN_WITH_TYPECHECK: "true"
run: pytest python/tests
continue-on-error: true
unit_tests_windows:
name: Unit Tests (Windows)
needs: lint_stylecheck
runs-on: windows-latest
steps:
- name: Set Timezone
run: tzutil /s "UTC"
- uses: actions/checkout@v4
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v5
name: Setup Python
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev --project python
- name: Display Python version
run: python --version
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
shell: cmd
run: |
echo PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell >> %GITHUB_ENV%
- name: Run Pytest suite
run: pytest python/tests
unit_tests_windows_local_tz:
name: Unit Tests (Windows) (Local TZ)
needs: lint_stylecheck
runs-on: windows-latest
steps:
- name: Set Timezone
run: tzutil /s "W. Europe Standard Time"
- uses: actions/checkout@v4
- name: Copy README
run: cp README.md python/
- uses: actions/setup-python@v5
name: Setup Python
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: true
working-directory: python
- name: Install dependencies
run: uv sync --extra dev --project python
- name: Display Python version
run: python --version
- name: Display pip freeze
run: uv pip freeze
- name: Set PYSPARK_SUBMIT_ARGS for Avro support
shell: cmd
run: |
echo PYSPARK_SUBMIT_ARGS=--packages org.apache.spark:spark-avro_2.12:3.5.5 pyspark-shell >> %GITHUB_ENV%
- name: Run Pytest suite
run: pytest python/tests