Update CI to Spark 4.0.0 #1357

	name: Python package

	on: [pull_request]

	jobs:
	build:
	runs-on: ubuntu-latest
	timeout-minutes: 20
	strategy:
	matrix:
	python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]

	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-java@v4
	with:
	distribution: 'temurin'
	java-version: 21
	- uses: vemonet/setup-spark@v1
	with:
	spark-version: '4.0.0'
	hadoop-version: '3'
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	cache: 'pip'
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt
	pip install -r requirements-dev.txt
	- name: Linting
	run: \|
	flake8
	pylint typedspark
	mypy .
	pyright .
	bandit typedspark/*/.py
	black --check .
	isort --check .
	docformatter --black -c */.py
	- name: Testing
	run: \|
	# we run this test seperately, to ensure that it is run without an active Spark session
	python -m pytest -m no_spark_session

	coverage run -m pytest
	coverage report -m --fail-under 100
	- name: Run notebooks
	run: \|
	for FILE in docs//.ipynb; do
	BASE=$(basename $FILE)
	cp $FILE .
	jupyter nbconvert --to notebook $BASE --execute
	done

Provide feedback