BUG: coerce empty-string IEEE NaN to null for pyarrow floats #40314

Workflow file for this run

.github/workflows/unit-tests.yml at 10498a8

	name: Unit Tests

	on:
	push:
	branches:
	- main
	- 3.0.x
	pull_request:
	branches:
	- main
	- 3.0.x
	paths-ignore:
	- "doc/**"
	- "web/**"

	permissions: {}

	defaults:
	run:
	shell: bash -euox pipefail {0}

	jobs:
	ubuntu:
	runs-on: ${{ matrix.platform }}
	permissions:
	contents: read
	timeout-minutes: 90
	strategy:
	matrix:
	platform: [ubuntu-24.04, ubuntu-24.04-arm]
	environment: [py311, py312, py313, py314]
	# Prevent the include jobs from overriding other jobs
	pytest_marker_expression: [""]
	pandas_future_infer_string: ["1"]
	pandas_future_python_scalars: ["0"]
	include:
	- name: "Downstream Compat"
	environment: downstream
	pytest_marker_expression: "not slow and not network and not single_cpu"
	pytest_target: "pandas/tests/test_downstream.py"
	platform: ubuntu-24.04
	- name: "Minimum Versions"
	environment: minimum-versions
	pytest_marker_expression: "not slow and not network and not single_cpu"
	platform: ubuntu-24.04
	- name: "Freethreading"
	environment: py313-freethreading
	pytest_marker_expression: "not slow and not network and not single_cpu"
	platform: ubuntu-24.04
	- name: "Without PyArrow"
	environment: no-pyarrow
	pytest_marker_expression: "not slow and not network and not single_cpu"
	platform: ubuntu-24.04
	- name: "Locale: it_IT"
	environment: py313
	pytest_marker_expression: "not slow and not network and not single_cpu"
	extra_apt: "language-pack-it"
	# Use the utf8 version as the default, it has no bad side-effect.
	lang: "it_IT.utf8"
	lc_all: "it_IT.utf8"
	# Also install it_IT (its encoding is ISO8859-1) but do not activate it.
	# It will be temporarily activated during tests with locale.setlocale
	extra_loc: "it_IT"
	platform: ubuntu-24.04
	- name: "Locale: zh_CN"
	environment: py313
	pytest_marker_expression: "not slow and not network and not single_cpu"
	extra_apt: "language-pack-zh-hans"
	# Use the utf8 version as the default, it has no bad side-effect.
	lang: "zh_CN.utf8"
	lc_all: "zh_CN.utf8"
	# Also install zh_CN (its encoding is gb2312) but do not activate it.
	# It will be temporarily activated during tests with locale.setlocale
	extra_loc: "zh_CN"
	platform: ubuntu-24.04
	- name: "PANDAS_FUTURE_INFER_STRING=0"
	environment: py313
	pandas_future_infer_string: "0"
	platform: ubuntu-24.04
	- name: "PANDAS_FUTURE_PYTHON_SCALARS=1"
	environment: py313
	pandas_future_python_scalars: "1"
	platform: ubuntu-24.04
	- name: "Numpy Nightly"
	environment: numpy-nightly
	pytest_marker_expression: "not slow and not network and not single_cpu"
	platform: ubuntu-24.04
	- name: "Pyarrow Nightly"
	environment: pyarrow-nightly
	pytest_marker_expression: "not slow and not network and not single_cpu"
	platform: ubuntu-24.04
	fail-fast: false
	name: ${{ matrix.name \|\| format('{0} {1}', matrix.platform, matrix.environment) }}
	env:
	LANG: ${{ matrix.lang \|\| 'C.UTF-8' }}
	LC_ALL: ${{ matrix.lc_all \|\| '' }}
	concurrency:
	# https://github.community/t/concurrecy-not-work-for-push/183068/7
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-${{ matrix.environment }}-${{ matrix.pytest_marker_expression }}-${{ matrix.extra_apt \|\| '' }}-${{ matrix.pandas_future_infer_string }}-${{ matrix.platform }}
	cancel-in-progress: true

	services:
	mysql:
	image: mysql:9
	env:
	MYSQL_ALLOW_EMPTY_PASSWORD: yes
	MYSQL_DATABASE: pandas
	options: >-
	--health-cmd "mysqladmin ping"
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 3306:3306

	postgres:
	image: postgres:18
	env:
	PGUSER: postgres
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: postgres
	POSTGRES_DB: pandas
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 5432:5432

	moto:
	image: motoserver/moto:5.1.18
	ports:
	- 5000:5000

	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	fetch-depth: 0

	- name: Generate extra locales
	# These extra locales will be available for locale.setlocale() calls in tests
	run: \|
	sudo apt-get update
	sudo apt-get install --yes ${{ matrix.extra_apt }}
	sudo locale-gen ${{ matrix.extra_loc }}
	if: ${{ matrix.extra_loc && matrix.extra_apt }}

	- name: Create virtual environment with Pixi
	uses: ./.github/actions/setup-pixi
	with:
	environment: ${{ matrix.environment }}

	- name: Build pandas
	id: build
	# matrix.name == 'Minimum Versions' can be removed once Cython >=3.2
	# python3.14t/cpython/pyatomic_gcc.h:63:10:
	# error: ‘__atomic_fetch_add_8’
	# writing 8 bytes into a region of size 0 overflows the destination [-Werror=stringop-overflow=]
	run: \|
	pixi run \
	--environment ${{ matrix.environment }} \
	build-pandas \
	--editable \
	${{ ! (matrix.name == 'Freethreading' \|\| matrix.name == 'Minimum Versions') && '-Csetup-args="--werror"' \|\| '' }}

	- name: Import check
	run: \|
	pixi run \
	--environment ${{ matrix.environment }} \
	ci-check-import

	- name: Test (not single_cpu)
	uses: ./.github/actions/run-tests
	id: test-not-single-cpu
	env:
	# Ensure port matches moto service
	PANDAS_MOTO_URL: "http://localhost:5000"
	with:
	environment: ${{ matrix.environment }}
	numprocesses: 'auto'
	pytest_marker_expression: ${{ matrix.pytest_marker_expression == '' && 'not single_cpu' \|\| matrix.pytest_marker_expression }}
	pytest_target: ${{ matrix.pytest_target \|\| 'pandas' }}
	pandas_future_infer_string: ${{ matrix.pandas_future_infer_string \|\| '1' }}
	pandas_future_python_scalars: ${{ matrix.pandas_future_python_scalars \|\| '0' }}

	- name: Test (single_cpu)
	id: test-single-cpu
	uses: ./.github/actions/run-tests
	env:
	# Ensure port matches moto service
	PANDAS_MOTO_URL: "http://localhost:5000"
	with:
	environment: ${{ matrix.environment }}
	numprocesses: 0
	pytest_marker_expression: 'single_cpu'
	pytest_target: ${{ matrix.pytest_target \|\| 'pandas' }}
	pandas_future_infer_string: ${{ matrix.pandas_future_infer_string \|\| '1' }}
	pandas_future_python_scalars: ${{ matrix.pandas_future_python_scalars \|\| '0' }}
	if: ${{ matrix.pytest_marker_expression == '' && (always() && steps.build.outcome == 'success')}}

	- name: Upload test coverage to Codecov
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v5
	with:
	flags: unittests
	name: codecov-pandas-${{ matrix.name \|\| format('{0}-{1}', matrix.platform, matrix.environment) }}
	token: ${{ secrets.CODECOV_TOKEN }}
	fail_ci_if_error: false
	disable_telem: true
	# coverage report appended for single_cpu + not single_cpu steps
	# provide most hollistic coverage report
	if: ${{ steps.test-not-single-cpu.outcome == 'success' && steps.test-single-cpu.outcome == 'success' }}

	macos-windows:
	permissions:
	contents: read
	timeout-minutes: 90
	strategy:
	matrix:
	os: [macos-15-intel, macos-15, windows-2025]
	environment: [py311, py312, py313, py314]
	fail-fast: false
	runs-on: ${{ matrix.os }}
	name: ${{ format('{0} {1}', matrix.os, matrix.environment) }}
	concurrency:
	# https://github.community/t/concurrecy-not-work-for-push/183068/7
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-${{ matrix.environment }}-${{ matrix.os }}
	cancel-in-progress: true

	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	fetch-depth: 0

	- name: Create virtual environment with Pixi
	uses: ./.github/actions/setup-pixi
	with:
	environment: ${{ matrix.environment }}

	- name: Remove link.EXE for Windows
	# https://github.com/Quansight-Labs/uarray/pull/310#issuecomment-3872989816
	run: rm /c/Program\ Files/Git/usr/bin/link.EXE
	if: ${{ matrix.os == 'windows-2025' }}

	- name: Build pandas
	id: build
	run: \|
	pixi run \
	--environment ${{ matrix.environment }} \
	build-pandas \
	--editable \
	"${{ matrix.os == 'windows-2025' && '-Csetup-args="--vsenv" -Csetup-args="--werror"' \|\| '-Csetup-args="--werror"' }}"

	- name: Test
	uses: ./.github/actions/run-tests
	with:
	environment: ${{ matrix.environment }}
	numprocesses: 'auto'
	pytest_marker_expression: 'not slow and not db and not network and not single_cpu'
	pytest_target: 'pandas'

	Linux-32-bit:
	runs-on: ubuntu-24.04
	permissions:
	contents: read
	container:
	image: quay.io/pypa/manylinux_2_28_i686
	options: --platform linux/386
	steps:
	- name: Checkout pandas Repo
	# actions/checkout does not work since it requires node
	run: \|
	git config --global --add safe.directory $PWD

	if [ $GITHUB_EVENT_NAME != pull_request ]; then
	git clone --recursive --branch=$GITHUB_REF_NAME https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE
	git reset --hard $GITHUB_SHA
	else
	git clone --recursive https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE
	git fetch origin $GITHUB_REF:my_ref_name
	git checkout $GITHUB_BASE_REF
	git -c user.email="you@example.com" merge --no-commit my_ref_name
	fi
	- name: Build environment
	# https://github.com/numpy/numpy/issues/24703#issuecomment-1722379388
	run: \|
	/opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
	. ~/virtualenvs/pandas-dev/bin/activate
	python -m pip install --no-cache-dir -U pip wheel meson[ninja] meson-python
	python -m pip install numpy -Csetup-args="-Dallow-noblas=true"
	# hypothesis 6.156+ ships a compiled extension with no i686 wheel, and
	# its sdist needs a Rust target rustup lacks on 32-bit. Cap until fixed.
	python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil "pytest>=8.3.4" pytest-xdist>=3.6.1 hypothesis>=6.156.5 pytest-cov
	python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror" -Csetup-args=-Dc_args="-Wno-overflow -Wno-type-limits -Wno-shift-count-overflow -Wno-sign-compare"
	python -m pip list --no-cache-dir

	- name: Run Tests
	run: \|
	. ~/virtualenvs/pandas-dev/bin/activate
	python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas
	concurrency:
	# https://github.community/t/concurrecy-not-work-for-push/183068/7
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-32bit
	cancel-in-progress: true

	Linux-Musl:
	runs-on: ubuntu-24.04
	permissions:
	contents: read
	container:
	image: quay.io/pypa/musllinux_1_2_x86_64
	steps:
	- name: Checkout pandas Repo
	# actions/checkout does not work since it requires node
	run: \|
	git config --global --add safe.directory $PWD

	if [ $GITHUB_EVENT_NAME != pull_request ]; then
	git clone --recursive --branch=$GITHUB_REF_NAME https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE
	git reset --hard $GITHUB_SHA
	else
	git clone --recursive https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE
	git fetch origin $GITHUB_REF:my_ref_name
	git checkout $GITHUB_BASE_REF
	git -c user.email="you@example.com" merge --no-commit my_ref_name
	fi
	- name: Configure System Packages
	run: \|
	apk update
	apk add musl-locales
	- name: Build environment
	run: \|
	/opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
	. ~/virtualenvs/pandas-dev/bin/activate
	python -m pip install --no-cache-dir -U pip wheel meson-python meson[ninja]
	python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil "pytest>=8.3.4" pytest-xdist>=3.6.1 hypothesis>=6.116.0 pytest-cov
	python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
	python -m pip list --no-cache-dir
	- name: Run Tests
	run: \|
	. ~/virtualenvs/pandas-dev/bin/activate
	python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas
	concurrency:
	# https://github.community/t/concurrecy-not-work-for-push/183068/7
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-musl
	cancel-in-progress: true

	Windows-MinGW:
	runs-on: windows-2025
	permissions:
	contents: read
	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	fetch-depth: 0

	- name: Create virtual environment with Pixi
	uses: ./.github/actions/setup-pixi
	with:
	environment: mingw

	- name: Build pandas
	id: build
	run: \|
	pixi run \
	--environment mingw \
	build-pandas \
	--editable \
	-Csetup-args="--werror"
	concurrency:
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-mingw
	cancel-in-progress: true

	Linux-Sanitizers:
	name: "Clang (ASAN/UBSAN)"
	runs-on: ubuntu-24.04
	permissions:
	contents: read
	timeout-minutes: 90
	concurrency:
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-sanitizers
	cancel-in-progress: true
	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	fetch-depth: 0

	- name: Create virtual environment with Pixi
	uses: ./.github/actions/setup-pixi
	with:
	environment: sanitizers

	- name: Build pandas
	id: build
	run: \|
	pixi run \
	--environment sanitizers \
	build-pandas \
	--editable \
	-Csetup-args="--werror" \
	-- \
	-Csetup-args="-Dbuildtype=debugoptimized" \
	-Csetup-args="-Db_sanitize=address,undefined" \
	-Csetup-args="-Db_lundef=false"

	- name: Get Sanitizer Path
	run: \|
	echo "ASAN_LIB_PATH=$(pixi run -e sanitizers clang -print-file-name=libclang_rt.asan-x86_64.so)" >> $GITHUB_ENV

	- name: Test
	uses: ./.github/actions/run-tests
	env:
	LD_PRELOAD: ${{ env.ASAN_LIB_PATH }}
	ASAN_OPTIONS: detect_leaks=0
	UBSAN_OPTIONS: halt_on_error=1:suppressions=${{ github.workspace }}/ci/suppressions/ubsan.txt
	with:
	environment: sanitizers
	numprocesses: "auto"
	pytest_marker_expression: "not slow and not db and not network and not single_cpu"
	pytest_target: "pandas"

	python-dev:
	# This job may or may not run depending on the state of the next
	# unreleased Python version. DO NOT DELETE IT.
	#
	# In general, this will remain frozen(present, but not running) until:
	# - The next unreleased Python version has released beta 1
	# - This version should be available on GitHub Actions.
	# - Our required build/runtime dependencies(numpy, Cython, python-dateutil)
	# support that unreleased Python version.
	# To unfreeze, comment out the ``if: false`` condition, and make sure you update
	# the name of the workflow and Python version in actions/setup-python ``python-version:``
	#
	# After it has been unfrozen, this file should remain unfrozen(present, and running) until:
	# - The next Python version has been officially released.
	# OR
	# - Most/All of our optional dependencies support the next Python version AND
	# - The next Python version has released a rc(we are guaranteed a stable ABI).
	# To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
	# to the corresponding posix/windows-macos/sdist etc. workflows.
	# Feel free to modify this comment as necessary.
	# if: false
	permissions:
	contents: read
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-24.04, macos-15-intel, macos-15, windows-2025]

	timeout-minutes: 90

	concurrency:
	# https://github.community/t/concurrecy-not-work-for-push/183068/7
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-${{ matrix.os }}-python-dev
	cancel-in-progress: true

	env:
	PYTEST_WORKERS: "auto"
	PATTERN: "not slow and not network and not clipboard and not single_cpu"
	PYTEST_TARGET: pandas

	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	fetch-depth: 0

	- name: Set up Python Dev Version
	uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
	with:
	python-version: '3.15-dev'

	- name: Build Environment
	run: \|
	python --version
	python -m pip install --upgrade pip wheel meson[ninja] meson-python
	python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
	python -m pip install versioneer[toml] python-dateutil tzdata cython hypothesis>=6.116.0 "pytest>=8.3.4" pytest-xdist>=3.6.1 pytest-cov
	python -m pip install -ve . --no-build-isolation --no-index --no-deps \
	-Csetup-args="--werror" \
	${{ matrix.os != 'windows-2025' && '-Csetup-args=-Dcpp_args="-Wno-missing-field-initializers"' \|\| '' }} \
	${{ matrix.os == 'windows-2025' && '-Csetup-args="--vsenv"' \|\| '' }}
	python -m pip list

	- name: Run Tests
	run: python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas

	# NOTE: this job must be kept in sync with the Pyodide build job in wheels.yml
	emscripten:
	# Note: the Python version, Emscripten toolchain version are determined
	# by the Pyodide version. The appropriate versions can be found in the
	# Pyodide repodata.json "info" field, or in the Makefile.envs file:
	# https://github.com/pyodide/pyodide/blob/stable/Makefile.envs#L2
	# The Node.js version can be determined via Pyodide:
	# https://pyodide.org/en/stable/usage/index.html#node-js
	name: Pyodide build
	runs-on: ubuntu-24.04
	permissions:
	contents: read
	concurrency:
	# https://github.community/t/concurrecy-not-work-for-push/183068/7
	group: ${{ github.event_name == 'push' && github.run_number \|\| github.ref }}-wasm
	cancel-in-progress: true
	steps:
	- name: Checkout pandas Repo
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	fetch-depth: 0

	- name: Set up Node.js
	uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
	with:
	node-version: 24

	- name: Set up Python
	uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
	with:
	python-version: '3.12'

	- name: Install pyodide-build
	run: python -m pip install pyodide-build

	- name: Save Emscripten version
	run: echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV

	- name: Set up Emscripten toolchain
	uses: emscripten-core/setup-emsdk@4528d102f7230f0e7b276855c01ea1159be0e984 # v16
	with:
	version: ${{ env.EMSCRIPTEN_VERSION }}
	actions-cache-folder: emsdk-cache

	- name: Build pandas for Pyodide
	run: pyodide build

	- name: Set up Pyodide virtual environment
	run: \|
	pyodide xbuildenv install 0.27.1
	pyodide venv .venv-pyodide
	source .venv-pyodide/bin/activate
	python -m pip install dist/*.whl

	- name: Test pandas for Pyodide
	run: \|
	source .venv-pyodide/bin/activate
	python -m pip install "pytest>=8.3.4" hypothesis pytest-cov
	# do not import pandas from the checked out repo
	cd ../..
	pytest -m 'not slow and not network and not clipboard and not single_cpu' --pyargs pandas

Navigation Menu

Uh oh!

Uh oh!

BUG: coerce empty-string IEEE NaN to null for pyarrow floats #40314

Workflow file

BUG: coerce empty-string IEEE NaN to null for pyarrow floats #40314

Uh oh!

Workflow file for this run