From 7d4bfa8a162bcbf91de20101d86d97a94666cf5d Mon Sep 17 00:00:00 2001 From: fhoehle Date: Thu, 9 Jul 2026 09:56:22 +0200 Subject: [PATCH] fix(polars): raise minimum polars version to 1.20.0 Drop the pre-1.0 compatibility code and update the CI/nox matrix to 1.20.0, 1.33.1 and 1.42.1. Add a DeprecationWarning for Array(width=...) pointing to shape=. Closes #2407 Signed-off-by: fhoehle --- .github/workflows/ci-tests.yml | 15 +++++++- AGENTS.md | 4 +- environment.yml | 2 +- noxfile.py | 4 +- pandera/api/polars/utils.py | 22 ----------- pandera/backends/narwhals/base.py | 6 +-- pandera/backends/narwhals/container.py | 3 +- pandera/backends/polars/base.py | 18 +++------ pandera/backends/polars/builtin_checks.py | 4 +- pandera/backends/polars/checks.py | 8 +--- pandera/backends/polars/components.py | 14 ++----- pandera/backends/polars/container.py | 17 +++++---- pandera/engines/polars_engine.py | 45 ++++++++++++++--------- pandera/typing/polars.py | 7 ---- pyproject.toml | 6 +-- requirements.txt | 2 +- tests/pandas/test_pydantic_dtype.py | 4 +- tests/polars/test_polars_check.py | 3 +- tests/polars/test_polars_components.py | 3 +- tests/polars/test_polars_config.py | 11 +----- tests/polars/test_polars_container.py | 16 +++----- tests/polars/test_polars_dtypes.py | 33 +++++++---------- tests/polars/test_polars_model.py | 9 +---- tests/polars/test_polars_typing.py | 9 +---- 24 files changed, 103 insertions(+), 162 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 0e95b7de8..9c9fe65a6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -33,7 +33,7 @@ jobs: fail-fast: true matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - polars-version: ["0.20.0", "1.33.1"] + polars-version: ["1.20.0", "1.33.1","1.42.1"] defaults: run: shell: bash -l {0} @@ -337,9 +337,20 @@ jobs: - xarray include: - extra: polars - polars-version: "0.20.0" + python-version: "3.10" + polars-version: "1.20.0" + - extra: polars + python-version: "3.11" + polars-version: "1.20.0" + - extra: polars + python-version: "3.12" + polars-version: "1.33.1" - extra: polars + python-version: "3.13" polars-version: "1.33.1" + - extra: polars + python-version: "3.14" + polars-version: "1.42.1" - extra: polars pydantic-version: "2.12.3" exclude: diff --git a/AGENTS.md b/AGENTS.md index 8a63770f0..6482c4fd5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,7 +156,7 @@ The nox `tests` session maps extras to test directories: `extra=None` runs - Python: 3.10, 3.11, 3.12, 3.13, 3.14 - Pandas: 2.1.1, 2.3.3 - Pydantic: 1.10.11, 2.12.3 -- Polars: 0.20.0, 1.33.1 +- Polars: 1.20.0, 1.33.1, 1.42.1 ## Code Quality @@ -191,7 +191,7 @@ prek run --all-files | Extra | Key packages | |--------------|---------------------------------------| | `pandas` | numpy, pandas >= 2.1.1 | -| `polars` | polars >= 0.20.0 | +| `polars` | polars >= 1.20.0 | | `pyspark` | pyspark[connect] >= 3.2.0 | | `ibis` | ibis-framework >= 9.0.0 | | `dask` | dask[dataframe], distributed | diff --git a/environment.yml b/environment.yml index d62071fda..f680c1e87 100644 --- a/environment.yml +++ b/environment.yml @@ -28,7 +28,7 @@ dependencies: - pyspark[connect] >= 3.2.0 # polars extra - - polars >= 0.20.0 + - polars >= 1.20.0 # xarray extra - xarray >= 2024.10.0 diff --git a/noxfile.py b/noxfile.py index 8a4ee5a0d..0d596d42a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -22,7 +22,7 @@ PYTHON_VERSIONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] PANDAS_VERSIONS = ["2.3.3", "3.0.0"] PYDANTIC_VERSIONS = ["1.10.11", "2.12.3"] -POLARS_VERSIONS = ["0.20.0", "1.33.1"] +POLARS_VERSIONS = ["1.20.0", "1.33.1", "1.42.1"] PACKAGE = "pandera" SOURCE_PATHS = PACKAGE, "tests", "noxfile.py" REQUIREMENT_PATH = "requirements.txt" @@ -184,7 +184,7 @@ def _testing_requirements( req = "pyarrow >= 13" if req == "ibis-framework" or req.startswith("ibis-framework "): req = "ibis-framework[duckdb] >= 11.0.0" - if req == "polars": + if req == "polars" or req.startswith("polars "): req = f"polars=={polars}" # for some reason uv will try to install an old version of dask, diff --git a/pandera/api/polars/utils.py b/pandera/api/polars/utils.py index fe70eefdb..21e9e62ff 100644 --- a/pandera/api/polars/utils.py +++ b/pandera/api/polars/utils.py @@ -3,7 +3,6 @@ import polars as pl from pandera.api.polars.types import PolarsCheckObjects -from pandera.backends.polars.utils import polars_version from pandera.config import ( ValidationDepth, get_config_context, @@ -11,27 +10,6 @@ ) -def get_lazyframe_schema(lf: pl.LazyFrame) -> dict[str, pl.DataType]: - """Get a dict of column names and dtypes from a polars LazyFrame.""" - if polars_version().release >= (1, 0, 0): - return lf.collect_schema() - return lf.schema - - -def get_lazyframe_column_dtypes(lf: pl.LazyFrame) -> list[pl.DataType]: - """Get a list of column dtypes from a polars LazyFrame.""" - if polars_version().release >= (1, 0, 0): - return lf.collect_schema().dtypes() - return [*lf.schema.values()] - - -def get_lazyframe_column_names(lf: pl.LazyFrame) -> list[str]: - """Get a list of column names from a polars LazyFrame.""" - if polars_version().release >= (1, 0, 0): - return lf.collect_schema().names() - return lf.columns - - def get_validation_depth(check_obj: PolarsCheckObjects) -> ValidationDepth: """Get validation depth for a given polars check object.""" is_dataframe = isinstance(check_obj, pl.DataFrame) diff --git a/pandera/backends/narwhals/base.py b/pandera/backends/narwhals/base.py index 0b8bce082..9520ff63f 100644 --- a/pandera/backends/narwhals/base.py +++ b/pandera/backends/narwhals/base.py @@ -571,11 +571,7 @@ def _build_eager_failure_case(fc, err: SchemaError, check_identifier): if resolved_co is not None: co_eager = _materialize(resolved_co) - try: - co_indexed = co_eager.with_row_index("index") - except AttributeError: - # Older polars: ``with_row_index`` was called ``with_row_count``. - co_indexed = co_eager.with_row_count("index") + co_indexed = co_eager.with_row_index("index") failing_indices = co_indexed.filter(~nw.col(CHECK_OUTPUT_KEY))[ "index" ].to_list() diff --git a/pandera/backends/narwhals/container.py b/pandera/backends/narwhals/container.py index 6c0393a45..47733def1 100644 --- a/pandera/backends/narwhals/container.py +++ b/pandera/backends/narwhals/container.py @@ -543,8 +543,7 @@ def run_index_checks( def collect_column_info(self, check_obj, schema): """Collect column metadata for the dataframe.""" - # Use collect_schema().names() — lazy-safe Narwhals equivalent of - # get_lazyframe_column_names() + frame_column_names = check_obj.collect_schema().names() column_names: list[Any] = [] diff --git a/pandera/backends/polars/base.py b/pandera/backends/polars/base.py index 54d8eaef3..cabde73a3 100644 --- a/pandera/backends/polars/base.py +++ b/pandera/backends/polars/base.py @@ -8,10 +8,6 @@ from pandera.api.base.error_handler import ErrorHandler from pandera.api.polars.types import CheckResult, PolarsFrame -from pandera.api.polars.utils import ( - get_lazyframe_column_dtypes, - get_lazyframe_schema, -) from pandera.backends.base import BaseSchemaBackend, CoreCheckResult from pandera.constants import CHECK_OUTPUT_KEY from pandera.errors import ( @@ -26,9 +22,9 @@ def is_float_dtype(check_obj: pl.LazyFrame, selector): """Check if a column/selector is a float.""" return all( dtype in {pl.Float32, pl.Float64} - for dtype in get_lazyframe_column_dtypes( - check_obj.select(pl.col(selector)) - ) + for dtype in check_obj.select(pl.col(selector)) + .collect_schema() + .dtypes() ) @@ -96,7 +92,7 @@ def run_check( else: # use check_result _failure_cases = check_result.failure_cases - if CHECK_OUTPUT_KEY in get_lazyframe_schema(_failure_cases): + if CHECK_OUTPUT_KEY in _failure_cases.collect_schema(): _failure_cases = _failure_cases.drop(CHECK_OUTPUT_KEY) failure_cases = _failure_cases.collect() @@ -167,10 +163,8 @@ def failure_cases_metadata( failure_cases_df = err.failure_cases # get row number of the failure cases - if hasattr(err.check_output, "with_row_index"): - _index_lf = err.check_output.with_row_index("index") - else: - _index_lf = err.check_output.with_row_count("index") + assert err.check_output is not None + _index_lf = err.check_output.with_row_index("index") index = _index_lf.filter(pl.col(CHECK_OUTPUT_KEY).eq(False))[ "index" diff --git a/pandera/backends/polars/builtin_checks.py b/pandera/backends/polars/builtin_checks.py index aa97ad847..aa9e65b45 100644 --- a/pandera/backends/polars/builtin_checks.py +++ b/pandera/backends/polars/builtin_checks.py @@ -133,6 +133,7 @@ def in_range( max_value. """ col = pl.col(data.key) + compare_min = col.ge(min_value) if include_min else col.gt(min_value) compare_max = col.le(max_value) if include_max else col.lt(max_value) @@ -296,6 +297,7 @@ def unique_values_eq(data: PolarsData, values: Iterable) -> bool: :param values: The set of values that must be present. May be any iterable. """ + # Use to_list(): polars < 1.21.0 can't call `unique` on Decimal columns. return ( - set(data.lazyframe.collect().get_column(data.key).unique()) == values + set(data.lazyframe.collect().get_column(data.key).to_list()) == values ) diff --git a/pandera/backends/polars/checks.py b/pandera/backends/polars/checks.py index 203f7425d..1cbb12f09 100644 --- a/pandera/backends/polars/checks.py +++ b/pandera/backends/polars/checks.py @@ -9,10 +9,6 @@ from pandera.api.base.checks import CheckResult from pandera.api.checks import Check from pandera.api.polars.types import PolarsData -from pandera.api.polars.utils import ( - get_lazyframe_column_names, - get_lazyframe_schema, -) from pandera.backends.base import BaseCheckBackend from pandera.backends.polars.utils import horizontal_concat from pandera.constants import CHECK_OUTPUT_KEY @@ -57,7 +53,7 @@ def apply(self, check_obj: PolarsData): if isinstance(out, bool): return out - if len(get_lazyframe_schema(out)) > 1: + if len(out.collect_schema()) > 1: # for checks that return a boolean dataframe, reduce to a single # boolean column. out = out.select( @@ -69,7 +65,7 @@ def apply(self, check_obj: PolarsData): ) else: out = out.rename( - {get_lazyframe_column_names(out)[0]: CHECK_OUTPUT_KEY} + {out.collect_schema().names()[0]: CHECK_OUTPUT_KEY} ) return out diff --git a/pandera/backends/polars/components.py b/pandera/backends/polars/components.py index dbb17d8dc..ab7ae413c 100644 --- a/pandera/backends/polars/components.py +++ b/pandera/backends/polars/components.py @@ -9,10 +9,6 @@ from pandera.api.base.error_handler import ErrorHandler, get_error_category from pandera.api.polars.components import Column from pandera.api.polars.types import PolarsData -from pandera.api.polars.utils import ( - get_lazyframe_column_names, - get_lazyframe_schema, -) from pandera.backends.base import CoreCheckResult from pandera.backends.polars.base import PolarsSchemaBackend, is_float_dtype from pandera.backends.polars.utils import horizontal_concat @@ -99,7 +95,7 @@ def validate( "regex pattern so that it matches at least one " "column." ), - failure_cases=get_lazyframe_column_names(check_obj), + failure_cases=check_obj.collect_schema().names(), check=f"no_regex_column_match('{schema.selector}')", reason_code=SchemaErrorReason.INVALID_COLUMN_NAME, ) @@ -155,7 +151,7 @@ def validate( return check_obj def get_regex_columns(self, schema, check_obj) -> Iterable: - return get_lazyframe_schema(check_obj.select(pl.col(schema.selector))) + return check_obj.select(pl.col(schema.selector)).collect_schema() def run_checks_and_handle_errors( self, @@ -269,7 +265,7 @@ def check_nullable( isna = check_obj.select(expr) passed = isna.select([pl.col("*").all()]).collect() results = [] - for column in get_lazyframe_column_names(isna): + for column in isna.collect_schema().names(): if passed.select(column).item(): continue failure_cases = ( @@ -378,9 +374,7 @@ def check_dtype( results = [] check_obj_subset = check_obj.select(schema.selector) - for column, obj_dtype in get_lazyframe_schema( - check_obj_subset - ).items(): + for column, obj_dtype in check_obj_subset.collect_schema().items(): results.append( CoreCheckResult( passed=schema.dtype.check( diff --git a/pandera/backends/polars/container.py b/pandera/backends/polars/container.py index ff90f98f3..77f8e6e4f 100644 --- a/pandera/backends/polars/container.py +++ b/pandera/backends/polars/container.py @@ -11,7 +11,6 @@ from pandera.api.base.error_handler import ErrorHandler, get_error_category from pandera.api.polars.container import DataFrameSchema from pandera.api.polars.types import PolarsData, PolarsFrame -from pandera.api.polars.utils import get_lazyframe_column_names from pandera.backends.base import ColumnInfo, CoreCheckResult from pandera.backends.polars.base import PolarsSchemaBackend from pandera.config import ValidationDepth, ValidationScope, get_config_context @@ -242,7 +241,7 @@ def collect_column_info(self, check_obj: pl.LazyFrame, schema): for col_name, col_schema in schema.columns.items(): if ( not col_schema.regex - and col_name not in get_lazyframe_column_names(check_obj) + and col_name not in check_obj.collect_schema().names() and col_schema.required ): absent_column_names.append(col_name) @@ -257,11 +256,11 @@ def collect_column_info(self, check_obj: pl.LazyFrame, schema): regex_match_patterns.append(col_schema.selector) except SchemaError: pass - elif col_name in get_lazyframe_column_names(check_obj): + elif col_name in check_obj.collect_schema().names(): column_names.append(col_name) # drop adjacent duplicated column names - destuttered_column_names = [*get_lazyframe_column_names(check_obj)] + destuttered_column_names = [*check_obj.collect_schema().names()] return ColumnInfo( sorted_column_names=dict.fromkeys(column_names), @@ -298,7 +297,7 @@ def collect_schema_components( # PydanticModel applies row-wise, so per-column components # are not created for it. columns = {} - for col_name in get_lazyframe_column_names(check_obj): + for col_name in check_obj.collect_schema().names(): columns[col_name] = Column(schema.dtype, name=str(col_name)) schema_components = [] @@ -506,7 +505,7 @@ def _coerce_dtype_helper( else "coerce" ) - lf_columns = get_lazyframe_column_names(obj) + lf_columns = obj.collect_schema().names() try: if schema.dtype is not None: @@ -525,8 +524,10 @@ def _coerce_dtype_helper( # a coercion failure reports the concrete column # name instead of the regex pattern (issue #2363, # mirroring the check path fixed in #2221). - matched_columns = get_lazyframe_column_names( + matched_columns = ( obj.select(pl.col(col_schema.selector)) + .collect_schema() + .names() ) for matched_column in matched_columns: obj = getattr(col_schema.dtype, coerce_fn)( @@ -660,7 +661,7 @@ def check_column_values_are_unique( check_output = None for lst in temp_unique: subset = [ - x for x in lst if x in get_lazyframe_column_names(check_obj) + x for x in lst if x in check_obj.collect_schema().names() ] duplicates = check_obj.select(subset).collect().is_duplicated() diff --git a/pandera/engines/polars_engine.py b/pandera/engines/polars_engine.py index 7878a54b4..cc52ba42d 100644 --- a/pandera/engines/polars_engine.py +++ b/pandera/engines/polars_engine.py @@ -12,7 +12,6 @@ from typing import ( Any, Literal, - Optional, TypedDict, Union, overload, @@ -21,8 +20,9 @@ import polars as pl from polars._typing import ColumnNameOrSelector, PythonDataType from polars.datatypes import DataTypeClass +from polars.datatypes._parse import parse_py_type_into_dtype from pydantic import BaseModel, ValidationError -from typing_extensions import NotRequired +from typing_extensions import NotRequired, deprecated from pandera import dtypes, errors from pandera.api.polars.types import PolarsData @@ -50,16 +50,7 @@ def convert_py_dtype_to_polars_dtype(dtype): if isinstance(dtype, DataTypeClass): return dtype - if polars_version().release < (1, 0, 0): - from polars.datatypes import py_type_to_dtype - - conversion_fn = py_type_to_dtype - else: - from polars.datatypes._parse import parse_py_type_into_dtype - - conversion_fn = parse_py_type_into_dtype - - return conversion_fn(dtype) + return parse_py_type_into_dtype(dtype) def polars_object_coercible( @@ -186,8 +177,6 @@ def try_coerce(self, data_container: PolarsDataContainer) -> pl.LazyFrame: raises a :class:`~pandera.errors.ParserError` if the coercion fails :raises: :class:`~pandera.errors.ParserError`: if coercion fails """ - from pandera.api.polars.utils import get_lazyframe_schema - if isinstance(data_container, pl.LazyFrame): data_container = PolarsData(data_container) @@ -208,7 +197,7 @@ def try_coerce(self, data_container: PolarsDataContainer) -> pl.LazyFrame: failure_cases = failure_cases.select(data_container.key) raise errors.ParserError( f"Could not coerce {_key} LazyFrame with schema " - f"{get_lazyframe_schema(data_container.lazyframe)} " + f"{data_container.lazyframe.collect_schema()} " f"into type {self.type}", failure_cases=failure_cases, parser_output=is_coercible, @@ -600,12 +589,24 @@ def __init__( ) -> None: ... @overload + @deprecated( + "The `width` argument of `Array` is deprecated, use `shape` instead." + ) def __init__( self, inner: PolarsDataType = ..., shape: Union[int, tuple[int, ...], None] = ..., *, - width: int | None = ..., + width: int, + ) -> None: ... + + @overload + def __init__( + self, + inner: PolarsDataType = ..., + shape: Union[int, tuple[int, ...], None] = ..., + *, + width: None = ..., ) -> None: ... def __init__( @@ -615,9 +616,19 @@ def __init__( *, width: int | None = None, ) -> None: + """Construct a Polars Array dtype. + + .. deprecated:: + The ``width`` argument is deprecated, use ``shape`` instead. + """ kwargs: _ArrayKwargs = {} if width is not None: - # width deprecated in polars 0.20.31, replaced by shape + warnings.warn( + "The `width` argument of `Array` is deprecated and will be " + "removed in a future version. Use `shape` instead.", + DeprecationWarning, + stacklevel=2, + ) kwargs["shape"] = width elif shape is not None: kwargs["shape"] = shape diff --git a/pandera/typing/polars.py b/pandera/typing/polars.py index e9acad46c..b3f271c0c 100644 --- a/pandera/typing/polars.py +++ b/pandera/typing/polars.py @@ -5,8 +5,6 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Generic, TypeVar -from packaging import version - from pandera.config import config_context from pandera.engines import PYDANTIC_V2 from pandera.errors import SchemaError, SchemaInitError @@ -26,11 +24,6 @@ from pydantic_core import core_schema -def polars_version(): - """Return the polars version.""" - return version.parse(pl.__version__) - - if TYPE_CHECKING: T = TypeVar("T") # pragma: no cover else: diff --git a/pyproject.toml b/pyproject.toml index 07436c9e5..1155b2c17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ ibis = [ "ibis-framework >= 9.0.0", "pyarrow-hotfix", ] -polars = ["polars >= 0.20.0"] +polars = ["polars >= 1.20.0"] xarray = [ "xarray >= 2024.10.0", "numpy >= 1.24.4", @@ -117,7 +117,7 @@ all = [ "shapely", "ibis-framework >= 9.0.0", "pyarrow-hotfix", - "polars >= 0.20.0", + "polars >= 1.20.0", "xarray >= 2024.10.0", "numpy >= 1.24.4", "narwhals >= 1.26.0", @@ -135,7 +135,7 @@ dev = [ "mypy == 1.10.0", "nox", "pip", - "polars >= 0.20.0", + "polars >= 1.20.0", "prek", "pyarrow >= 13", "python-multipart", diff --git a/requirements.txt b/requirements.txt index 4540b9aa4..3a527f5d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ scipy pandas-stubs scipy-stubs pyspark[connect] >= 3.2.0 -polars >= 0.20.0 +polars >= 1.20.0 xarray >= 2024.10.0 narwhals >= 1.26.0 modin diff --git a/tests/pandas/test_pydantic_dtype.py b/tests/pandas/test_pydantic_dtype.py index c4e168a9e..20b92e7e4 100644 --- a/tests/pandas/test_pydantic_dtype.py +++ b/tests/pandas/test_pydantic_dtype.py @@ -161,7 +161,9 @@ class Row(BaseModel): name: str = Field(alias="Name") amount: float = Field(alias="Amount in local currency") - schema = pa.DataFrameSchema(dtype=PydanticModel(Row), coerce=True, strict=True) + schema = pa.DataFrameSchema( + dtype=PydanticModel(Row), coerce=True, strict=True + ) data = pd.DataFrame(columns=["Name", "Amount in local currency"]) validated = schema.validate(data) assert validated.columns.tolist() == [ diff --git a/tests/polars/test_polars_check.py b/tests/polars/test_polars_check.py index 798e135c9..eb8f47bd9 100644 --- a/tests/polars/test_polars_check.py +++ b/tests/polars/test_polars_check.py @@ -6,7 +6,6 @@ import pytest import pandera.polars as pa -from pandera.api.polars.utils import get_lazyframe_schema from pandera.config import CONFIG from pandera.constants import CHECK_OUTPUT_KEY @@ -180,7 +179,7 @@ def test_polars_element_wise_dataframe_check(lf): validated_data = schema.validate(lf) assert validated_data.collect().equals(lf.collect()) - for col in get_lazyframe_schema(lf): + for col in lf.collect_schema(): invalid_lf = lf.with_columns(**{col: pl.Series([-1, 2, -4, 3])}) try: schema.validate(invalid_lf) diff --git a/tests/polars/test_polars_components.py b/tests/polars/test_polars_components.py index 438ac1b5b..b27a45821 100644 --- a/tests/polars/test_polars_components.py +++ b/tests/polars/test_polars_components.py @@ -9,7 +9,6 @@ import pytest import pandera.polars as pa -from pandera.api.polars.utils import get_lazyframe_schema from pandera.backends.base import CoreCheckResult from pandera.backends.polars.components import ColumnBackend from pandera.config import CONFIG @@ -130,7 +129,7 @@ def test_get_regex_columns(kwargs): backend = ColumnBackend() data = pl.DataFrame({f"col_{i}": [1, 2, 3] for i in range(10)}).lazy() matched_columns = backend.get_regex_columns(column_schema, data) - assert matched_columns == get_lazyframe_schema(data) + assert matched_columns == data.collect_schema() no_match_data = data.rename( lambda c: c.replace( diff --git a/tests/polars/test_polars_config.py b/tests/polars/test_polars_config.py index 6f0f1a03a..b29c8e71c 100644 --- a/tests/polars/test_polars_config.py +++ b/tests/polars/test_polars_config.py @@ -6,7 +6,6 @@ import pandera.polars as pa from pandera.api.base.error_handler import ErrorCategory -from pandera.api.polars.utils import get_lazyframe_schema from pandera.config import ( CONFIG, ValidationDepth, @@ -15,7 +14,6 @@ get_config_global, reset_config_context, ) -from pandera.engines.polars_engine import polars_version @pytest.fixture(scope="function") @@ -197,14 +195,9 @@ def test_coerce_validation_depth_none(validation_depth_none, schema): # simply calling validation shouldn't raise a coercion error, since we're # casting the types lazily validated_data = schema.validate(data) - assert get_lazyframe_schema(validated_data)["a"] == pl.Int64 + assert validated_data.collect_schema()["a"] == pl.Int64 - ErrorCls = ( - pl.exceptions.InvalidOperationError - if polars_version().release >= (1, 0, 0) - else pl.exceptions.ComputeError - ) - with pytest.raises(ErrorCls): + with pytest.raises(pl.exceptions.InvalidOperationError): validated_data.collect() # when validation explicitly with PANDERA_VALIDATION_DEPTH=SCHEMA_AND_DATA diff --git a/tests/polars/test_polars_container.py b/tests/polars/test_polars_container.py index 751e0fdd9..b32477450 100644 --- a/tests/polars/test_polars_container.py +++ b/tests/polars/test_polars_container.py @@ -7,23 +7,17 @@ import pytest from hypothesis import given from hypothesis import strategies as st +from polars._typing import PolarsDataType from polars.testing import assert_frame_equal from polars.testing.parametric import column, dataframes import pandera.polars as pa from pandera import Check as C from pandera.api.polars.types import PolarsData -from pandera.api.polars.utils import get_lazyframe_column_names +from pandera.config import CONFIG from pandera.engines import polars_engine as pe from pandera.polars import Column, DataFrameModel, DataFrameSchema -try: - from polars._typing import PolarsDataType # type: ignore -except NameError: - from polars.type_aliases import PolarsDataType # type: ignore - -from pandera.config import CONFIG - @pytest.fixture def ldf_basic(): @@ -518,7 +512,7 @@ def test_regex_selector( assert result.equals(ldf_for_regex_match.collect()) - for column in get_lazyframe_column_names(ldf_for_regex_match): + for column in ldf_for_regex_match.collect_schema().names(): # this should raise an error since columns are not nullable by default modified_data = transform_fn(ldf_for_regex_match, column) # Container wraps component errors in SchemaErrors; accept both @@ -540,7 +534,7 @@ def test_regex_selector( # has nothing to validate, matching the pandas backend. # See https://github.com/unionai-oss/pandera/issues/2364 modified_data = ldf_for_regex_match.drop( - get_lazyframe_column_names(ldf_for_regex_match) + ldf_for_regex_match.collect_schema().names() ) modified_data.pipe(schema.validate).collect() # should not raise @@ -863,7 +857,7 @@ def test_dataframe_schema_with_kwargs_nested_types(lf_with_nested_types): class ModelWithDtypeKwargs(DataFrameModel): list_col: pl.List = pa.Field(dtype_kwargs={"inner": pl.Int64()}) array_col: pl.Array = pa.Field( - dtype_kwargs={"inner": pl.Int64(), "shape": 3, "width": None} + dtype_kwargs={"inner": pl.Int64(), "shape": 3} ) struct_col: pl.Struct = pa.Field( dtype_kwargs={"fields": {"a": pl.Utf8(), "b": pl.Float64()}} diff --git a/tests/polars/test_polars_dtypes.py b/tests/polars/test_polars_dtypes.py index e08375c04..55e0d941a 100644 --- a/tests/polars/test_polars_dtypes.py +++ b/tests/polars/test_polars_dtypes.py @@ -17,10 +17,12 @@ import pandera.backends.polars.utils as polars_utils import pandera.errors from pandera.api.polars.types import PolarsData -from pandera.api.polars.utils import get_lazyframe_column_dtypes from pandera.constants import CHECK_OUTPUT_KEY from pandera.engines import polars_engine as pe -from pandera.engines.polars_engine import polars_object_coercible +from pandera.engines.polars_engine import ( + polars_object_coercible, + polars_version, +) POLARS_NUMERIC_DTYPES = [ pl.Int8, @@ -65,6 +67,8 @@ all_types = numeric_dtypes + temporal_types + other_types +pl_decimal_precision = 38 if polars_version().release >= (1, 34, 0) else 28 + def test_backend_polars_version(): """Test the backend polars_version helper.""" @@ -149,7 +153,7 @@ def test_coerce_no_cast(dtype, data): def test_coerce_no_cast_special(to_dtype, strategy): """Test that dtypes can be coerced without casting.""" coerced = to_dtype.coerce(data_container=strategy) - for dtype in get_lazyframe_column_dtypes(coerced): + for dtype in coerced.collect_schema().dtypes(): assert dtype == to_dtype.type @@ -192,7 +196,7 @@ def test_coerce_cast(from_dtype, to_dtype, strategy, data): s = data.draw(strategy(from_dtype.type)) coerced = to_dtype.coerce(data_container=s) - for dtype in get_lazyframe_column_dtypes(coerced): + for dtype in coerced.collect_schema().dtypes(): assert dtype == to_dtype.type @@ -213,33 +217,22 @@ def test_coerce_cast_special(pandera_dtype, data_container): """Test that dtypes can be coerced with casting.""" coerced = pandera_dtype.coerce(data_container=data_container) - for dtype in get_lazyframe_column_dtypes(coerced): + for dtype in coerced.collect_schema().dtypes(): assert dtype == pandera_dtype.type if isinstance(pandera_dtype, pe.Decimal): - if pe.polars_version().release < (1, 0, 0): - pytest.xfail( - reason="polars < 1.0.0 has a bug that turns decimals to floats" - ) df = coerced.collect() for dtype in df.dtypes: assert dtype == pl.Decimal -ErrorCls = ( - pl.exceptions.InvalidOperationError - if pe.polars_version().release >= (1, 0, 0) - else pl.exceptions.ComputeError -) - - @pytest.mark.parametrize( "pl_to_dtype, container, exception_cls", [ ( pe.Int8(), pl.LazyFrame({"0": [1000, 100, 200]}), - ErrorCls, + pl.exceptions.InvalidOperationError, ), ( pe.Bool(), @@ -249,12 +242,12 @@ def test_coerce_cast_special(pandera_dtype, data_container): ( pe.Int64(), pl.LazyFrame({"0": ["1", "b"]}), - ErrorCls, + pl.exceptions.InvalidOperationError, ), ( pe.Decimal(precision=2, scale=1), pl.LazyFrame({"0": [100.11, 2, 3]}), - ErrorCls, + pl.exceptions.InvalidOperationError, ), ( pe.Category(categories=["a", "b", "c"]), @@ -381,7 +374,7 @@ def test_polars_object_coercible(to_dtype, container, result): "polars_dtype, expected_dtype", [ (pl.Decimal(5, 2), pe.Decimal(5, 2)), - (pl.Decimal(None, 2), pe.Decimal(38, 2)), + (pl.Decimal(None, 2), pe.Decimal(pl_decimal_precision, 2)), ], ) def test_polars_decimal_from_parametrized_dtype(polars_dtype, expected_dtype): diff --git a/tests/polars/test_polars_model.py b/tests/polars/test_polars_model.py index a889dec8d..142c72add 100644 --- a/tests/polars/test_polars_model.py +++ b/tests/polars/test_polars_model.py @@ -158,13 +158,6 @@ class ModelWithOptional(DataFrameModel): assert ModelWithOptional.to_schema() == schema -ErrorCls = ( - pl.exceptions.InvalidOperationError - if pe.polars_version().release >= (1, 0, 0) - else pl.exceptions.ComputeError -) - - @pytest.mark.parametrize( "column_mod,exception_cls", [ @@ -172,7 +165,7 @@ class ModelWithOptional(DataFrameModel): # values in ldf_basic will cause the error outside of pandera validation pytest.param( {"string_col": pl.Int64}, - ErrorCls, + pl.exceptions.InvalidOperationError, marks=pytest.mark.xfail( condition=CONFIG.use_narwhals_backend, reason="Narwhals raises narwhals.exceptions.InvalidOperationError, not polars.exceptions.InvalidOperationError", diff --git a/tests/polars/test_polars_typing.py b/tests/polars/test_polars_typing.py index 8c3be5822..99188d81f 100644 --- a/tests/polars/test_polars_typing.py +++ b/tests/polars/test_polars_typing.py @@ -11,7 +11,7 @@ from pandera.engines import PYDANTIC_V2 from pandera.errors import SchemaInitError from pandera.typing.formats import Formats -from pandera.typing.polars import DataFrame, Series, polars_version +from pandera.typing.polars import DataFrame, Series try: if PYDANTIC_V2: @@ -23,13 +23,6 @@ PYDANTIC_INSTALLED = False -def test_polars_version(): - """Test the polars_version function.""" - # We need to check equality as strings because Version objects don't compare - # directly to string versions - assert str(polars_version()) == pl.__version__ - - def test_type_vars(): """Test TYPE_CHECKING behavior for TypeVar T.""" # This is a bit tricky to test as it's a conditional import