feat(pyarrow): add first-class pyarrow.Table schema validation

.github/workflows/ci-tests.yml

@@ -335,6 +335,7 @@ jobs:
         - modin-ray
         - ibis
         - xarray
+        - pyarrow
         include:
         - extra: polars
           polars-version: "0.20.0"

docs/source/index.md

@@ -159,6 +159,7 @@ pip install 'pandera[modin-dask]'   # validate modin dataframes with dask
 pip install 'pandera[geopandas]'    # validate geopandas geodataframes
 pip install 'pandera[polars]'       # validate polars dataframes
 pip install 'pandera[ibis]'         # validate ibis tables
+pip install 'pandera[pyarrow]'      # validate pyarrow tables
 pip install 'pandera[xarray]'       # validate xarray data structures
 pip install 'pandera[narwhals]'     # use the Narwhals-powered backend
 ```

docs/source/pyarrow.md

@@ -0,0 +1,138 @@
+---
+file_format: mystnb
+---
+
+% pandera documentation for pyarrow
+
+```{currentmodule} pandera.pyarrow
+```
+
+(pyarrow)=
+
+# Data Validation with PyArrow
+
+*new in 0.33.0*
+
+[PyArrow](https://arrow.apache.org/docs/python/) tables are a common exchange
+format between systems, and `pyarrow.Table` carries its own schema. Pandera
+supports validating them directly so you get dtype checking, value checks and
+class-based models on top of that schema.
+
+## Installation
+
+```bash
+pip install 'pandera[pyarrow]'
+```
+
+Validation is performed by pandera's
+{ref}`narwhals backend <narwhals-backend>`, so `narwhals` is installed
+alongside `pyarrow`.
+
+## `DataFrameSchema`
+
+Import the pyarrow entry point and define a schema as you would for any other
+backend:
+
+```{code-cell} python
+import pyarrow
+import pandera.pyarrow as pa
+
+schema = pa.DataFrameSchema(
+    {
+        "state": pa.Column(str),
+        "city": pa.Column(str),
+        "price": pa.Column(int, pa.Check.in_range(5, 20)),
+    }
+)
+
+table = pyarrow.table(
+    {
+        "state": ["FL", "FL", "CA", "CA"],
+        "city": ["Orlando", "Miami", "Los Angeles", "San Francisco"],
+        "price": [8, 12, 10, 16],
+    }
+)
+
+schema.validate(table)
+```
+
+`validate` returns a `pyarrow.Table`, so schemas drop into an existing pipeline
+without changing types.
+
+## `DataFrameModel`
+
+```{code-cell} python
+class Schema(pa.DataFrameModel):
+    state: str
+    city: str
+    price: int = pa.Field(in_range={"min_value": 5, "max_value": 20})
+
+
+Schema.validate(table)
+```
+
+Annotate function signatures with `pandera.typing.pyarrow.Table` and use
+{func}`~pandera.decorators.check_types` to validate inputs and outputs:
+
+```{code-cell} python
+from pandera.typing.pyarrow import Table
+
+
+@pa.check_types
+def transform(df: Table[Schema]) -> Table[Schema]:
+    return df
+
+
+transform(table)
+```
+
+## Supported data types
+
+Columns accept native pyarrow types, python builtins, and their string
+aliases — all resolve to the same underlying pandera datatype:
+
+```{code-cell} python
+pa.DataFrameSchema(
+    {
+        "a": pa.Column(pyarrow.int64()),
+        "b": pa.Column(int),
+        "c": pa.Column("int64"),
+    }
+)
+```
+
+Parametrized pyarrow types such as `pyarrow.timestamp("us")`,
+`pyarrow.decimal128(10, 2)` and `pyarrow.list_(pyarrow.int32())` are supported.
+
+## Custom checks
+
+Check functions receive a {class}`~pandera.api.pyarrow.types.PyArrowData`
+container holding the native table and the column key, mirroring `PolarsData`
+and `IbisData` on the other backends:
+
+```{code-cell} python
+import pyarrow.compute as pc
+
+schema = pa.DataFrameSchema(
+    {"price": pa.Column(int, pa.Check(lambda data: pc.greater(data.table[data.key], 0)))}
+)
+schema.validate(table)
+```
+
+A check function taking two positional arguments receives the native table and
+the key directly, i.e. `check_fn(table, key)`.
+
+## Validation depth
+
+Unlike a `polars.LazyFrame` or an `ibis.Table`, a `pyarrow.Table` is always
+fully materialized in memory, so pandera runs both schema-level and data-level
+checks by default. Set `PANDERA_VALIDATION_DEPTH=SCHEMA_ONLY` (or use
+{func}`~pandera.config.config_context`) to skip data-level checks.
+
+## Limitations
+
+- **`coerce=True` is not applied.** Coercion is not yet implemented in the
+  narwhals backend that serves pyarrow; a `SchemaWarning` is emitted and a
+  dtype mismatch is reported as a `WRONG_DATATYPE` error instead.
+- **Data synthesis strategies are not supported**, matching the polars and ibis
+  backends.

docs/source/supported_libraries.md

@@ -23,6 +23,8 @@ Pandera supports validation of the following DataFrame libraries:
   - Validate Polars dataframes. Polars is a blazingly fast dataframe library.
 * - {ref}`Ibis <ibis>`
   - Validate Ibis tables. Ibis is the portable Python dataframe library.
+* - {ref}`PyArrow <pyarrow>`
+  - Validate PyArrow tables. Arrow is the in-memory columnar exchange format.
 * - {ref}`PySpark SQL <native-pyspark>`
   - A data processing library for large-scale data.
 :::
@@ -33,6 +35,7 @@ Pandera supports validation of the following DataFrame libraries:
 
 Polars <polars>
 Ibis <ibis>
+PyArrow <pyarrow>
 PySpark SQL <pyspark_sql>
 ```
 

noxfile.py

@@ -221,6 +221,7 @@ def _testing_requirements(
     "ibis",
     "xarray",
     "narwhals",  # TEST-03: narwhals backend runs with polars+ibis co-installed
+    "pyarrow",  # pyarrow.Table validation, served by the narwhals backends
 }
 for extra in OPTIONAL_DEPENDENCIES:
     if extra == "pandas":

pandera/api/pyarrow/components.py

@@ -0,0 +1,174 @@
+"""Core PyArrow schema component specifications."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from pandera.api.base.types import CheckList
+from pandera.api.dataframe.components import ComponentSchema
+from pandera.api.pyarrow.types import (
+    PyArrowCheckObjects,
+    PyArrowDtypeInputTypes,
+)
+from pandera.api.pyarrow.utils import resolve_dtype
+from pandera.backends.pyarrow.register import register_pyarrow_backends
+from pandera.utils import is_regex
+
+logger = logging.getLogger(__name__)
+
+
+class Column(ComponentSchema[PyArrowCheckObjects]):
+    """Validate types and properties of pyarrow table columns."""
+
+    def __init__(
+        self,
+        dtype: PyArrowDtypeInputTypes | None = None,
+        checks: CheckList | None = None,
+        nullable: bool = False,
+        unique: bool = False,
+        coerce: bool = False,
+        required: bool = True,
+        name: str | None = None,
+        regex: bool = False,
+        title: str | None = None,
+        description: str | None = None,
+        default: Any | None = None,
+        metadata: dict | None = None,
+        drop_invalid_rows: bool = False,
+        **column_kwargs,
+    ) -> None:
+        """Create column validator object.
+
+        :param dtype: datatype of the column. Accepts native ``pyarrow``
+            types (e.g. ``pyarrow.int64()``), narwhals dtypes, the pandera
+            abstract datatypes, supported python builtins (``int``, ``float``,
+            ``str``, ``bool``) and their string aliases.
+        :param checks: checks to verify validity of the column
+        :param nullable: Whether or not column can contain null values.
+        :param unique: whether column values should be unique
+        :param coerce: If True, when schema.validate is called the column will
+            be coerced into the specified dtype. This has no effect on columns
+            where ``dtype=None``.
+        :param required: Whether or not column is allowed to be missing
+        :param name: column name in the table to validate. Names in the format
+            '^{regex_pattern}$' are treated as regular expressions. During
+            validation, this schema will be applied to any columns matching
+            this pattern.
+        :param regex: whether the ``name`` attribute should be treated as a
+            regex pattern to apply to multiple columns in a table.
+        :param title: A human-readable label for the column.
+        :param description: An arbitrary textual description of the column.
+        :param default: The default value for missing values in the column.
+        :param metadata: An optional key value data.
+        :param drop_invalid_rows: if True, drop invalid rows on validation.
+
+        :raises SchemaInitError: if impossible to build schema from parameters
+
+        :example:
+
+        >>> import pyarrow
+        >>> import pandera.pyarrow as pa
+        >>>
+        >>> schema = pa.DataFrameSchema({"column": pa.Column(str)})
+        >>> schema.validate(pyarrow.table({"column": ["foo", "bar"]}))
+        pyarrow.Table
+        column: string
+        ----
+        column: [["foo","bar"]]
+        """
+        super().__init__(
+            dtype=dtype,
+            checks=checks,
+            nullable=nullable,
+            unique=unique,
+            coerce=coerce,
+            name=name,
+            title=title,
+            description=description,
+            default=default,
+            metadata=metadata,
+            drop_invalid_rows=drop_invalid_rows,
+            **column_kwargs,
+        )
+        self.required = required
+        self.regex = regex
+        self.name = name
+
+        self.set_regex()
+
+    @staticmethod
+    def register_default_backends(check_obj_cls: type):
+        register_pyarrow_backends()
+
+    @property
+    def dtype(self):
+        return self._dtype
+
+    @dtype.setter
+    def dtype(self, value) -> None:
+        self._dtype = resolve_dtype(value)
+
+    @property
+    def selector(self):
+        if self.name is not None and not is_regex(self.name) and self.regex:
+            return f"^{self.name}$"
+        return self.name
+
+    def set_regex(self):
+        if self.name is None:
+            return
+
+        if is_regex(self.name) and not self.regex:
+            logger.info(
+                f"Column schema '{self.name}' is a regex expression. "
+                "Setting regex=True."
+            )
+            self.regex = True
+
+    def set_name(self, name: str):
+        """Set or modify the name of a column object.
+
+        :param str name: the name of the column object
+        """
+        self.name = name
+        self.set_regex()
+        return self
+
+    @property
+    def properties(self) -> dict[str, Any]:
+        """Get column properties."""
+        return {
+            "dtype": self.dtype,
+            "parsers": self.parsers,
+            "checks": self.checks,
+            "nullable": self.nullable,
+            "unique": self.unique,
+            "report_duplicates": self.report_duplicates,
+            "coerce": self.coerce,
+            "required": self.required,
+            "name": self.name,
+            "regex": self.regex,
+            "title": self.title,
+            "description": self.description,
+            "default": self.default,
+            "metadata": self.metadata,
+        }
+
+    def strategy(self, *, size=None):
+        """Data synthesis is not supported for pyarrow schemas."""
+        raise NotImplementedError(
+            "Data synthesis is not supported with pyarrow schemas."
+        )
+
+    def strategy_component(self):
+        """Data synthesis is not supported for pyarrow schemas."""
+        raise NotImplementedError(
+            "Data synthesis is not supported with pyarrow schemas."
+        )
+
+    def example(self, size=None):
+        """Data synthesis is not supported for pyarrow schemas."""
+        raise NotImplementedError(
+            "Data synthesis is not supported with pyarrow schemas."
+        )