Merge branch 'main' into pre-commit-ci-update-config

Author: flying-sheep

.gitignore

@@ -12,6 +12,7 @@ __pycache__/
 # Distribution / packaging
 /dist/
 /ci/min-deps.txt
+/ci/pre-deps.txt
 /requirements*.lock
 /.python-version
 

docs/release-notes/2008.bugfix.md

@@ -0,0 +1 @@
+Lower bound `xarray` by `2025.06.01`.  {class}`pandas.arrays.StringArray` was previously used as the in-memory `nullable-string-array` container in `xarray`, but due to {issue}`pydata/xarray#10419` now uses {class}`numpy.ndarray` with an object data type.  {user}`ilan-gold`

hatch.toml

@@ -20,10 +20,14 @@ extra-dependencies = [ "ipykernel" ]
 env-vars.UV_CONSTRAINT = "ci/constraints.txt"
 overrides.matrix.deps.env-vars = [
     { if = [ "pre" ], key = "UV_PRERELEASE", value = "allow" },
+    { if = [ "pre" ], key = "UV_CONSTRAINT", value = "ci/pre-deps.txt" },
     { if = [ "min" ], key = "UV_CONSTRAINT", value = "ci/constraints.txt ci/min-deps.txt" },
 ]
 overrides.matrix.deps.pre-install-commands = [
     { if = [ "min" ], value = "uv run ci/scripts/min-deps.py pyproject.toml --all-extras -o ci/min-deps.txt" },
+    # To prevent situations like https://github.com/pydata/xarray/issues/10419 going forward
+    { if = [ "pre" ], value = "echo xarray @ git+https://github.com/pydata/xarray.git > ci/pre-deps.txt" },
+
 ]
 overrides.matrix.deps.python = [
     { if = [ "min" ], value = "3.11" },

pyproject.toml

@@ -60,7 +60,7 @@ Home-page = "https://github.com/scverse/anndata"
 dev = [
     # runtime dev version generation
     "hatch-vcs",
-    "anndata[dev-doc,test]",
+    "anndata[dev-doc]",
 ]
 doc = [
     "sphinx>=8.2.1",
@@ -109,7 +109,7 @@ gpu = [ "cupy" ]
 cu12 = [ "cupy-cuda12x" ]
 cu11 = [ "cupy-cuda11x" ]
 # requests and aiohttp needed for zarr remote data
-lazy = [ "xarray>=2025.04.0", "aiohttp", "requests", "anndata[dask]" ]
+lazy = [ "xarray>=2025.06.1", "aiohttp", "requests", "anndata[dask]" ]
 # https://github.com/dask/dask/issues/11290
 # https://github.com/dask/dask/issues/11752
 dask = [ "dask[array]>=2023.5.1,!=2024.8.*,!=2024.9.*,<2025.2.0" ]

src/anndata/_core/xarray.py

@@ -1,12 +1,26 @@
 from __future__ import annotations
 
 import warnings
+from functools import wraps
 
 import pandas as pd
 
 from ..compat import XDataArray, XDataset, XVariable
 
 
+def requires_xarray(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            import xarray  # noqa: F401
+        except ImportError as e:
+            msg = "xarray is required to read dataframes lazily. Please install xarray."
+            raise ImportError(msg) from e
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
 class Dataset2D(XDataset):
     """
     A wrapper class meant to enable working with lazy dataframe data.
@@ -119,10 +133,18 @@ def __getitem__(self, idx) -> Dataset2D:
         return ret
 
     def to_memory(self, *, copy=False) -> pd.DataFrame:
+        # https://github.com/pydata/xarray/issues/10419
+        non_nullable_string_cols = {
+            col
+            for col in self.columns
+            if not self[col].attrs.get("is_nullable_string", False)
+        }
         df = self.to_dataframe()
         index_key = self.attrs.get("indexing_key", None)
         if df.index.name != index_key and index_key is not None:
             df = df.set_index(index_key)
+        for col in set(self.columns) - non_nullable_string_cols:
+            df[col] = pd.array(self[col].data, dtype="string")
         df.index.name = None  # matches old AnnData object
         return df
 

src/anndata/_io/specs/lazy_methods.py

@@ -12,7 +12,7 @@
 
 import anndata as ad
 from anndata._core.file_backing import filename, get_elem_name
-from anndata._core.xarray import Dataset2D
+from anndata._core.xarray import Dataset2D, requires_xarray
 from anndata.abc import CSCDataset, CSRDataset
 from anndata.compat import DaskArray, H5Array, H5Group, XDataArray, ZarrArray, ZarrGroup
 
@@ -241,6 +241,9 @@ def _gen_xarray_dict_iterator_from_elems(
                 attrs={
                     "base_path_or_zarr_group": v.base_path_or_zarr_group,
                     "elem_name": v.elem_name,
+                    "is_nullable_string": isinstance(v, MaskedArray)
+                    and v.dtype  # CategoricalArray dtype access requires a read nad is not necessary here
+                    == np.dtype("O"),
                 },
             )
         elif k == dim_name:
@@ -258,6 +261,7 @@ def _gen_xarray_dict_iterator_from_elems(
 
 @_LAZY_REGISTRY.register_read(ZarrGroup, IOSpec("dataframe", "0.2.0"))
 @_LAZY_REGISTRY.register_read(H5Group, IOSpec("dataframe", "0.2.0"))
+@requires_xarray
 def read_dataframe(
     elem: H5Group | ZarrGroup,
     *,
@@ -297,6 +301,7 @@ def read_dataframe(
 
 @_LAZY_REGISTRY.register_read(ZarrGroup, IOSpec("categorical", "0.2.0"))
 @_LAZY_REGISTRY.register_read(H5Group, IOSpec("categorical", "0.2.0"))
+@requires_xarray
 def read_categorical(
     elem: H5Group | ZarrGroup,
     *,
@@ -317,6 +322,7 @@ def read_categorical(
     )
 
 
+@requires_xarray
 def read_nullable(
     elem: H5Group | ZarrGroup,
     *,

src/anndata/experimental/backed/_io.py

@@ -13,6 +13,7 @@
 from testing.anndata._doctest import doctest_needs
 
 from ..._core.anndata import AnnData
+from ..._core.xarray import requires_xarray
 from ..._settings import settings
 from ...compat import ZarrGroup, is_zarr_v2
 from .. import read_dispatched
@@ -25,6 +26,7 @@
 
 
 @doctest_needs("xarray")
+@requires_xarray
 def read_lazy(
     store: PathLike[str] | str | MutableMapping | ZarrGroup | h5py.Dataset,
     *,
@@ -81,13 +83,6 @@ def read_lazy(
     AnnData object with n_obs × n_vars = 490 × 33452
         obs: 'donor_id', 'self_reported_ethnicity_ontology_term_id', 'organism_ontology_term_id'...
     """
-    try:
-        import xarray  # noqa: F401
-    except ImportError as e:
-        msg = (
-            "xarray is required to use the `read_lazy` function. Please install xarray."
-        )
-        raise ImportError(msg) from e
     is_h5_store = isinstance(store, h5py.Dataset | h5py.File | h5py.Group)
     is_h5 = (
         isinstance(store, PathLike | str) and Path(store).suffix == ".h5ad"

src/anndata/experimental/backed/_lazy_arrays.py

@@ -3,6 +3,7 @@
 from functools import cached_property
 from typing import TYPE_CHECKING, Generic, TypeVar
 
+import numpy as np
 import pandas as pd
 
 from anndata._core.index import _subset
@@ -18,8 +19,6 @@
     from pathlib import Path
     from typing import Literal
 
-    import numpy as np
-
     from anndata._core.index import Index
     from anndata.compat import ZarrGroup
 
@@ -137,7 +136,7 @@ def __init__(
 
     def __getitem__(
         self, key: xr.core.indexing.ExplicitIndexer
-    ) -> xr.core.extension_array.PandasExtensionArray:
+    ) -> xr.core.extension_array.PandasExtensionArray | np.ndarray:
         values = self._values[key]
         mask = self._mask[key]
         if self._dtype_str == "nullable-integer":
@@ -146,8 +145,9 @@ def __getitem__(
         elif self._dtype_str == "nullable-boolean":
             extension_array = pd.arrays.BooleanArray(values, mask=mask)
         elif self._dtype_str == "nullable-string-array":
-            values[mask] = pd.NA
-            extension_array = pd.array(values, dtype=pd.StringDtype())
+            # https://github.com/pydata/xarray/issues/10419
+            values[mask] = np.nan
+            return values
         else:
             msg = f"Invalid dtype_str {self._dtype_str}"
             raise RuntimeError(msg)
@@ -163,7 +163,8 @@ def dtype(self):
         elif self._dtype_str == "nullable-boolean":
             return pd.BooleanDtype()
         elif self._dtype_str == "nullable-string-array":
-            return pd.StringDtype()
+            # https://github.com/pydata/xarray/issues/10419
+            return np.dtype("O")
         msg = f"Invalid dtype_str {self._dtype_str}"
         raise RuntimeError(msg)
 

src/anndata/tests/helpers.py

@@ -6,6 +6,7 @@
 from collections import Counter, defaultdict
 from collections.abc import Mapping
 from functools import partial, singledispatch, wraps
+from importlib.util import find_spec
 from string import ascii_letters
 from typing import TYPE_CHECKING
 
@@ -311,7 +312,6 @@ def gen_adata(  # noqa: PLR0913
         (csr, csc)
     """
     import dask.array as da
-    import xarray as xr
 
     if random_state is None:
         random_state = np.random.default_rng()
@@ -325,10 +325,11 @@ def gen_adata(  # noqa: PLR0913
     obs.rename(columns=dict(cat="obs_cat"), inplace=True)
     var.rename(columns=dict(cat="var_cat"), inplace=True)
 
-    if obs_xdataset:
-        obs = XDataset.from_dataframe(obs)
-    if var_xdataset:
-        var = XDataset.from_dataframe(var)
+    if has_xr := find_spec("xarray"):
+        if obs_xdataset:
+            obs = XDataset.from_dataframe(obs)
+        if var_xdataset:
+            var = XDataset.from_dataframe(var)
 
     if X_type is None:
         X = None
@@ -341,27 +342,28 @@ def gen_adata(  # noqa: PLR0913
         df=gen_typed_df(M, obs_names, dtypes=obs_dtypes),
         awk_2d_ragged=gen_awkward((M, None)),
         da=da.random.random((M, 50)),
-        xdataset=xr.Dataset.from_dataframe(
-            gen_typed_df(M, obs_names, dtypes=obs_dtypes)
-        ),
-    )
-    obsm = {k: v for k, v in obsm.items() if type(v) in obsm_types}
-    obsm = maybe_add_sparse_array(
-        mapping=obsm,
-        types=obsm_types,
-        format=sparse_fmt,
-        random_state=random_state,
-        shape=(M, 100),
     )
     varm = dict(
         array=np.random.random((N, 50)),
         sparse=sparse.random(N, 100, format=sparse_fmt, random_state=random_state),
         df=gen_typed_df(N, var_names, dtypes=var_dtypes),
         awk_2d_ragged=gen_awkward((N, None)),
         da=da.random.random((N, 50)),
-        xdataset=xr.Dataset.from_dataframe(
+    )
+    if has_xr:
+        obsm["xdataset"] = XDataset.from_dataframe(
+            gen_typed_df(M, obs_names, dtypes=obs_dtypes)
+        )
+        varm["xdataset"] = XDataset.from_dataframe(
             gen_typed_df(N, var_names, dtypes=var_dtypes)
-        ),
+        )
+    obsm = {k: v for k, v in obsm.items() if type(v) in obsm_types}
+    obsm = maybe_add_sparse_array(
+        mapping=obsm,
+        types=obsm_types,
+        format=sparse_fmt,
+        random_state=random_state,
+        shape=(M, 100),
     )
     varm = {k: v for k, v in varm.items() if type(v) in varm_types}
     varm = maybe_add_sparse_array(

tests/lazy/test_read.py

@@ -25,6 +25,7 @@
 
     from anndata._types import AnnDataElem
 
+
 pytestmark = pytest.mark.skipif(not find_spec("xarray"), reason="xarray not installed")