fix: add compat with pandas 3 (#3929)

Co-authored-by: Ilan Gold <ilanbassgold@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

.github/workflows/ci.yml

@@ -60,8 +60,13 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv tool install --with='click!=8.3.0' hatch
+          echo "::group::Install hatch"
+          uv tool install hatch
+          echo "::endgroup::"
+          echo "::group::Create environment"
           hatch -v env create ${{ matrix.env.name }}
+          echo "::endgroup::"
+          hatch run ${{ matrix.env.name }}:session-info scanpy anndata
 
       - name: Run tests
         if: matrix.env.test-type == null

docs/release-notes/3929.fix.md

@@ -0,0 +1 @@
+Fix compatibility with pandas 3.0 {smaller}`P Angerer`

hatch.toml

@@ -35,6 +35,7 @@ overrides.matrix.deps.python = [
 ]
 overrides.matrix.deps.extra-dependencies = [
     { if = [ "pre" ], value = "anndata @ git+https://github.com/scverse/anndata.git" },
+    { if = [ "pre" ], value = "pandas>=3rc0" },
 ]
 overrides.matrix.deps.dependency-groups = [
     { if = [ "stable", "pre", "low-vers" ], value = "test" },

pyproject.toml

@@ -54,7 +54,7 @@ dependencies = [
     "numpy>=2",
     "fast-array-utils[accel,sparse]>=1.2.1",
     "matplotlib>=3.9",
-    "pandas >=2.2.2, <3.0.0rc0",
+    "pandas >=2.2.2",
     "scipy>=1.13",
     "seaborn>=0.13.2",
     "h5py>=3.11",

src/scanpy/_utils/__init__.py

@@ -29,6 +29,7 @@
 
 import h5py
 import numpy as np
+import pandas as pd
 from anndata._core.sparse_dataset import BaseCompressedSparseDataset
 from packaging.version import Version
 
@@ -44,6 +45,7 @@
     from anndata import AnnData
     from igraph import Graph
     from numpy.typing import ArrayLike, NDArray
+    from pandas._typing import Dtype as PdDtype
 
     from .._compat import CSRBase
     from ..neighbors import NeighborsParams, RPForestDict
@@ -79,6 +81,7 @@
     "sanitize_anndata",
     "select_groups",
     "update_params",
+    "with_cat_dtype",
 ]
 
 
@@ -287,7 +290,7 @@ def get_igraph_from_adjacency(adjacency: CSBase, *, directed: bool = False) -> G
     import igraph as ig
 
     sources, targets = adjacency.nonzero()
-    weights = dematrix(adjacency[sources, targets]).ravel()
+    weights = dematrix(adjacency[sources, targets]).ravel() if len(sources) else []
     g = ig.Graph(directed=directed)
     g.add_vertices(adjacency.shape[0])  # this adds adjacency.shape[0] vertices
     g.add_edges(list(zip(sources, targets, strict=True)))
@@ -494,6 +497,23 @@ def moving_average(a: np.ndarray, n: int):
     return ret[n - 1 :] / n
 
 
+@singledispatch
+def with_cat_dtype[X: pd.Series | pd.CategoricalIndex | pd.Categorical](
+    x: X, dtype: PdDtype
+) -> X:
+    raise NotImplementedError
+
+
+@with_cat_dtype.register(pd.Series)
+def _(x: pd.Series, dtype: PdDtype) -> pd.Series:
+    return x.cat.set_categories(x.cat.categories.astype(dtype))
+
+
+@with_cat_dtype.register(pd.Categorical | pd.CategoricalIndex)
+def _[X: pd.Categorical | pd.CategoricalIndex](x: X, dtype: PdDtype) -> X:
+    return x.set_categories(x.categories.astype(dtype))
+
+
 # --------------------------------------------------------------------------------
 # Deal with tool parameters
 # --------------------------------------------------------------------------------

src/scanpy/external/exporting.py

@@ -219,7 +219,7 @@ def spring_project(  # noqa: PLR0912, PLR0915
     np.save(subplot_dir / "cell_filter.npy", np.arange(x.shape[0]))
 
     # Write 2-D coordinates, after adjusting to roughly match SPRING's default d3js force layout parameters
-    coords = coords - coords.min(0)[None, :]
+    coords = coords - coords.min(axis=0)[None, :]
     coords = (
         coords * (np.array([1000, 1000]) / coords.ptp(0))[None, :]
         + np.array([200, -200])[None, :]
@@ -342,8 +342,8 @@ def _get_color_stats_genes(color_stats, x, gene_list):
     means, variances = mean_var(x, axis=0, correction=1)
     stdevs = np.zeros(variances.shape, dtype=float)
     stdevs[variances > 0] = np.sqrt(variances[variances > 0])
-    mins = x.min(0).todense().A1
-    maxes = x.max(0).todense().A1
+    mins = x.min(axis=0).todense().A1
+    maxes = x.max(axis=0).todense().A1
 
     pctl = 99.6
     pctl_n = (100 - pctl) / 100.0 * x.shape[0]

src/scanpy/get/get.py

@@ -259,8 +259,8 @@ def obs_df(
     >>> plotdf = sc.get.obs_df(
     ...     pbmc, keys=["CD8B", "n_genes"], obsm_keys=[("X_umap", 0), ("X_umap", 1)]
     ... )
-    >>> plotdf.columns
-    Index(['CD8B', 'n_genes', 'X_umap-0', 'X_umap-1'], dtype='object')
+    >>> plotdf.columns.astype("string")
+    Index(['CD8B', 'n_genes', 'X_umap-0', 'X_umap-1'], dtype='string')
     >>> plotdf.plot.scatter("X_umap-0", "X_umap-1", c="CD8B")  # doctest: +SKIP
     <Axes: xlabel='X_umap-0', ylabel='X_umap-1'>
 

src/scanpy/plotting/_anndata.py

@@ -749,7 +749,7 @@ def violin(  # noqa: PLR0912, PLR0913, PLR0915
     layer: str | None = None,
     density_norm: DensityNorm = "width",
     order: Sequence[str] | None = None,
-    multi_panel: bool | None = None,
+    multi_panel: bool = False,
     xlabel: str = "",
     ylabel: str | Sequence[str] | None = None,
     rotation: float | None = None,
@@ -1202,11 +1202,11 @@ def heatmap(  # noqa: PLR0912, PLR0913, PLR0915
     ).issubset(categories)
 
     if standard_scale == "obs":
-        obs_tidy = obs_tidy.sub(obs_tidy.min(1), axis=0)
-        obs_tidy = obs_tidy.div(obs_tidy.max(1), axis=0).fillna(0)
+        obs_tidy = obs_tidy.sub(obs_tidy.min(axis=1), axis=0)
+        obs_tidy = obs_tidy.div(obs_tidy.max(axis=1), axis=0).fillna(0)
     elif standard_scale == "var":
-        obs_tidy -= obs_tidy.min(0)
-        obs_tidy = (obs_tidy / obs_tidy.max(0)).fillna(0)
+        obs_tidy -= obs_tidy.min(axis=0)
+        obs_tidy = (obs_tidy / obs_tidy.max(axis=0)).fillna(0)
     elif standard_scale is None:
         pass
     else:

src/scanpy/plotting/_dotplot.py

@@ -213,11 +213,13 @@ def __init__(  # noqa: PLR0913
                 dot_color_df = self.obs_tidy.groupby(level=0, observed=True).mean()
 
             if standard_scale == "group":
-                dot_color_df = dot_color_df.sub(dot_color_df.min(1), axis=0)
-                dot_color_df = dot_color_df.div(dot_color_df.max(1), axis=0).fillna(0)
+                dot_color_df = dot_color_df.sub(dot_color_df.min(axis=1), axis=0)
+                dot_color_df = dot_color_df.div(
+                    dot_color_df.max(axis=1), axis=0
+                ).fillna(0)
             elif standard_scale == "var":
-                dot_color_df -= dot_color_df.min(0)
-                dot_color_df = (dot_color_df / dot_color_df.max(0)).fillna(0)
+                dot_color_df -= dot_color_df.min(axis=0)
+                dot_color_df = (dot_color_df / dot_color_df.max(axis=0)).fillna(0)
             elif standard_scale is None:
                 pass
             else:
@@ -696,10 +698,10 @@ def _dotplot(  # noqa: PLR0912, PLR0913, PLR0915
                 group_axis = 1
         if standard_scale is not None:
             dot_color = dot_color.sub(
-                dot_color.min((group_axis + 1) % 2), axis=group_axis
+                dot_color.min(axis=1 - group_axis), axis=group_axis
             )
             dot_color = dot_color.div(
-                dot_color.max((group_axis + 1) % 2), axis=group_axis
+                dot_color.max(axis=1 - group_axis), axis=group_axis
             ).fillna(0)
         # make scatter plot in which
         # x = var_names

src/scanpy/plotting/_matrixplot.py

@@ -180,11 +180,11 @@ def __init__(  # noqa: PLR0913
             )
 
             if standard_scale == "group":
-                values_df = values_df.sub(values_df.min(1), axis=0)
-                values_df = values_df.div(values_df.max(1), axis=0).fillna(0)
+                values_df = values_df.sub(values_df.min(axis=1), axis=0)
+                values_df = values_df.div(values_df.max(axis=1), axis=0).fillna(0)
             elif standard_scale == "var":
-                values_df -= values_df.min(0)
-                values_df = (values_df / values_df.max(0)).fillna(0)
+                values_df -= values_df.min(axis=0)
+                values_df = (values_df / values_df.max(axis=0)).fillna(0)
             elif standard_scale is None:
                 pass
             else: