docs: Sphinx 9 and fix member skipping (#2298)

Author: flying-sheep

docs/_templates/autosummary/class-minimal.rst

@@ -0,0 +1,24 @@
+{{ fullname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+..  autoclass:: {{ objname }}
+    :show-inheritance:
+
+    {% block attributes %}
+    {%- for item in attributes %}
+    {%- if loop.first %}
+    .. rubric:: Attributes
+    {% endif %}
+    .. autoattribute:: {{ item }}
+    {%- endfor %}
+    {% endblock %}
+
+    {% block methods %}
+    {%- for item in methods if item != "__init__" and item not in inherited_members %}
+    {%- if loop.first %}
+    .. rubric:: Methods
+    {% endif %}
+    .. automethod:: {{ item }}
+    {%- endfor %}
+    {% endblock %}

docs/_templates/autosummary/class.rst

@@ -4,30 +4,28 @@
 
 .. add toctree option to make autodoc generate the pages
 
-.. autoclass:: {{ objname }}
+..  autoclass:: {{ objname }}
 
-   {% block attributes %}
-   {% if attributes %}
-   .. rubric:: Attributes
+    {% block attributes %}
+    {%- for item in attributes %}
+    {%- if loop.first %}
+    .. rubric:: Attributes
 
-   .. autosummary::
-      :toctree: .
-   {% for item in attributes %}
-      ~{{ name }}.{{ item }}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
+    ..  autosummary::
+        :toctree: .
+    {% endif %}
+        ~{{ name }}.{{ item }}
+    {%- endfor %}
+    {% endblock %}
 
-   {% block methods %}
-   {% if methods %}
-   .. rubric:: Methods
+    {% block methods %}
+    {%- for item in methods if item != "__init__" and item not in inherited_members %}
+    {%- if loop.first %}
+    .. rubric:: Methods
 
-   .. autosummary::
-      :toctree: .
-   {% for item in methods %}
-      {%- if item != '__init__' %}
-      ~{{ name }}.{{ item }}
-      {%- endif -%}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
+    ..  autosummary::
+        :toctree: .
+    {% endif %}
+        ~{{ name }}.{{ item }}
+    {%- endfor %}
+    {% endblock %}

docs/api.md

@@ -173,17 +173,27 @@ Types used by the former:
 ```{eval-rst}
 .. autosummary::
    :toctree: generated/
+   :template: class-minimal
+   :signatures: none
 
    experimental.IOSpec
    experimental.Read
    experimental.Write
    experimental.ReadCallback
    experimental.WriteCallback
-   experimental.StorageType
    experimental.backed.MaskedArray
    experimental.backed.CategoricalArray
    experimental.backed.Dataset2D
    experimental.Dataset2DIlocIndexer
+
+..
+    this is not a class/protocol so since the above
+    specifies a template, it gets used.
+
+.. autosummary::
+   :toctree: generated/
+
+   experimental.StorageType
 ```
 
 (extensions-api)=
@@ -237,9 +247,18 @@ Types used by the former:
 ```{eval-rst}
 .. autosummary::
    :toctree: generated/
+   :template: class-minimal
 
    abc.CSRDataset
    abc.CSCDataset
+```
+
+.. these are types, not classes
+
+```{eval-rst}
+.. autosummary::
+   :toctree: generated/
+
    typing.Index
    typing.AxisStorable
    typing.RWAble

docs/conf.py

@@ -60,7 +60,6 @@
     "scanpydoc",  # needs to be before linkcode
     "sphinx.ext.linkcode",
     "IPython.sphinxext.ipython_console_highlighting",
-    "sphinx_toolbox.more_autodoc.autoprotocol",
     *(p.stem for p in _extension_dir.glob("*.py")),
 ]
 myst_enable_extensions = [
@@ -144,6 +143,7 @@ def res(
 )
 
 qualname_overrides = {
+    "types.EllipsisType": ("py:data", "Ellipsis"),
     "h5py._hl.group.Group": "h5py.Group",
     "h5py._hl.files.File": "h5py.File",
     "h5py._hl.dataset.Dataset": "h5py.Dataset",
@@ -157,6 +157,7 @@ def res(
     "anndata._types.WriteCallback": "anndata.experimental.WriteCallback",
     "anndata._types.Read": "anndata.experimental.Read",
     "anndata._types.Write": "anndata.experimental.Write",
+    "anndata._types.StorageType": "anndata.experimental.StorageType",
     "anndata._types.Dataset2DIlocIndexer": "anndata.experimental.Dataset2DIlocIndexer",
     "zarr.core.array.Array": "zarr.Array",
     "zarr.core.group.Group": "zarr.Group",
@@ -168,19 +169,27 @@ def res(
     "anndata.compat.CupySparseMatrix": "cupyx.scipy.sparse.spmatrix",
     "anndata.compat.XDataArray": "xarray.DataArray",
     "anndata.compat.XDataset": "xarray.Dataset",
+    "anndata.compat.Index": "anndata.typing.Index",
     "awkward.highlevel.Array": "ak.Array",
     "numpy.int64": ("py:attr", "numpy.int64"),
     "numpy.dtypes.StringDType": ("py:attr", "numpy.dtypes.StringDType"),
     "pandas.DataFrame.iloc": ("py:attr", "pandas.DataFrame.iloc"),
     "pandas.DataFrame.loc": ("py:attr", "pandas.DataFrame.loc"),
+    "pandas.core.series.Series": "pandas.Series",
+    "pandas.core.arrays.categorical.Categorical": "pandas.Categorical",
+    "pandas.core.arrays.base.ExtensionArray": "pandas.api.extensions.ExtensionArray",
     "pandas.core.dtypes.dtypes.BaseMaskedDtype": "pandas.api.extensions.ExtensionDtype",
-    # should be fixed soon: https://github.com/tox-dev/sphinx-autodoc-typehints/pull/516
-    "types.EllipsisType": ("py:data", "types.EllipsisType"),
-    "pathlib._local.Path": "pathlib.Path",
 }
 autodoc_type_aliases = dict(
     NDArray=":data:`~numpy.typing.NDArray`",
     AxisStorable=":data:`~anndata.typing.AxisStorable`",
+    # The following are TypeVars in `anndata._types`, and aren’t actually exported,
+    # yet this bug causes them to create issues:
+    # - https://github.com/python/cpython/issues/124089
+    # - https://github.com/tox-dev/sphinx-autodoc-typehints/issues/580
+    K=":class:`zarr.Array` | :class:`h5py.Dataset`",
+    S=":class:`anndata.experimental.StorageType`",
+    RWAble=":class:`anndata.typing.RWAble`",
 )
 
 # -- Social cards ---------------------------------------------------------

docs/extensions/autosummary_skip_inherited.py

@@ -1,28 +1,58 @@
-"""Sphinx extension to not skip abstract methods."""
+"""Sphinx extension to not skip abstract/protocol methods."""
 
 from __future__ import annotations
 
+import abc
+import sys
+from traceback import walk_stack
 from typing import TYPE_CHECKING
 
-if TYPE_CHECKING:
-    from typing import Literal
+if sys.version_info >= (3, 13):
+    from typing import get_protocol_members, is_protocol
+else:
+    from typing_extensions import get_protocol_members, is_protocol
 
+if TYPE_CHECKING:
     from sphinx.application import Sphinx
     from sphinx.ext.autodoc import Options
+    from sphinx.ext.autodoc._property_types import _AutodocObjType
 
 
-def autodoc_skip_member(  # noqa: PLR0917
+def no_skip_abc_members(  # noqa: PLR0917
     app: Sphinx,
-    what: Literal["module", "class", "exception", "function", "method", "attribute"],
+    what: _AutodocObjType,
     name: str,
     obj: object,
     skip: bool,  # noqa: FBT001
     options: Options,
-):
-    if what == "method" and getattr(obj, "__isabstractmethod__", False):
+) -> bool | None:
+    if what in {"module", "class", "exception", "decorator"}:
+        return None  # speed up by not getting parent of non-class members
+
+    # Find parent class
+    for frame, _ in walk_stack(None):
+        if frame.f_code.co_name == "_get_members" and frame.f_code.co_filename.endswith(
+            "/generate.py"
+        ):
+            parent = frame.f_locals["obj"]
+            if not isinstance(parent, type):
+                return None
+            break
+    else:
+        return None
+
+    # Don’t skip abstract methods or properties
+    if issubclass(type(parent), abc.ABCMeta) and getattr(
+        obj, "__isabstractmethod__", False
+    ):
         return False
+
+    # If we’re documenting a protocol attribute, include it
+    if is_protocol(parent) and name in get_protocol_members(parent):
+        return False
+
     return None
 
 
-def setup(app: Sphinx):
-    app.connect("autodoc-skip-member", autodoc_skip_member)
+def setup(app: Sphinx) -> None:
+    app.connect("autodoc-skip-member", no_skip_abc_members)

docs/extensions/no_skip_abc_members.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Generic, get_origin
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+
+def skip_private_bases(
+    app: Sphinx, name: str, obj: type, _unused, bases: list[type]
+) -> None:
+    bases[:] = [
+        b
+        for b in bases
+        if b is not object
+        if get_origin(b) is not Generic
+        if not b.__name__.startswith("_")
+    ]
+
+
+def setup(app: Sphinx) -> None:
+    app.connect("autodoc-process-bases", skip_private_bases)

docs/extensions/skip_private_bases.py

@@ -59,19 +59,18 @@ dev = [
     { include-group = "dev-doc" },
 ]
 doc = [
-    "sphinx>=8.2.1,<9",                  # https://github.com/tox-dev/sphinx-autodoc-typehints/issues/586
+    "sphinx>=9.1.0",
     "sphinx-book-theme>=1.1.0",
-    "sphinx-autodoc-typehints>=2.2.0",
+    "sphinx-autodoc-typehints>=3.6.2",
     "sphinx-issues>=5.0.1",
     "sphinx-copybutton",
-    "sphinx-toolbox>=3.8.0",
     "sphinxext.opengraph",
     "myst-nb",
-    "scanpydoc[theme,typehints] >=0.16",
+    "scanpydoc[theme,typehints] >=0.17.1",
     "awkward>=2.6.3",
-    "IPython",                           # For syntax highlighting in notebooks
-    "myst_parser",
-    "sphinx_design>=0.5.0",
+    "IPython",                             # For syntax highlighting in notebooks
+    "myst-parser",
+    "sphinx-design",
     "anndata[dask]",
     # for unreleased changes
     { include-group = "dev-doc" },

pyproject.toml

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 import warnings
+from collections.abc import Hashable, Mapping
 from dataclasses import dataclass
 from functools import wraps
-from typing import TYPE_CHECKING, overload
+from typing import TYPE_CHECKING, Self, overload
 
 import numpy as np
 import pandas as pd
@@ -13,14 +14,7 @@
 from ..compat import XDataArray, XDataset, XVariable, pandas_as_str
 
 if TYPE_CHECKING:
-    from collections.abc import (
-        Callable,
-        Collection,
-        Hashable,
-        Iterable,
-        Iterator,
-        Mapping,
-    )
+    from collections.abc import Callable, Collection, Iterable, Iterator
     from typing import Any, Literal
 
     from .._types import Dataset2DIlocIndexer
@@ -39,10 +33,8 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
     return wrapper
 
 
-class Dataset2D:
+class Dataset2D(Mapping[Hashable, XDataArray | Self]):
     r"""
-    Bases :class:`~collections.abc.Mapping`\ [:class:`~collections.abc.Hashable`, :class:`~xarray.DataArray` | :class:`~anndata.experimental.backed.Dataset2D`\ ]
-
     A wrapper class meant to enable working with lazy dataframe data according to
     :class:`~anndata.AnnData`'s internal API.  This class ensures that "dataframe-invariants"
     are respected, namely that there is only one 1d dim and coord with the same name i.e.,