Backport PR #4106 on branch 1.12.x (test: adapt to x/layers unification) (#4107)

Author: flying-sheep

src/scanpy/_utils/_doctests.py

@@ -13,14 +13,20 @@ def decorator(func: F) -> F:
     return decorator
 
 
-def doctest_skip[F: Callable](reason: str) -> Callable[[F], F]:
-    """Mark function so doctest is skipped."""
+def doctest_skipif[F: Callable](
+    condition: bool = True,  # noqa: FBT001, FBT002
+    /,
+    *,
+    reason: str,
+) -> Callable[[F], F]:
+    """Mark function so doctest is skipped (if `condition` is met)."""
     if not reason:
         msg = "reason must not be empty"
         raise ValueError(msg)
 
     def decorator(func: F) -> F:
-        func._doctest_skip_reason = reason
+        if condition:
+            func._doctest_skip_reason = reason
         return func
 
     return decorator

src/scanpy/datasets/_datasets.py

@@ -7,11 +7,12 @@
 import numpy as np
 import pandas as pd
 from anndata import AnnData, OldFormatWarning
+from packaging.version import Version
 
 from .. import _utils
-from .._compat import deprecated, old_positionals
+from .._compat import deprecated, old_positionals, pkg_version
 from .._settings import settings
-from .._utils._doctests import doctest_internet, doctest_needs
+from .._utils._doctests import doctest_internet, doctest_needs, doctest_skipif
 from ..readwrite import read, read_h5ad, read_visium
 from ._utils import check_datasetdir_exists
 
@@ -54,6 +55,12 @@
 HERE = Path(__file__).parent
 
 
+_doctest_skipif_old_anndata = doctest_skipif(
+    pkg_version("anndata") < Version("0.13.dev0"), reason="old anndata"
+)
+
+
+@_doctest_skipif_old_anndata
 @old_positionals(
     "n_variables", "n_centers", "cluster_std", "n_observations", "random_state"
 )
@@ -92,6 +99,7 @@ def blobs(
     >>> sc.datasets.blobs()
     AnnData object with n_obs × n_vars = 640 × 11
         obs: 'blobs'
+        layers: None
 
     """
     import sklearn.datasets
@@ -106,6 +114,7 @@ def blobs(
     return AnnData(x, obs=dict(blobs=y.astype(str)))
 
 
+@_doctest_skipif_old_anndata
 @doctest_internet
 @check_datasetdir_exists
 def burczynski06() -> AnnData:
@@ -127,13 +136,15 @@ def burczynski06() -> AnnData:
         ...
     AnnData object with n_obs × n_vars = 127 × 22283
         obs: 'groups'
+        layers: None
 
     """
     filename = settings.datasetdir / "burczynski06/GDS1615_full.soft.gz"
     url = "https://exampledata.scverse.org/scanpy/GDS1615_full.soft.gz"
     return read(filename, backup_url=url)
 
 
+@_doctest_skipif_old_anndata
 def krumsiek11() -> AnnData:
     r"""Simulated myeloid progenitors :cite:p:`Krumsiek2011`.
 
@@ -159,6 +170,7 @@ def krumsiek11() -> AnnData:
     AnnData object with n_obs × n_vars = 640 × 11
         obs: 'cell_type'
         uns: 'iroot', 'highlights'
+        layers: None
 
     """  # noqa: D401
     with settings.verbosity.override("error"):  # suppress output...
@@ -176,6 +188,7 @@ def krumsiek11() -> AnnData:
     return adata
 
 
+@_doctest_skipif_old_anndata
 @doctest_internet
 @doctest_needs("openpyxl")
 @check_datasetdir_exists
@@ -203,6 +216,7 @@ def moignard15() -> AnnData:
     AnnData object with n_obs × n_vars = 3934 × 42
         obs: 'exp_groups'
         uns: 'iroot', 'exp_groups_colors'
+        layers: None
 
     """
     filename = settings.datasetdir / "moignard15/nbt.3154-S3.xlsx"
@@ -236,6 +250,7 @@ def moignard15() -> AnnData:
     return adata
 
 
+@_doctest_skipif_old_anndata
 @doctest_internet
 @check_datasetdir_exists
 def paul15() -> AnnData:
@@ -257,6 +272,7 @@ def paul15() -> AnnData:
     AnnData object with n_obs × n_vars = 2730 × 3451
         obs: 'paul15_clusters'
         uns: 'iroot'
+        layers: None
 
     """
     import h5py
@@ -296,6 +312,7 @@ def paul15() -> AnnData:
     return adata
 
 
+@_doctest_skipif_old_anndata
 def toggleswitch() -> AnnData:
     """Simulated toggleswitch.
 
@@ -315,6 +332,7 @@ def toggleswitch() -> AnnData:
         ...
     AnnData object with n_obs × n_vars = 200 × 2
         uns: 'iroot'
+        layers: None
 
     """  # noqa: D401
     filename = HERE / "toggleswitch.txt"
@@ -323,6 +341,7 @@ def toggleswitch() -> AnnData:
     return adata
 
 
+@_doctest_skipif_old_anndata
 def pbmc68k_reduced() -> AnnData:
     r"""Subsampled and processed 68k PBMCs.
 
@@ -353,11 +372,13 @@ def pbmc68k_reduced() -> AnnData:
         obsm: 'X_pca', 'X_umap'
         varm: 'PCs'
         obsp: 'connectivities', 'distances'
+        layers: None
 
     """
     return read_h5ad(HERE / "10x_pbmc68k_reduced.h5ad")
 
 
+@_doctest_skipif_old_anndata
 @doctest_internet
 @check_datasetdir_exists
 def pbmc3k() -> AnnData:
@@ -402,6 +423,7 @@ def pbmc3k() -> AnnData:
     >>> sc.datasets.pbmc3k()
     AnnData object with n_obs × n_vars = 2700 × 32738
         var: 'gene_ids'
+        layers: None
 
     """
     url = "https://exampledata.scverse.org/scanpy/pbmc3k_raw.h5ad"
@@ -411,6 +433,7 @@ def pbmc3k() -> AnnData:
     return adata
 
 
+@_doctest_skipif_old_anndata
 @doctest_internet
 @check_datasetdir_exists
 def pbmc3k_processed() -> AnnData:
@@ -442,6 +465,7 @@ def pbmc3k_processed() -> AnnData:
         obsm: 'X_pca', 'X_tsne', 'X_umap', 'X_draw_graph_fr'
         varm: 'PCs'
         obsp: 'distances', 'connectivities'
+        layers: None
 
     """  # noqa: D401
     url = "https://exampledata.scverse.org/scanpy/pbmc3k.h5ad"
@@ -509,6 +533,7 @@ def _download_visium_dataset(
 
 
 @deprecated("Use `squidpy.datasets.visium` instead.")
+@_doctest_skipif_old_anndata
 @doctest_internet
 @check_datasetdir_exists
 def visium_sge(
@@ -550,6 +575,7 @@ def visium_sge(
         var: 'gene_ids', 'feature_types', 'genome'
         uns: 'spatial'
         obsm: 'spatial'
+        layers: None
 
     """  # noqa: D401
     spaceranger_version = "1.1.0" if "V1_" in sample_id else "1.2.0"

src/scanpy/datasets/_ebi_expression_atlas.py

@@ -14,6 +14,7 @@
 from .._settings import settings
 from .._utils._doctests import doctest_internet
 from ..readwrite import _download
+from ._datasets import _doctest_skipif_old_anndata
 from ._utils import check_datasetdir_exists
 
 if TYPE_CHECKING:
@@ -107,6 +108,7 @@ def read_expression_from_archive(archive: ZipFile) -> anndata.AnnData:
     return adata
 
 
+@_doctest_skipif_old_anndata
 @doctest_internet
 def ebi_expression_atlas(
     accession: str, *, filter_boring: bool = False
@@ -140,6 +142,7 @@ def ebi_expression_atlas(
     >>> sc.datasets.ebi_expression_atlas("E-MTAB-4888")  # doctest: +ELLIPSIS
     AnnData object with n_obs × n_vars = 2261 × 23899
         obs: 'Sample Characteristic[organism]', 'Sample Characteristic Ontology Term[organism]', ..., 'Factor Value[cell type]', 'Factor Value Ontology Term[cell type]'
+        layers: None
 
     """
     experiment_dir = settings.datasetdir / accession

src/scanpy/external/pp/_hashsolo.py

@@ -39,7 +39,7 @@
 
 from ..._compat import old_positionals
 from ..._utils import check_nonnegative_integers
-from ..._utils._doctests import doctest_skip
+from ..._utils._doctests import doctest_skipif
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -295,7 +295,7 @@ def _calculate_bayes_rule(
 @old_positionals(
     "priors", "pre_existing_clusters", "number_of_noise_barcodes", "inplace"
 )
-@doctest_skip("Illustrative but not runnable doctest code")
+@doctest_skipif(reason="Illustrative but not runnable doctest code")
 def hashsolo(
     adata: AnnData,
     cell_hashing_columns: Sequence[str],

src/scanpy/tools/_ingest.py

@@ -11,7 +11,7 @@
 from .._compat import CSBase, old_positionals
 from .._settings import settings
 from .._utils import NeighborsView, raise_not_implemented_error_if_backed_type
-from .._utils._doctests import doctest_skip
+from .._utils._doctests import doctest_skipif
 from ..neighbors import FlatTree
 
 if TYPE_CHECKING:
@@ -30,7 +30,7 @@
     "neighbors_key",
     "inplace",
 )
-@doctest_skip("illustrative short example but not runnable")
+@doctest_skipif(reason="illustrative short example but not runnable")
 def ingest(
     adata: AnnData,
     adata_ref: AnnData,

src/testing/scanpy/_pytest/__init__.py

@@ -17,6 +17,14 @@
     from collections.abc import Generator, Iterable, Mapping
 
 
+MARK_RETRY_DOWNLOAD = pytest.mark.flaky(
+    reruns=5,
+    reruns_delay=2,
+    # The list of matches here probably needs to be expanded
+    only_rerun=[r"ConnectionError", r"HTTPError"],
+)
+
+
 _original_settings: Mapping[str, object] | None = None
 
 
@@ -107,7 +115,7 @@ def pytest_collection_modifyitems(
         # `--run-internet` passed
         if "internet" in item.keywords:
             item.add_marker(skipif_not_run_internet)
-            item.add_marker(pytest.mark.flaky(reruns=5, reruns_delay=2))
+            item.add_marker(MARK_RETRY_DOWNLOAD)
 
 
 def _modify_doctests(request: pytest.FixtureRequest) -> None:
@@ -128,7 +136,7 @@ def _modify_doctests(request: pytest.FixtureRequest) -> None:
     if getattr(func, "_doctest_internet", False):
         if not request.config.getoption("--internet-tests"):
             pytest.skip(reason="need --internet-tests option to run")
-        request.applymarker(pytest.mark.flaky(reruns=5, reruns_delay=2))
+        request.applymarker(MARK_RETRY_DOWNLOAD)
 
 
 assert "scanpy" not in sys.modules, (

tests/test_datasets.py

@@ -12,8 +12,10 @@
 import numpy as np
 import pytest
 from anndata.tests.helpers import assert_adata_equal
+from packaging.version import Version
 
 import scanpy as sc
+from scanpy._compat import pkg_version
 from testing.scanpy._helpers import data
 from testing.scanpy._pytest.marks import needs
 
@@ -173,6 +175,7 @@ def test_download_failure() -> None:
 DS_MARKS = defaultdict(list, moignard15=[needs.openpyxl])
 
 
+@pytest.mark.skipif(pkg_version("anndata") < Version("0.13.dev0"), reason="old anndata")
 @pytest.mark.parametrize(
     "ds_name",
     [