(fix): force dask cluster tests to be run in same process (#1932)

* (fix): group `dask` tests together on same process

* (fix): use `pytest-xdist` by default

Author: ilan-gold

pyproject.toml

@@ -61,7 +61,7 @@ Home-page = "https://github.com/scverse/anndata"
 dev = [
     # runtime dev version generation
     "hatch-vcs",
-    "anndata[dev-doc,dev-test]",
+    "anndata[dev-doc,test]",
 ]
 doc = [
     "sphinx>=8.2.1",
@@ -91,20 +91,20 @@ test = [
     "pytest-randomly",
     "pytest-memray",
     "pytest-mock",
+    "pytest-xdist[psutil]",
     "filelock",
     "matplotlib",
     "scikit-learn",
     "openpyxl",
     "joblib",
     "boltons",
     "scanpy>=1.10",
-    "httpx",              # For data downloading
+    "httpx",                # For data downloading
     "dask[distributed]",
     "awkward>=2.3",
     "pyarrow",
     "anndata[dask]",
 ]
-dev-test = [ "pytest-xdist[psutil]" ] # local test speedups
 gpu = [ "cupy" ]
 cu12 = [ "cupy-cuda12x" ]
 cu11 = [ "cupy-cuda11x" ]
@@ -144,6 +144,7 @@ addopts = [
     "--doctest-modules",
     "--pyargs",
     "-ptesting.anndata._pytest",
+    "--dist=loadgroup",
 ]
 filterwarnings = [
     "ignore::anndata._warnings.OldFormatWarning",

tests/conftest.py

@@ -13,6 +13,7 @@
     from dask.base import normalize_seq
 else:
     from dask.tokenize import normalize_seq
+
 from filelock import FileLock
 from scipy import sparse
 

tests/lazy/test_concat.py

@@ -214,6 +214,7 @@ def test_concat_to_memory_var(
         stores_for_concat[store_idx].reset_key_trackers()
 
 
+@pytest.mark.xdist_group("dask")
 def test_concat_data_with_cluster_to_memory(
     adata_remote: AnnData, join: Join_T, local_cluster_addr: str
 ) -> None:

tests/test_dask.py

@@ -110,6 +110,7 @@ def test_dask_write(adata, tmp_path, diskfmt):
     assert isinstance(orig.varm["a"], DaskArray)
 
 
+@pytest.mark.xdist_group("dask")
 def test_dask_distributed_write(
     adata: AnnData,
     tmp_path: Path,

tests/test_io_elementwise.py

@@ -336,6 +336,7 @@ def test_read_lazy_subsets_nd_dask(store, n_dims, chunks):
         assert_equal(X_from_disk[index], X_dask_from_disk[index])
 
 
+@pytest.mark.xdist_group("dask")
 def test_read_lazy_h5_cluster(
     sparse_format: Literal["csr", "csc"], tmp_path: Path, local_cluster_addr: str
 ) -> None: