feat: add layer param to filter_rank_genes_groups (#3999)

Author: flying-sheep

docs/release-notes/3999.feat.md

@@ -0,0 +1 @@
+Add `layer` parameter to {func}`~scanpy.tl.filter_rank_genes_groups` {smaller}`P Angerer`

src/scanpy/tools/_rank_genes_groups.py

@@ -18,7 +18,7 @@
     get_literal_vals,
     raise_not_implemented_error_if_backed_type,
 )
-from ..get import _check_mask
+from ..get import _check_mask, _get_obs_rep
 
 if TYPE_CHECKING:
     from collections.abc import Generator, Iterable
@@ -763,6 +763,7 @@ def filter_rank_genes_groups(  # noqa: PLR0912
     *,
     key: str | None = None,
     groupby: str | None = None,
+    layer: str | None = None,
     use_raw: bool | None = None,
     key_added: str = "rank_genes_groups_filtered",
     min_in_group_fraction: float = 0.25,
@@ -789,6 +790,7 @@ def filter_rank_genes_groups(  # noqa: PLR0912
     adata
     key
     groupby
+    layer
     use_raw
     key_added
     min_in_group_fraction
@@ -799,8 +801,7 @@ def filter_rank_genes_groups(  # noqa: PLR0912
 
     Returns
     -------
-    Same output as :func:`scanpy.tl.rank_genes_groups` but with filtered genes names set to
-    `nan`
+    Same output as :func:`scanpy.tl.rank_genes_groups` but with filtered genes names set to `nan`.
 
     Examples
     --------
@@ -821,7 +822,9 @@ def filter_rank_genes_groups(  # noqa: PLR0912
         groupby = adata.uns[key]["params"]["groupby"]
 
     if use_raw is None:
-        use_raw = adata.uns[key]["params"]["use_raw"]
+        use_raw = adata.uns[key]["params"]["use_raw"] if layer is None else False
+
+    x = _get_obs_rep(adata, use_raw=use_raw, layer=layer)
 
     same_params = (
         adata.uns[key]["params"]["groupby"] == groupby
@@ -872,7 +875,8 @@ def filter_rank_genes_groups(  # noqa: PLR0912
         var_names = gene_names[cluster].values
 
         if not use_logfolds or not use_fraction:
-            sub_x = adata.raw[:, var_names].X if use_raw else adata[:, var_names].X
+            var_idx = (adata.raw if use_raw else adata).var_names.get_indexer(var_names)
+            sub_x = x[:, var_idx]
             in_group = (adata.obs[groupby] == cluster).to_numpy()
             x_in = sub_x[in_group]
             x_out = sub_x[~in_group]

tests/test_filter_rank_genes_groups.py

@@ -62,21 +62,30 @@
         pytest.param("rest", True, True, id="rest-pts-abs"),
     ],
 )
-def test_filter_rank_genes_groups(reference, pts, abs):
+@pytest.mark.parametrize("layer", [None, "layer"], ids=["raw", "layer"])
+def test_filter_rank_genes_groups(
+    *, reference: str, pts: bool, abs: bool, layer: str | None
+) -> None:
     adata = pbmc68k_reduced()
+    if layer is not None:
+        adata.layers[layer] = adata.raw.X
+        del adata.X
+        del adata.raw
 
     rank_genes_groups(
         adata,
         "bulk_labels",
         reference=reference,
         pts=pts,
         method="wilcoxon",
+        layer=layer,
         rankby_abs=abs,
         n_genes=5,
     )
     if abs:
         filter_rank_genes_groups(
             adata,
+            layer=layer,
             compare_abs=True,
             min_in_group_fraction=-1,
             max_out_group_fraction=1,
@@ -85,6 +94,7 @@ def test_filter_rank_genes_groups(reference, pts, abs):
     else:
         filter_rank_genes_groups(
             adata,
+            layer=layer,
             min_in_group_fraction=0.25,
             min_fold_change=1,
             max_out_group_fraction=0.5,