feat: add Harmony to scanpy (#3953)

Co-authored-by: Philipp A. <flying-sheep@web.de>
Co-authored-by: Copilot <copilot@github.com>
Co-authored-by: Ilay Kavitzky <ilay.kavitzky@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Cuiwei Gao <48gaocuiwei@gmail.com>
Co-authored-by: Jhonatan Felix <108437587+JhonatanFelix@users.noreply.github.com>

Author: 7 people

docs/api/preprocessing.md

@@ -47,18 +47,24 @@ For visual quality control, see {func}`~scanpy.pl.highest_expr_genes` and
    pp.recipe_seurat
 ```
 
-## Batch effect correction
+(pp-data-integration)=
 
-Also see {ref}`data-integration`. Note that a simple batch correction method is available via {func}`pp.regress_out`. Checkout {mod}`scanpy.external` for more.
+## Data integration
+
+Batch effect correction and other data integration.
+Note that a simple batch correction method is available via {func}`pp.regress_out`.
 
 ```{eval-rst}
 .. autosummary::
    :nosignatures:
    :toctree: generated/
 
    pp.combat
+   pp.harmony_integrate
 ```
 
+Also see {ref}`data integration tools <data-integration>` and external {ref}`external data integration <external-data-integration>`.
+
 ## Doublet detection
 
 ```{eval-rst}

docs/conf.py

@@ -176,6 +176,7 @@
     "pp.downsample_counts": (["np", "sp[csr]"], []),
     "pp.filter_cells": (["np", "sp", "da"], []),
     "pp.filter_genes": (["np", "sp", "da"], []),
+    "pp.harmony_integrate": (["np"], []),
     "pp.highly_variable_genes": (["np", "sp", "da"], ["da[sp[csc]]"]),
     "pp.log1p": (["np", "sp", "da"], []),
     "pp.neighbors": (["np", "sp"], []),

docs/external/preprocessing.md

@@ -5,6 +5,11 @@
 .. currentmodule:: scanpy.external
 ```
 
+Previously found here, but now part of scanpy’s main API:
+- {func}`scanpy.pp.harmony_integrate`
+- {func}`scanpy.pp.scrublet`
+- {func}`scanpy.pp.scrublet_simulate_doublets`
+
 (external-data-integration)=
 
 ## Data integration
@@ -14,10 +19,8 @@
    :toctree: ../generated/
 
    pp.bbknn
-   pp.harmony_integrate
    pp.mnn_correct
    pp.scanorama_integrate
-
 ```
 
 ## Sample demultiplexing
@@ -38,5 +41,4 @@ Note that the fundamental limitations of imputation are still under [debate](htt
    :toctree: ../generated/
 
    pp.magic
-
 ```

docs/references.bib

@@ -657,6 +657,16 @@ @article{Ntranos2019
   pages     = {163--166},
 }
 
+@article{Patikas2026,
+  author    = {Patikas, Nikolaos and Yao, Hongcheng and Madhu, Roopa and Raychaudhuri, Soumya and Hemberg, Martin and Korsunsky, Ilya},
+  title     = {Integration of large, complex single-cell datasets with Harmony2},
+  url       = {https://doi.org/10.1101/2026.03.16.711825},
+  doi       = {10.1101/2026.03.16.711825},
+  journal   = {bioRxiv},
+  publisher = {Cold Spring Harbor Laboratory},
+  year      = {2026},
+}
+
 @article{Paul2015,
   author    = {Paul, Franziska and Arkin, Ya’ara and Giladi, Amir and Jaitin, Diego Adhemar and Kenigsberg, Ephraim and Keren-Shaul, Hadas and Winter, Deborah and Lara-Astiaso, David and Gury, Meital and Weiner, Assaf and David, Eyal and Cohen, Nadav and Lauridsen, Felicia Kathrine Bratt and Haas, Simon and Schlitzer, Andreas and Mildner, Alexander and Ginhoux, Florent and Jung, Steffen and Trumpp, Andreas and Porse, Bo Torben and Tanay, Amos and Amit, Ido},
   title     = {Transcriptional Heterogeneity and Lineage Commitment in Myeloid Progenitors},

docs/release-notes/1.10.0.md

@@ -25,7 +25,7 @@ Some highlights:
 * {func}`scanpy.datasets.blobs` now accepts a `random_state` argument {pr}`2683` {smaller}`E Roellin`
 * {func}`scanpy.pp.pca` and {func}`scanpy.pp.regress_out` now accept a layer argument {pr}`2588` {smaller}`S Dicks`
 * {func}`scanpy.pp.subsample` with `copy=True` can now be called in backed mode {pr}`2624` {smaller}`E Roellin`
-* {func}`scanpy.external.pp.harmony_integrate` now runs with 64 bit floats improving reproducibility {pr}`2655` {smaller}`S Dicks`
+* {func}`scanpy.pp.harmony_integrate` now runs with 64 bit floats improving reproducibility {pr}`2655` {smaller}`S Dicks`
 * {func}`scanpy.tl.rank_genes_groups` no longer warns that it's default was changed from t-test_overestim_var to t-test {pr}`2798` {smaller}`L Heumos`
 * `scanpy.pp.calculate_qc_metrics` now allows `qc_vars` to be passed as a string {pr}`2859` {smaller}`N Teyssier`
 * {func}`scanpy.tl.leiden` and {func}`scanpy.tl.louvain` now store clustering parameters in the key provided by the `key_added` parameter instead of always writing to (or overwriting) a default key {pr}`2864` {smaller}`J Fan`

docs/release-notes/1.11.0.md

@@ -30,7 +30,7 @@ Release candidates:
 
 #### Documentation
 
-- {guilabel}`rc1` Improve {func}`~scanpy.external.pp.harmony_integrate` docs {smaller}`D Kühl` ({pr}`3362`)
+- {guilabel}`rc1` Improve {func}`~scanpy.pp.harmony_integrate` docs {smaller}`D Kühl` ({pr}`3362`)
 - {guilabel}`rc1` Raise {exc}`FutureWarning` when calling deprecated {mod}`scanpy.pp` functions {smaller}`P Angerer` ({pr}`3380`)
 - {guilabel}`rc1` {smaller}`P Angerer` ({pr}`3407`)
 

docs/release-notes/3953.feat.md

@@ -0,0 +1 @@
+Add {func}`scanpy.pp.harmony_integrate` with Harmony1 and Harmony2 support for batch correction {smaller}`S Dicks, P Angerer`

pyproject.toml

@@ -88,7 +88,6 @@ bbknn = [ "bbknn" ]
 dask = [ "anndata[dask]", "dask[array]>=2024.5.1" ]
 # PCA acceleration
 dask-ml = [ "dask-ml", "scanpy[dask]" ]
-harmony = [ "harmonypy" ]
 leiden = [ "igraph>=0.10.8", "leidenalg>=0.10.1" ]
 louvain = [ "igraph", "louvain>=0.8.2", "setuptools" ]
 magic = [ "magic-impute>=2.0.4" ]
@@ -137,6 +136,7 @@ doc = [
 ]
 test-min = [
     "dependency-groups",    # for CI scripts doctests
+    "pooch",
     "pytest",
     "pytest-cov",           # only for use from VS Code
     "pytest-mock",

src/scanpy/external/pp/__init__.py

@@ -3,24 +3,37 @@
 from __future__ import annotations
 
 from ..._compat import deprecated
-from ...preprocessing import _scrublet
 from ._bbknn import bbknn
-from ._harmony_integrate import harmony_integrate
 from ._hashsolo import hashsolo
 from ._magic import magic
 from ._mnn_correct import mnn_correct
 from ._scanorama_integrate import scanorama_integrate
 
-scrublet = deprecated("Import from sc.pp instead")(_scrublet.scrublet)
-scrublet_simulate_doublets = deprecated("Import from sc.pp instead")(
-    _scrublet.scrublet_simulate_doublets
-)
-
 __all__ = [
     "bbknn",
-    "harmony_integrate",
     "hashsolo",
     "magic",
     "mnn_correct",
     "scanorama_integrate",
 ]
+
+
+@deprecated("Import from sc.pp instead")
+def harmony_integrate(*args, **kwargs):
+    from ...preprocessing import harmony_integrate
+
+    return harmony_integrate(*args, **kwargs)
+
+
+@deprecated("Import from sc.pp instead")
+def scrublet(*args, **kwargs):
+    from ...preprocessing import scrublet
+
+    return scrublet(*args, **kwargs)
+
+
+@deprecated("Import from sc.pp instead")
+def scrublet_simulate_doublets(*args, **kwargs):
+    from ...preprocessing import scrublet_simulate_doublets
+
+    return scrublet_simulate_doublets(*args, **kwargs)