Merge pull request #211 from scverse/2.0.8

2.1.0 changes

Author: PauBadiaM

CHANGELOG.md

@@ -8,7 +8,60 @@ and this project adheres to [Semantic Versioning][].
 [keep a changelog]: https://keepachangelog.com/en/1.0.0/
 [semantic versioning]: https://semver.org/spec/v2.0.0.html
 
-## 2.0.8
+## 2.1.0
+
+### Changes
+
+- Updated logo
+- `decoupler._download._download` now returns bytes instead of a dataframe. To transform to `pandas.DataFrame` use `decoupler._download._bytes_to_pandas`
+- Enrichment methods and pseudobulking now work with backed AnnData objects, useful when working with big datasets and memory is limited
+
+### Bugfixes
+- Fixed error in `pl.obsm` where default value of `cmap_obs` was not properly set.
+
+## 2.0.7
+
+### Changes
+- Added `pre-commit` functionality to the repository
+
+## 2.0.6
+
+### Changes
+- Modified links and paths to follow scverse's repository
+
+## 2.0.5
+
+### Bugfixes
+- Fixed error message when extra dependencies where not installed
+
+## 2.0.4
+
+### Bugfixes
+- Fixed `dcor` import bug as an external dependency
+
+## 2.0.3
+
+### Bugfixes
+- Fixed error in `pp.pseudobulk` when obs columns were not categorical
+
+## 2.0.2
+
+### Changes
+- Allowed ordering functions (`pp.bin_order`, `pl.order` and `pl.order_targets`) not to be bound between 0 and 1
+- Added ipywidgets as dependency
+- Silenced xgboost warnings
+
+### Bugfixes
+- Handled corner case in `bm.metric.auc` when scores are all 0
+- Fixed error in `bm.metric.hmean` when metrics were str instead of list
+- Fixed error when `obs` column is a list in `pp.pseudobulk`
+
+## 2.0.1
+
+### Bugfixes
+- Fixed an error in `pp.pseudobulk` when handling empty samples or features
+
+## 2.0.0
 
 Major update to accomodate the scverse template {cite}`scverse`.
 

README.md

@@ -1,5 +1,5 @@
 # decoupler - Ensemble of methods to infer enrichment scores
-<img src="https://github.com/saezlab/decoupleR/blob/master/inst/figures/logo.svg?raw=1" align="right" width="120" class="no-scaled-link" />
+<img src="https://raw.githubusercontent.com/scverse/decoupler/refs/heads/2.0.8/docs/_static/images/logo.svg" align="right" width="120" class="no-scaled-link" />
 
 
 [![Tests][badge-tests]][tests]

docs/_static/images/logo.png

@@ -4,7 +4,7 @@ requires = [ "hatchling" ]
 
 [project]
 name = "decoupler"
-version = "2.0.8"
+version = "2.1.0"
 description = "Python package to perform enrichment analysis from omics data."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -66,6 +66,7 @@ optional-dependencies.test = [
   "gseapy",
   "igraph",
   "ipywidgets",
+  "memory-profiler",
   "pytest",
   "scanpy",
   "statsmodels",

docs/_static/images/logo.svg

@@ -13,8 +13,7 @@
 def _download(
     url: str,
     verbose: bool = False,
-    **kwargs,
-) -> pd.DataFrame:
+) -> bytes:
     assert isinstance(url, str), "url must be str"
     # Download with progress bar
     m = f"Downloading {url}"
@@ -27,9 +26,13 @@ def _download(
                 if chunk:
                     chunks.append(chunk)
                     pbar.update(len(chunk))
-    # Read into pandas
+    # Read into bytes
     data = io.BytesIO(b"".join(chunks))
-    df = pd.read_csv(data, **kwargs)
     m = "Download finished"
     _log(m, level="info", verbose=verbose)
+    return data
+
+
+def _bytes_to_pandas(data: bytes, **kwargs) -> pd.DataFrame:
+    df = pd.read_csv(data, **kwargs)
     return df

pyproject.toml

@@ -1,7 +1,7 @@
 from anndata import AnnData
 
 from decoupler._docs import docs
-from decoupler._download import _download
+from decoupler._download import _bytes_to_pandas, _download
 from decoupler._log import _log
 
 
@@ -38,7 +38,8 @@ def hsctgfb(
         "https://www.ncbi.nlm.nih.gov/geo/download/?acc=GSE151251&format=file&"
         "file=GSE151251%5FHSCs%5FCtrl%2Evs%2EHSCs%5FTGFb%2Ecounts%2Etsv%2Egz"
     )
-    adata = _download(url, compression="gzip", sep="\t", verbose=verbose)
+    adata = _download(url, verbose=verbose)
+    adata = _bytes_to_pandas(adata, compression="gzip", sep="\t")
     # Transform to AnnData
     adata = adata.drop_duplicates("GeneName").set_index("GeneName").iloc[:, 5:].T
     adata.columns.name = None
@@ -92,9 +93,11 @@ def knocktf(
     assert isinstance(thr_fc, int | float) or thr_fc is None, "thr_fc must be numeric or None"
     # Download
     url = "https://zenodo.org/record/7035528/files/knockTF_expr.csv?download=1"
-    adata = _download(url, sep=",", index_col=0, verbose=verbose)
+    adata = _download(url, verbose=verbose)
+    adata = _bytes_to_pandas(adata, sep=",", index_col=0)
     url = "https://zenodo.org/record/7035528/files/knockTF_meta.csv?download=1"
-    obs = _download(url, sep=",", index_col=0, verbose=verbose)
+    obs = _download(url, verbose=verbose)
+    obs = _bytes_to_pandas(obs, sep=",", index_col=0)
     obs = obs.rename(columns={"TF": "source"}).assign(type_p=-1)
     # Make anndata
     adata = AnnData(X=adata, obs=obs)

src/decoupler/_download.py

@@ -55,21 +55,33 @@ def _run(
 ) -> tuple[pd.DataFrame, pd.DataFrame] | AnnData | None:
     _log(f"{name} - Running {name}", level="info", verbose=verbose)
     # Process data
-    mat, obs, var = extract(data, layer=layer, raw=raw, empty=empty, verbose=verbose)
-    sparse = sps.issparse(mat)
+    mat, obs, var = extract(data, layer=layer, raw=raw, empty=empty, verbose=verbose, bsize=bsize)
+    issparse = sps.issparse(mat)
+    isbacked = isinstance(mat, tuple)
     # Process net
     net = prune(features=var, net=net, tmin=tmin, verbose=verbose)
     # Handle stat type
     if adj:
         sources, targets, adjm = adjmat(features=var, net=net, verbose=verbose)
-        # Handle sparse
-        if sparse:
+        # Handle batches
+        if issparse or isbacked:
             nbatch = int(np.ceil(obs.size / bsize))
             es, pv = [], []
             for i in tqdm(range(nbatch), disable=not verbose):
+                if i == 0 and verbose:
+                    batch_verbose = True
+                else:
+                    batch_verbose = False
                 srt, end = i * bsize, i * bsize + bsize
-                bmat = mat[srt:end].toarray()
-                bes, bpv = func(bmat, adjm, verbose=verbose, **kwargs)
+                if sps.issparse(mat):
+                    bmat = mat[srt:end].toarray()
+                else:
+                    bmat, msk_col = mat
+                    bmat = bmat[srt:end, :]
+                    if sps.issparse(bmat):
+                        bmat = bmat.toarray()
+                    bmat = bmat[:, msk_col]
+                bes, bpv = func(bmat, adjm, verbose=batch_verbose, **kwargs)
                 es.append(bes)
                 pv.append(bpv)
             es = np.vstack(es)
@@ -79,7 +91,23 @@ def _run(
             es = pd.DataFrame(es, index=obs, columns=sources)
     else:
         sources, cnct, starts, offsets = idxmat(features=var, net=net, verbose=verbose)
-        es, pv = func(mat, cnct, starts, offsets, verbose=verbose, **kwargs)
+        if isbacked:
+            nbatch = int(np.ceil(obs.size / bsize))
+            es, pv = [], []
+            for i in tqdm(range(nbatch), disable=not verbose):
+                if i == 0 and verbose:
+                    batch_verbose = True
+                else:
+                    batch_verbose = False
+                srt, end = i * bsize, i * bsize + bsize
+                bmat, msk_col = mat
+                bmat = bmat[srt:end, msk_col]
+                bes, bpv = func(bmat, cnct, starts, offsets, verbose=batch_verbose, **kwargs)
+                es.append(bes)
+                pv.append(bpv)
+            es = np.vstack(es)
+        else:
+            es, pv = func(mat, cnct, starts, offsets, verbose=verbose, **kwargs)
         es = pd.DataFrame(es, index=obs, columns=sources)
     # Handle pvals and FDR correction
     if test:

src/decoupler/ds/_bulk.py

@@ -1,7 +1,7 @@
 import pandas as pd
 
 from decoupler._docs import docs
-from decoupler._download import _download
+from decoupler._download import _bytes_to_pandas, _download
 from decoupler.op._dtype import _infer_dtypes
 from decoupler.op._translate import translate
 
@@ -45,6 +45,7 @@ def collectri(
     """
     url = "https://zenodo.org/records/8192729/files/CollecTRI_regulons.csv?download=1"
     ct = _download(url, verbose=verbose)
+    ct = _bytes_to_pandas(ct)
     # Update resources
     resources = []
     for str_res in ct["resources"]:

src/decoupler/mt/_run.py

@@ -1,7 +1,7 @@
 import pandas as pd
 
 from decoupler._docs import docs
-from decoupler._download import URL_INT, _download
+from decoupler._download import URL_INT, _bytes_to_pandas, _download
 from decoupler._log import _log
 from decoupler.op._dtype import _infer_dtypes
 from decoupler.op._translate import translate
@@ -71,7 +71,8 @@ def dorothea(
     url = URL_INT + url_ext
     m = f"dorothea - Accessing DoRothEA (levels {str_levels}) with {license} license and weights={weights}"
     _log(m, level="info", verbose=verbose)
-    do = _download(url, sep="\t", verbose=verbose)
+    do = _download(url, verbose=verbose)
+    do = _bytes_to_pandas(do, sep="\t")
     # Filter extra columns
     do = do[
         [