Added attempts to zenodo download

Author: PauBadiaM

CHANGELOG.md

@@ -11,8 +11,9 @@ and this project adheres to [Semantic Versioning][].
 ## 2.1.3
 
 ### Changes
-- `pp.adjmat` now returns the same features as used as input instead of the subset of `net`.
-- `pp.pseudobulk` now returns the same order features as used as input instead of shuffling them.
+- `pp.adjmat` now returns the same features as used as input instead of the subset of `net`
+- `pp.pseudobulk` now returns the same order features as used as input instead of shuffling them
+- Added 5 attemps to `_download` to mitigate 429 Client Error from Zenodo downloads
 
 ## 2.1.2
 

src/decoupler/_download.py

@@ -3,14 +3,15 @@
 import pandas as pd
 import requests
 from tqdm import tqdm
+import time
 
 from decoupler._log import _log
 
 URL_DBS = "https://omnipathdb.org/annotations?databases="
 URL_INT = "https://omnipathdb.org/interactions/?genesymbols=1&"
 
 
-def _download(
+def _download_chunks(
     url: str,
     verbose: bool = False,
 ) -> io.BytesIO:
@@ -33,6 +34,35 @@ def _download(
     return data
 
 
+def _download(
+    url: str,
+    verbose: bool = False,
+    retries: int = 5,
+    wait_time: int = 20,
+) -> io.BytesIO:
+    m = f"Downloading {url}"
+    _log(m, level="info", verbose=verbose)
+    data = None
+    for attempt in range(1, retries + 1):
+        try:
+            data = _download_chunks(url, verbose=False)
+            break
+        except requests.exceptions.HTTPError as e:
+            status_code = e.response.status_code if e.response is not None else None
+            if status_code == 429 and attempt < retries:
+                _log(
+                    f"429 Too Many Requests for {url}. Retrying in {wait_time}s (attempt {attempt + 1}/{retries})",
+                    level="warn",
+                    verbose=verbose,
+                )
+                time.sleep(wait_time)
+                continue
+            raise  # Not a 429 or no retries left: re-raise
+    m = "Download finished"
+    _log(m, level="info", verbose=verbose)
+    return data
+
+
 def _bytes_to_pandas(data: io.BytesIO, **kwargs) -> pd.DataFrame:
     df = pd.read_csv(data, **kwargs)
     return df