Backport PR #4011 on branch 1.12.x (refactor: Move datasets to scverse AWS S3 urls) (#4021)

Co-authored-by: Lukas Heumos <lukas.heumos@posteo.net>

Author: meeseeksmachine

docs/release-notes/4011.feat.md

@@ -0,0 +1 @@
+Use scverse S3 Cloudfront URLs for datasets {smaller}`zethson`

pyproject.toml

@@ -52,6 +52,7 @@ classifiers = [
 dynamic = [ "version" ]
 dependencies = [
     "anndata>=0.10.8",
+    "certifi",
     "fast-array-utils[accel,sparse]>=1.2.1",
     "h5py>=3.11",
     "joblib",

src/scanpy/_utils/__init__.py

@@ -847,7 +847,7 @@ def select_groups(
     return groups_order_subset, groups_masks_obs
 
 
-def check_presence_download(filename: Path, backup_url):
+def check_presence_download(filename: Path, backup_url: str):
     """Check if file is present otherwise download."""
     if not filename.is_file():
         from ..readwrite import _download

src/scanpy/datasets/_datasets.py

@@ -130,7 +130,7 @@ def burczynski06() -> AnnData:
 
     """
     filename = settings.datasetdir / "burczynski06/GDS1615_full.soft.gz"
-    url = "ftp://ftp.ncbi.nlm.nih.gov/geo/datasets/GDS1nnn/GDS1615/soft/GDS1615_full.soft.gz"
+    url = "https://exampledata.scverse.org/scanpy/GDS1615_full.soft.gz"
     return read(filename, backup_url=url)
 
 
@@ -206,7 +206,9 @@ def moignard15() -> AnnData:
 
     """
     filename = settings.datasetdir / "moignard15/nbt.3154-S3.xlsx"
-    backup_url = "https://static-content.springer.com/esm/art%3A10.1038%2Fnbt.3154/MediaObjects/41587_2015_BFnbt3154_MOESM4_ESM.xlsx"
+    backup_url = (
+        "https://exampledata.scverse.org/scanpy/41587_2015_BFnbt3154_MOESM4_ESM.xlsx"
+    )
     adata = read(filename, sheet="dCt_values.txt", backup_url=backup_url)
     # filter out 4 genes as in Haghverdi et al. (2016)
     gene_subset = ~np.isin(adata.var_names, ["Eif2b1", "Mrpl19", "Polr2a", "Ubc"])
@@ -241,9 +243,8 @@ def paul15() -> AnnData:
 
     Non-logarithmized raw data.
 
-    The data has been sent out by Email from the Amit Lab. An R version for
-    loading the data can be found `here
-    <https://github.com/theislab/scAnalysisTutorial>`_.
+    The data has been sent out by Email from the Amit Lab.
+    An R version for loading the data can be found `here <https://github.com/theislab/scAnalysisTutorial>`_.
 
     Returns
     -------
@@ -262,7 +263,7 @@ def paul15() -> AnnData:
 
     filename = settings.datasetdir / "paul15/paul15.h5"
     filename.parent.mkdir(exist_ok=True)
-    backup_url = "https://falexwolf.de/data/paul15.h5"
+    backup_url = "https://exampledata.scverse.org/scanpy/paul15.h5"
     _utils.check_presence_download(filename, backup_url)
     with h5py.File(filename, "r") as f:
         # Coercing to float32 for backwards compatibility
@@ -332,8 +333,7 @@ def pbmc68k_reduced() -> AnnData:
     It was saved keeping only 724 cells and 221 highly variable genes.
 
     The saved file contains the annotation of cell types (key: `'bulk_labels'`),
-    UMAP coordinates, louvain clustering and gene rankings based on the
-    `bulk_labels`.
+    UMAP coordinates, louvain clustering and gene rankings based on the `bulk_labels`.
 
     .. [#norm] Back when the dataset was created, :func:`~scanpy.pp.normalize_per_cell` was used instead.
     .. _PBMC 68k dataset: https://www.10xgenomics.com/datasets/fresh-68-k-pbm-cs-donor-a-1-standard-1-1-0
@@ -404,7 +404,7 @@ def pbmc3k() -> AnnData:
         var: 'gene_ids'
 
     """
-    url = "https://falexwolf.de/data/pbmc3k_raw.h5ad"
+    url = "https://exampledata.scverse.org/scanpy/pbmc3k_raw.h5ad"
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=OldFormatWarning)
         adata = read(settings.datasetdir / "pbmc3k_raw.h5ad", backup_url=url)
@@ -444,7 +444,7 @@ def pbmc3k_processed() -> AnnData:
         obsp: 'distances', 'connectivities'
 
     """  # noqa: D401
-    url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+    url = "https://exampledata.scverse.org/scanpy/pbmc3k.h5ad"
 
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=OldFormatWarning)
@@ -475,7 +475,7 @@ def _download_visium_dataset(
     if base_dir is None:
         base_dir = settings.datasetdir
 
-    url_prefix = f"https://cf.10xgenomics.com/samples/spatial-exp/{spaceranger_version}/{sample_id}"
+    url_prefix = f"https://exampledata.scverse.org/scanpy/visium/{spaceranger_version}/{sample_id}"
 
     sample_dir = base_dir / sample_id
     sample_dir.mkdir(exist_ok=True)

src/scanpy/readwrite.py

@@ -1061,9 +1061,10 @@ def _get_filename_from_key(key, ext=None) -> Path:
 
 
 def _download(url: str, path: Path):
-    from urllib.error import URLError
+    from ssl import create_default_context
     from urllib.request import Request, urlopen
 
+    from certifi import contents
     from tqdm.auto import tqdm
 
     blocksize = 1024 * 8
@@ -1072,26 +1073,7 @@ def _download(url: str, path: Path):
     try:
         req = Request(url, headers={"User-agent": "scanpy-user"})
 
-        try:
-            open_url = urlopen(req)
-        except URLError:
-            if not url.startswith("https://"):
-                raise  # No need to try using certifi
-
-            msg = "Failed to open the url with default certificates."
-            try:
-                from certifi import where
-            except ImportError as e:
-                e.add_note(f"{msg} Please install `certifi` and try again.")
-                raise
-            else:
-                logg.warning(f"{msg} Trying to use certifi.")
-
-            from ssl import create_default_context
-
-            open_url = urlopen(req, context=create_default_context(cafile=where()))
-
-        with open_url as resp:
+        with urlopen(req, context=create_default_context(cadata=contents())) as resp:
             total = resp.info().get("content-length", None)
             with (
                 tqdm(