Problems with datasets downloads

[toncho11]

I have the impression that C:\data\MNE-schirrmeister2017-data\robintibor\high-gamma-dataset\raw\master\data\train\ is being downloaded every time. It is downloading even if it was downloaded correctly at least one time. I think it already did reduce the life of my SSD.

I am using the benchmark code.
I am even using njobs = 1

[toncho11]

I am currently experimenting with this code.

The code replaces the default downloader in MOABB and adds this functionality:

• Skips files already downloaded - avoids unnecessary downloads
• Resumes partial downloads - no loss of progress after interruptions
• Retries on download errors - handles unstable servers reliably
• Avoids corrupted partial files

import os
import requests
from pathlib import Path
import moabb.datasets.download as dl

MAX_RETRIES = 20
CHUNK_SIZE = 1024 * 1024  # 1 MB

def robust_downloader(url, dst, pooch=None):

    dst = Path(dst)
    tmp = dst.with_suffix(".partial")

    # -------------------------------------------------------------
    # 1) FULL FILE EXISTS → SKIP
    # -------------------------------------------------------------
    if dst.exists():
        print(f"[skip] {dst.name} already exists.")
        return str(dst)

    # -------------------------------------------------------------
    # 2) DETERMINE IF WE SHOULD RESUME
    # -------------------------------------------------------------
    headers = {}
    if tmp.exists():
        downloaded = tmp.stat().st_size
        headers["Range"] = f"bytes={downloaded}-"
        print(f"[resume] Resuming {dst.name} at {downloaded} bytes")
    else:
        downloaded = 0
        print(f"[start] Beginning download: {dst.name}")

    # Ensure folder exists
    dst.parent.mkdir(parents=True, exist_ok=True)

    # -------------------------------------------------------------
    # 3) RETRY LOOP (resume is preserved across retries)
    # -------------------------------------------------------------
    for attempt in range(1, MAX_RETRIES + 1):
        try:
            print(f"[download] {dst.name} — Attempt {attempt}/{MAX_RETRIES}")

            with requests.get(url, stream=True, timeout=60, headers=headers) as r:
                r.raise_for_status()

                # Append if resuming, else write new file
                mode = "ab" if tmp.exists() else "wb"
                with open(tmp, mode) as f:
                    for chunk in r.iter_content(CHUNK_SIZE):
                        if chunk:
                            f.write(chunk)

            # Success! Move temp file into final location
            tmp.rename(dst)
            print(f"[OK] Finished: {dst.name}\n")
            return str(dst)

        except Exception as e:
            print(f"[ERR] {e} — retrying...\n")

    # -------------------------------------------------------------
    # 4) All retries exhausted → Delete broken partial & fail
    # -------------------------------------------------------------
    print(f"[FAIL] Could not download {dst.name} after {MAX_RETRIES} retries.")
    if tmp.exists():
        tmp.unlink()
    raise RuntimeError(f"Download failed: {url}")

_original_data_dl = dl.data_dl

def patched_data_dl(url, sign, path, force_update=False, verbose=True):
    filename = os.path.basename(url)

    if path is None:
        path = os.environ.get("MOABB_CACHE_DIR",
               os.environ.get("MNE_DATA", None))

        if path is None:
            raise RuntimeError(
                f"MOABB downloader received path=None for {filename}, "
                "and no fallback directory was found. "
                "Set MOABB_CACHE_DIR or MNE_DATA before running benchmark()."
            )

    path = str(path)
    dst = Path(path) / filename

    return robust_downloader(url, dst)

# Install the dl replacement globally
dl.data_dl = patched_data_dl