Fix download logic (#230)

* 👽 Remove workaround now that upstream change has been merged

* ✨ Check dataset status before deciding what to do with it

* ✨ Add simple function to load in chunks using native Anemoi functionality

* ✨ Add an initialise function to wrap Anemoi Init

* ♻️ Replace download Create with download in parts

* ⚰️ Remove load_in_parts and associated tracking functionality as this is handled directly by Anemoi

* ⚰️ Remove init command and CLI

* ⚰️ Remove load command and CLI

* 🚚 Alphabetise DataDownloader methods

* 🚸 Also check whether statistics are ready before finalising

* 🐛 Remove invalid datasets whether or not they appear to be complete

* ⚰️ Removed tests for removed code

* ⚗️ Test whether chunk-checking is sufficient to cause downloads to resume

* 🔧 Default to statistics=False when inspecting

* 🥅 Exit gracefully when inspect fails during download

* 🔧 Do not set progress when detailed is set or the same information is printed twice

* 🐛 If only the statistics are missing we need to finalise

* 🚨 Perform explicit type checking to avoid linting errors

* 🐛 Finalise when statistics are *not* ready

* 🐛 Ensure that typer.Exit is raised

* 🥅 Raise exception if status cannot be retrieved, rather than using default values

* ⚰️ Removed the unused finalise CLI command

Author: jemrobinson

icenet_mp/cli/datasets.py

@@ -32,93 +32,18 @@ def create(
 
 @datasets_cli.command("inspect")
 @hydra_adaptor
-def inspect(config: DictConfig) -> None:
-    """Inspect all datasets."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        logger.info("Working on %s.", downloader.name)
-        downloader.inspect()
-
-
-@datasets_cli.command("init")
-@hydra_adaptor
-def init(
+def inspect(
     config: DictConfig,
     *,
-    overwrite: Annotated[
-        bool, typer.Option(help="Specify whether to overwrite existing datasets")
+    statistics: Annotated[
+        bool, typer.Option(help="Specify whether to show dataset statistics")
     ] = False,
 ) -> None:
-    """Create all datasets."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        logger.info("Working on %s.", downloader.name)
-        downloader.init(overwrite=overwrite)
-
-
-@datasets_cli.command("load")
-@hydra_adaptor
-def load(
-    config: DictConfig,
-    parts: Annotated[str, typer.Option(help="The part to process, specified as 'i/n'")],
-) -> None:
-    """Load dataset in parts."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        logger.info("Working on %s.", downloader.name)
-        downloader.load(parts=parts)
-
-
-@datasets_cli.command("load_in_parts")
-@hydra_adaptor
-def load_in_parts(
-    config: DictConfig,
-    *,
-    continue_on_error: Annotated[
-        bool, typer.Option(help="Continue to next part on error")
-    ] = True,
-    force_reset: Annotated[
-        bool,
-        typer.Option(
-            help="Clear existing progress part_tracker file and start from part 1"
-        ),
-    ] = False,
-    dataset: Annotated[
-        str | None, typer.Option(help="Run only a single dataset by name")
-    ] = None,
-    total_parts: Annotated[
-        int, typer.Option(help="Override default total parts (10)")
-    ] = 10,
-    overwrite: Annotated[
-        bool,
-        typer.Option(help="Delete the dataset directory before loading"),
-    ] = False,
-) -> None:
-    """Load all parts for all datasets in parts, recording progress so runs can be resumed."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        if dataset and downloader.name != dataset:
-            logger.info("Not loading %s.", downloader.name)
-            continue
-        logger.info("Working on %s.", downloader.name)
-        downloader.load_in_parts(
-            continue_on_error=continue_on_error,
-            force_reset=force_reset,
-            total_parts=total_parts,
-            overwrite=overwrite,
-        )
-
-
-@datasets_cli.command("finalise")
-@hydra_adaptor
-def finalise(
-    config: DictConfig,
-) -> None:
-    """Finalise loaded dataset."""
+    """Inspect all datasets."""
     factory = DataDownloaderFactory(config)
     for downloader in factory.downloaders:
         logger.info("Working on %s.", downloader.name)
-        downloader.finalise()
+        downloader.inspect(statistics=statistics)
 
 
 if __name__ == "__main__":