Fix download logic

icenet_mp/cli/datasets.py

@@ -32,93 +32,18 @@ def create(
 
 @datasets_cli.command("inspect")
 @hydra_adaptor
-def inspect(config: DictConfig) -> None:
-    """Inspect all datasets."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        logger.info("Working on %s.", downloader.name)
-        downloader.inspect()
-
-
-@datasets_cli.command("init")
-@hydra_adaptor
-def init(
+def inspect(
     config: DictConfig,
     *,
-    overwrite: Annotated[
-        bool, typer.Option(help="Specify whether to overwrite existing datasets")
+    statistics: Annotated[
+        bool, typer.Option(help="Specify whether to show dataset statistics")
     ] = False,
 ) -> None:
-    """Create all datasets."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        logger.info("Working on %s.", downloader.name)
-        downloader.init(overwrite=overwrite)
-
-
-@datasets_cli.command("load")
-@hydra_adaptor
-def load(
-    config: DictConfig,
-    parts: Annotated[str, typer.Option(help="The part to process, specified as 'i/n'")],
-) -> None:
-    """Load dataset in parts."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        logger.info("Working on %s.", downloader.name)
-        downloader.load(parts=parts)
-
-
-@datasets_cli.command("load_in_parts")
-@hydra_adaptor
-def load_in_parts(
-    config: DictConfig,
-    *,
-    continue_on_error: Annotated[
-        bool, typer.Option(help="Continue to next part on error")
-    ] = True,
-    force_reset: Annotated[
-        bool,
-        typer.Option(
-            help="Clear existing progress part_tracker file and start from part 1"
-        ),
-    ] = False,
-    dataset: Annotated[
-        str | None, typer.Option(help="Run only a single dataset by name")
-    ] = None,
-    total_parts: Annotated[
-        int, typer.Option(help="Override default total parts (10)")
-    ] = 10,
-    overwrite: Annotated[
-        bool,
-        typer.Option(help="Delete the dataset directory before loading"),
-    ] = False,
-) -> None:
-    """Load all parts for all datasets in parts, recording progress so runs can be resumed."""
-    factory = DataDownloaderFactory(config)
-    for downloader in factory.downloaders:
-        if dataset and downloader.name != dataset:
-            logger.info("Not loading %s.", downloader.name)
-            continue
-        logger.info("Working on %s.", downloader.name)
-        downloader.load_in_parts(
-            continue_on_error=continue_on_error,
-            force_reset=force_reset,
-            total_parts=total_parts,
-            overwrite=overwrite,
-        )
-
-
-@datasets_cli.command("finalise")
-@hydra_adaptor
-def finalise(
-    config: DictConfig,
-) -> None:
-    """Finalise loaded dataset."""
+    """Inspect all datasets."""
     factory = DataDownloaderFactory(config)
     for downloader in factory.downloaders:
         logger.info("Working on %s.", downloader.name)
-        downloader.finalise()
+        downloader.inspect(statistics=statistics)
 
 
 if __name__ == "__main__":