New Code architecture

mds/__init__.py

@@ -0,0 +1,5 @@
+"""General mds-toolbox setup"""
+
+
+def setup():
+    pass

src/download/copernicus.py → mds/core/copernicus.py

@@ -3,7 +3,7 @@
 import requests
 
 from typing import List
-from src.download import utils
+from mds.core import utils
 
 
 GET_MANDATORY_ATTRS = ["filter", "output_directory", "dataset_id"]

src/download/mds_s3.py → mds/core/mds_s3.py

@@ -2,10 +2,10 @@
 from multiprocessing import Pool
 from typing import List
 
-from src.download.s3file import S3File
-from src.download import s3_singleton
-from src.download import utils
-from src.lib import logging_config
+from mds.core.s3file import S3File
+from mds.core import s3_singleton
+from mds.core import utils
+from mds.utils import logging_config
 
 # conf
 logger = logging_config.set_up("mds_s3")

src/download/s3_singleton.py → mds/core/s3_singleton.py

@@ -7,8 +7,8 @@
 from botocore import UNSIGNED
 from botocore.config import Config
 
-from src.download.s3file import S3File
-from src.lib import logging_config
+from mds.core.s3file import S3File
+from mds.utils import logging_config
 
 # conf
 lock = multiprocessing.Lock()

src/download/utils.py → mds/core/utils.py

@@ -7,7 +7,7 @@
 
 from typing import Sequence, Callable
 
-from src.lib import logging_config
+from mds.utils import logging_config
 
 logger = logging_config.set_up("utils")
 

src/download/wrapper.py → mds/core/wrapper.py

@@ -14,9 +14,9 @@
 from botocore import UNSIGNED
 from botocore.config import Config
 
-from src.download import utils, copernicus
-from src.download.utils import etag_match
-from src.lib import logging_config
+from mds.core import utils, copernicus
+from mds.core.utils import etag_match
+from mds.utils import logging_config
 
 # log
 logger = logging_config.set_up("mds")

mds/mng/__init__.py

@@ -0,0 +1,16 @@
+import logging
+import traceback
+
+from mds.mng.cli import cli
+
+logger = logging.getLogger("mds")
+
+
+def start_from_command_line_interface():
+    """Access point to CLI API"""
+    try:
+        cli()
+    except Exception as e:
+        logger.debug(traceback.format_exc())
+        logger.error(e)
+        exit(1)

mds/mng/app.py

@@ -0,0 +1 @@
+"""Entry point to mds-toolbox API"""

mds.py → mds/mng/cli.py

@@ -9,16 +9,16 @@
 
 import click
 
-from src.download import wrapper
-from src.download import mds_s3
+from mds.core import wrapper
+from mds.core import mds_s3
 
 
 @click.group()
-def main():
+def cli():
     pass
 
 
-@main.command()
+@cli.command()
 @click.option(
     "-o", "--output-directory", required=True, type=str, help="Output directory"
 )
@@ -83,7 +83,7 @@ def subset(**kwargs):
     wrapper.mds_download("subset", **kwargs)
 
 
-@main.command()
+@cli.command()
 @click.option(
     "-f", "--filter", required=False, type=str, help="Filter on the online files"
 )
@@ -132,7 +132,7 @@ def get(**kwargs):
         wrapper.mds_download("get", **kwargs)
 
 
-@main.command()
+@cli.command()
 @click.argument("dataset_id", type=str)
 @click.argument("mds_filter", type=str)
 @click.option(
@@ -143,7 +143,7 @@ def file_list(*args, **kwargs):
     print(f"{' '.join(mds_file_list)}")
 
 
-@main.command()
+@cli.command()
 @click.option(
     "-e",
     "--s3_file",
@@ -180,7 +180,7 @@ def etag(**kwargs):
         print(f"{s3_file.name} {s3_file.etag}")
 
 
-@main.command()
+@cli.command()
 @click.option(
     "-b", "--bucket", "s3_bucket", required=True, type=str, help="Bucket name"
 )
@@ -249,7 +249,7 @@ def s3_get(**kwargs):
     mds_s3.download_files(**kwargs)
 
 
-@main.command()
+@cli.command()
 @click.option(
     "-b",
     "--bucket",
@@ -287,4 +287,4 @@ def s3_list(**kwargs):
 
 
 if __name__ == "__main__":
-    main()
+    cli()

run_cli.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+"""Temporary script to start mds-toolbox from CLI"""
+
+from mds.mng import cli
+
+
+def main():
+    cli()
+
+
+if __name__ == "__main__":
+    main()

tests/test_mds.py

@@ -2,8 +2,8 @@
 import os
 import pytest
 
-import src.download.wrapper
-from src.download import utils
+import mds.core.wrapper
+from mds.core import utils
 
 
 def test_get_temporary_directory():
@@ -28,7 +28,7 @@ def test_cwf():
 
 def test_mds_download_wrong_mode(tmp_path):
     with pytest.raises(ValueError):
-        src.download.wrapper.mds_download("wrong")
+        mds.download.wrapper.mds_download("wrong")
 
 
 def test_mds_get_not_found(tmp_path):
@@ -39,7 +39,7 @@ def test_mds_get_not_found(tmp_path):
         )
         output_path = f"{tmp_path}"
 
-        src.download.wrapper.mds_download(
+        mds.download.wrapper.mds_download(
             "get",
             filter=output_filename,
             output_directory=output_path,
@@ -52,7 +52,7 @@ def test_mds_get_download(tmp_path):
     output_filename = "20231231_d-CMCC--TEMP-MFSeas8-MEDATL-b20240109_an-sv09.00.nc"
     output_path = f"{tmp_path}"
 
-    src.download.wrapper.mds_download(
+    mds.download.wrapper.mds_download(
         "get", filter=output_filename, output_directory=output_path, dataset_id=dataset
     )
 
@@ -63,7 +63,7 @@ def test_mds_get_list():
     dataset = "cmems_mod_med_phy-tem_anfc_4.2km_P1D-m"
     mds_filter = "*-CMCC--TEMP-MFSeas8-MEDATL-b20240109_an-sv09.00.nc"
 
-    result = src.download.wrapper.mds_list(dataset, mds_filter)
+    result = mds.download.wrapper.mds_list(dataset, mds_filter)
 
     assert isinstance(result, list)
     # best analysis keep on mds

tests/test_utils.py

@@ -1,4 +1,4 @@
-from src.download import utils
+from mds.core import utils
 
 
 def test_split_list_into_parts():