Configurarion package and new log settings

mds/__init__.py

@@ -1,5 +1,21 @@
-"""General mds-toolbox setup"""
+from mds.conf import settings
+from mds.core import mds_s3
+from mds.core import wrapper
+from mds.utils.log import configure_logging
 
 
-def setup():
-    pass
+def setup(**kwargs) -> None:
+    """
+    General mds-toolbox setup
+
+    Args:
+        **kwargs: extra arguments to apply as app settings
+    """
+    settings.configure(**kwargs)
+    configure_logging(settings.LOGGING_CONFIG, settings.LOGGING, settings.LOG_LEVEL)
+
+
+__all__ = [
+    mds_s3.__name__,
+    wrapper.__name__,
+]

mds/conf/__init__.py

@@ -0,0 +1,34 @@
+from mds.conf import global_settings
+
+# List of modules to load settings from
+TO_LOAD = [global_settings]
+
+
+class Settings:
+    def __init__(self, *modules):
+        """
+        Initialize the Settings instance with the provided modules.
+
+        Args:
+            *modules: Variable length argument list of modules to load settings from.
+        """
+        for module in modules:
+            for setting in dir(module):
+                if setting.isupper():
+                    setattr(self, setting, getattr(module, setting))
+
+    def configure(self, **ext_settings):
+        """
+        Configure the settings instance by setting new values or overriding existing ones.
+
+        Args:
+            **ext_settings: Arbitrary keyword arguments representing settings to be configured.
+                            Only capital keywords are considered.
+        """
+        for key, value in ext_settings.items():
+            if key.isupper():
+                setattr(self, key, value)
+
+
+# Create a Settings instance as unique entry point to the app settings
+settings = Settings(*TO_LOAD)

mds/conf/global_settings.py

@@ -0,0 +1,11 @@
+########################
+# LOG
+########################
+
+# The callable to use to configure logging
+LOGGING_CONFIG = "logging.config.dictConfig"
+
+# Custom logging configuration.
+LOGGING = {}
+
+LOG_LEVEL = "INFO"

mds/core/mds_s3.py

@@ -1,14 +1,14 @@
+import logging
 import os
 from multiprocessing import Pool
 from typing import List
 
-from mds.core.s3file import S3File
 from mds.core import s3_singleton
 from mds.core import utils
-from mds.utils import logging_config
+from mds.core.s3file import S3File
 
 # conf
-logger = logging_config.set_up("mds_s3")
+logger = logging.getLogger("mds")
 THREADS_TIMEOUT = 10 * 60
 
 

mds/core/s3_singleton.py

@@ -1,19 +1,19 @@
-import glob
 import fnmatch
+import glob
+import logging
 import multiprocessing
-import boto3
 import os
 
+import boto3
 from botocore import UNSIGNED
 from botocore.config import Config
 
 from mds.core.s3file import S3File
-from mds.utils import logging_config
 
 # conf
+logger = logging.getLogger("mds")
 lock = multiprocessing.Lock()
 S3_ENDPOINT = "https://s3.waw3-1.cloudferro.com"
-logger = logging_config.set_up("s3")
 
 
 class Singleton(type):

mds/core/utils.py

@@ -1,15 +1,14 @@
 import datetime
 import hashlib
 import json
+import logging
 import os
 import shutil
 import time
-
 from typing import Sequence, Callable
 
-from mds.utils import logging_config
 
-logger = logging_config.set_up("utils")
+logger = logging.getLogger("mds")
 
 
 def cwd() -> str:

mds/core/wrapper.py

@@ -16,10 +16,8 @@
 
 from mds.core import utils, copernicus
 from mds.core.utils import etag_match
-from mds.utils import logging_config
 
-# log
-logger = logging_config.set_up("mds")
+logger = logging.getLogger("mds")
 
 # conf
 DOWNLOAD_MODES = ["subset", "get"]

mds/mng/cli.py

@@ -9,8 +9,17 @@
 
 import click
 
-from mds.core import wrapper
-from mds.core import mds_s3
+from mds import mds_s3
+from mds import wrapper
+from mds.mng import initializer
+
+verbose = click.option(
+    "--log-level",
+    "LOG_LEVEL",
+    type=click.Choice(["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL", "QUIET"]),
+    default="INFO",
+    help="Verbosity level based on standard logging library",
+)
 
 
 @click.group()
@@ -79,6 +88,8 @@ def cli():
 )
 @click.option("-n", "--username", type=str, default=None, help="Username")
 @click.option("-w", "--password", type=str, default=None, help="Password")
+@verbose
+@initializer.init_app()
 def subset(**kwargs):
     wrapper.mds_download("subset", **kwargs)
 
@@ -124,6 +135,8 @@ def subset(**kwargs):
 )
 @click.option("-n", "--username", type=str, default=None, help="Username")
 @click.option("-w", "--password", type=str, default=None, help="Password")
+@verbose
+@initializer.init_app()
 def get(**kwargs):
     update = kwargs.pop("update")
     if update:
@@ -138,6 +151,8 @@ def get(**kwargs):
 @click.option(
     "-g", "--dataset-version", type=str, default=None, help="Dataset version or tag"
 )
+@verbose
+@initializer.init_app()
 def file_list(*args, **kwargs):
     mds_file_list = wrapper.mds_list(*args, **kwargs)
     print(f"{' '.join(mds_file_list)}")
@@ -174,6 +189,8 @@ def file_list(*args, **kwargs):
     default=None,
     help="Pattern to filter data (no regex)",
 )
+@verbose
+@initializer.init_app()
 def etag(**kwargs):
     s3_files = wrapper.mds_etag(**kwargs)
     for s3_file in s3_files:
@@ -245,6 +262,8 @@ def etag(**kwargs):
     default=False,
     help="Update the file if it changes on the server using etag information",
 )
+@verbose
+@initializer.init_app()
 def s3_get(**kwargs):
     mds_s3.download_files(**kwargs)
 
@@ -281,6 +300,8 @@ def s3_get(**kwargs):
 @click.option(
     "-r", "--recursive", is_flag=True, default=False, help="List recursive all s3 files"
 )
+@verbose
+@initializer.init_app()
 def s3_list(**kwargs):
     s3_files = mds_s3.get_file_list(**kwargs)
     print(f"{' '.join([f.file for f in s3_files])}")

mds/mng/initializer.py

@@ -0,0 +1,38 @@
+"""Entry point to mds-toolbox API"""
+
+import functools
+
+import mds
+
+
+def init_app():
+    """Initialize the application functions before the execution"""
+
+    def decorated(func):
+        @functools.wraps(func)
+        def setup(**kwargs):
+            """
+            Extract app settings and configure application setup, then start mds functions.
+
+            Args:
+                **kwargs: Arbitrary keyword arguments passed to the function.
+            """
+            user_settings = dict()
+            # Assume that upper settings are app settings related
+            upper_keys = [k for k in kwargs.keys() if k.isupper()]
+
+            # Remove upper settings from plot arguments to avoid crash in PlotSettings class
+            for k in upper_keys:
+                user_settings[k] = kwargs.pop(k)
+
+            # Perform general app initialization
+            mds.setup(**user_settings)
+
+            # Start application
+            return func(**kwargs)
+
+        # Return the setup function which wraps the original function
+        return setup
+
+    # Return the decorator function
+    return decorated

mds/utils/log.py

@@ -0,0 +1,61 @@
+import logging.config
+
+from mds.conf import settings
+from mds.utils.module_loading import import_string
+
+LOGGER_NAME = "mds"
+
+DEFAULT_LOGGING = {
+    "disable_existing_loggers": False,
+    "formatters": {
+        "blank": {"format": "%(message)s"},
+        "simple": {
+            "datefmt": "%Y-%m-%dT%H:%M:%SZ",
+            "format": "[%(asctime)s] - %(levelname)s - %(message)s",
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "simple",
+            "level": "DEBUG",
+            "stream": "ext://sys.stderr",
+        },
+    },
+    "loggers": {
+        LOGGER_NAME: {
+            "handlers": ["console"],
+            "level": settings.LOG_LEVEL,
+        },
+    },
+    "version": 1,
+}
+
+
+def configure_logging(
+    logging_config: str, logging_settings: dict, log_level: str
+) -> None:
+    """
+
+    Args:
+        logging_config: Callable to use to configure logging
+        logging_settings: Dictionary of logging settings
+        log_level: Logging level
+    """
+    if logging_config:
+        logging_config_func = import_string(logging_config)
+
+        # Call custom logging config function first to avoid overrides
+        if logging_settings:
+            logging_config_func(logging_settings)
+
+    if log_level:
+        # Update logging level before applying default config
+        DEFAULT_LOGGING["loggers"][LOGGER_NAME]["level"] = log_level
+
+    # Apply the updated logging config
+    logging.config.dictConfig(DEFAULT_LOGGING)
+
+    # Ensure logger picks up the new level
+    logger = logging.getLogger(LOGGER_NAME)
+    logger.setLevel(log_level)