Merge pull request #12 from SciCatProject/logging

Refactor logging configuration.

Author: YooSunYoung

config.20240405.json

@@ -31,12 +31,12 @@
     "config_file" : "config.json",
     "verbose" : false,
     "file_log" : false,
-    "log_file_suffix" : ".scicat_ingestor_log",
+    "log_filepath_prefix" : ".scicat_ingestor_log",
     "file_log_timestamp" : false,
     "log_level" : "INFO",
     "system_log" : false,
     "system_log_facility" : "mail",
-    "log_prefix" : " SFI: ",
+    "log_message_prefix" : " SFI: ",
     "check_by_job_id" : true,
     "pyscicat": null
   }

resources/config.sample.json

@@ -31,12 +31,12 @@
     "config_file" : "config.json",
     "verbose" : false,
     "file_log" : false,
-    "log_file_suffix" : ".scicat_ingestor_log",
+    "log_filepath_prefix" : ".scicat_ingestor_log",
     "file_log_timestamp" : false,
     "log_level" : "INFO",
     "system_log" : false,
     "system_log_facility" : "mail",
-    "log_prefix" : " SFI: ",
+    "log_message_prefix" : " SFI: ",
     "check_by_job_id" : true,
     "pyscicat": null
   }

src/scicat_configuration.py

@@ -36,9 +36,9 @@ def build_main_arg_parser() -> argparse.ArgumentParser:
         default=False,
     )
     group.add_argument(
-        '--log-file-suffix',
-        dest='log_file_suffix',
-        help='Suffix of the log file name',
+        '--log-filepath-prefix',
+        dest='log_filepath_prefix',
+        help='Prefix of the log file path',
         default='.scicat_ingestor_log',
     )
     group.add_argument(
@@ -62,8 +62,8 @@ def build_main_arg_parser() -> argparse.ArgumentParser:
         default='mail',
     )
     group.add_argument(
-        '--log-prefix',
-        dest='log_prefix',
+        '--log-message-prefix',
+        dest='log_message_prefix',
         help='Prefix for log messages',
         default=' SFI: ',
     )
@@ -92,13 +92,13 @@ class RunOptions:
     config_file: str
     verbose: bool
     file_log: bool
-    log_file_suffix: str
+    log_filepath_prefix: str
     file_log_timestamp: bool
     system_log: bool
-    system_log_facility: str
-    log_prefix: str
+    log_message_prefix: str
     log_level: str
     check_by_job_id: bool
+    system_log_facility: Optional[str] = None
     pyscicat: Optional[str] = None
 
 
@@ -109,6 +109,19 @@ class ScicatConfig:
     run_options: RunOptions
     """Merged configuration dictionary with command line arguments."""
 
+    def to_dict(self) -> dict:
+        """Return the configuration as a dictionary."""
+        from dataclasses import asdict
+
+        # Deep copy the original dictionary recursively
+        original_dict = dict(self.original_dict)
+        for key, value in original_dict.items():
+            if isinstance(value, Mapping):
+                original_dict[key] = dict(value)
+
+        copied = ScicatConfig(original_dict, self.run_options)
+        return asdict(copied)
+
 
 def build_scicat_config(input_args: argparse.Namespace) -> ScicatConfig:
     """Merge configuration from the configuration file and input arguments."""

src/scicat_ingestor.py

@@ -1,11 +1,16 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2024 ScicatProject contributors (https://github.com/ScicatProject)
 from scicat_configuration import build_main_arg_parser, build_scicat_config
+from scicat_logging import build_logger
 
 
 def main() -> None:
     """Main entry point of the app."""
     arg_parser = build_main_arg_parser()
     arg_namespace = arg_parser.parse_args()
     config = build_scicat_config(arg_namespace)
-    print(config)
+    logger = build_logger(config)
+
+    # Log the configuration as dictionary so that it is easier to read from the logs
+    logger.info('Starting the Scicat Ingestor with the following configuration:')
+    logger.info(config.to_dict())

src/scicat_logging.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2024 ScicatProject contributors (https://github.com/ScicatProject)
+import logging
+import logging.handlers
+from datetime import datetime
+
+from scicat_configuration import ScicatConfig
+
+
+def build_logger(config: ScicatConfig) -> logging.Logger:
+    """Build a logger and configure it according to the ``config``."""
+    run_options = config.run_options
+
+    # Build logger and formatter
+    logger = logging.getLogger('esd extract parameters')
+    formatter = logging.Formatter(
+        run_options.log_message_prefix
+        + '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+
+    # Add FileHandler
+    if run_options.file_log:
+        file_name_components = [run_options.log_filepath_prefix]
+        if run_options.file_log_timestamp:
+            file_name_components.append(datetime.now().strftime('%Y%m%d%H%M%S%f'))
+        file_name_components.append('.log')
+
+        file_name = '_'.join(file_name_components)
+        file_handler = logging.FileHandler(file_name, mode='w', encoding='utf-8')
+        logger.addHandler(file_handler)
+
+    # Add SysLogHandler
+    if run_options.system_log:
+        logger.addHandler(logging.handlers.SysLogHandler(address='/dev/log'))
+
+    # Set the level and formatter for all handlers
+    logger.setLevel(run_options.log_level)
+    for handler in logger.handlers:
+        handler.setLevel(run_options.log_level)
+        handler.setFormatter(formatter)
+
+    # Add StreamHandler
+    # streamer handler is added last since it is using different formatter
+    if run_options.verbose:
+        from rich.logging import RichHandler
+
+        logger.addHandler(RichHandler(level=run_options.log_level))
+
+    return logger

tests/test_logging.py

@@ -0,0 +1,32 @@
+import pathlib
+
+import pytest
+
+from scicat_configuration import RunOptions, ScicatConfig
+
+
+@pytest.fixture
+def scicat_config(tmp_path: pathlib.Path) -> ScicatConfig:
+    return ScicatConfig(
+        original_dict=dict(),
+        run_options=RunOptions(
+            config_file='test',
+            verbose=True,
+            file_log=True,
+            log_filepath_prefix=(tmp_path / pathlib.Path('test')).as_posix(),
+            file_log_timestamp=True,
+            system_log=False,
+            system_log_facility=None,
+            log_message_prefix='test',
+            log_level='DEBUG',
+            check_by_job_id=True,
+            pyscicat='test',
+        ),
+    )
+
+
+def test_scicat_logging_build_logger(scicat_config: ScicatConfig) -> None:
+    from scicat_logging import build_logger
+
+    logger = build_logger(scicat_config)
+    assert len(logger.handlers) == 2  # FileHandler and StreamHandler

tox.ini

@@ -6,8 +6,8 @@ isolated_build = true
 deps = -r requirements/test.txt
 setenv =
   JUPYTER_PLATFORM_DIRS = 1
-commands = pytest {posargs}
-           scicat_ingestor --help  # Minimal test of the script
+commands = scicat_ingestor --help  # Minimal test of the script
+           pytest {posargs}
 
 [testenv:nightly]
 deps = -r requirements/nightly.txt