chores: store progres by including logging and configuration modules.

Author: ecarrenolozano

.gitignore

@@ -21,3 +21,8 @@ __pycache__/
 /.pytest_cache/
 /.cache/
 /data/
+
+
+#Other
+demo/*
+sandbox/*

project_route.md

@@ -0,0 +1,88 @@
+# Rationale
+
+Our architecture consists of a number of Python packages working together. These packages require some common technical functionalities, such as logging or config handling, which are provided in well established solutions such as the logging module of Python standard library. In the past, we relied on our own implementations instead of the established solutions. In a previous attempt we intended to share these implementations across the packages, and we created the pypath-common package for this. Pypath-common came with minimal changes, it was mostly reorganization of existing code, and aimed for a working—but not optimal and future proof—solution asap. To address this shortcoming, we should migrate to standard solutions. To control these standard solutions in a way tailored to our software ecosystem, we should create a minimal layer on top of them. Below we specify what we expect from this new component.
+
+# Specification
+
+## Config handling
+
+- Discovery and merging of configs in priority order
+- Working directory, user, package built-in
+- Format: YAML
+- Choose an established solution, e.g. https://hydra.cc/docs/intro/, https://omegaconf.readthedocs.io/en/latest/ 
+- Propagate config parameters to lower level packages
+
+## Logging
+
+- Registry of “our packages”
+- Control of dispatching messages to our log file
+- Formatter
+- Log traceback
+
+### Core requirements logging
+- Plug-and-play logging and session management for Python packages.
+- Configuration via a YAML file (user provides a config file, no code changes needed for most settings).
+- Automatic logger and session setup with a single initialize(config_path) call.
+- Console and file logging enabled by default.
+- Log file rotation: When a log file exceeds 10 MB, a new file is created (configurable).
+- Timestamped log files: Log files include the date in their filename.
+- Log directory is created automatically if it does not exist.
+- Logger exclusion: User can specify loggers (e.g., pandas, matplotlib) to suppress or set to a higher log level via config.
+- Configurable log format, log level, log directory, app name, max file size, and backup count via YAML.
+- Session management: Centralized access to config and logger.
+- Demo folder: Example usage and configuration provided.
+- Tests folder: For unit tests (structure ready).
+- Use DictConfig to pass the logging configuration
+- Create one logger for each major component of our apps and software ecosystem
+- Also generate Json logs
+- Include the time zone in timestamps
+- Use a queue handler to make log calls non-blocking and async
+
+Optional/Future Requirements
+- Per-application log files: Ability to create a dedicated log file for each application (config-ready, not yet implemented).
+- Extensible for more session/config features in the future.
+
+## Session
+
+- Has one logger and one config
+- Keeps things together and provides access anywhere in the code
+
+
+---
+
+## Implementation Steps / Checklist
+
+1. **Config handling: YAML loader and merging**
+   - Use OmegaConf or Hydra for config loading.
+   - Support merging configs from working dir, user, and package defaults.
+   - Expose config as a DictConfig object.
+
+2. **Logger setup: rotation, format, exclusion**
+   - Implement logger setup with rotation, timestamped files, and both console and file handlers.
+   - Allow exclusion or level control of 3rd-party loggers via config.
+   - Make all parameters (format, level, dir, app name, size, backup count) configurable.
+
+3. **Session management: central access**
+   - Implement a Session class to hold config and logger.
+   - Provide global access and ensure singleton pattern.
+
+4. **Demo and example config**
+   - Provide a demo script and YAML config showing all features, including logger exclusion and rotation.
+
+5. **Component loggers and JSON logs**
+   - Support one logger per major component.
+   - Add optional JSON log output, configurable via YAML.
+
+6. **Time zone in timestamps**
+   - Ensure log timestamps include time zone info, configurable via YAML.
+
+7. **Async logging with queue handler**
+   - Implement non-blocking, async logging using a queue handler for file and/or console logs.
+
+8. **Per-application log files (future)**
+   - Design config and code to allow separate log files per app, but implement later.
+
+9. **Unit tests for all modules**
+   - Add tests for config loading, logger setup, and session management in the tests/ folder.
+
+---

pyproject.toml

@@ -29,6 +29,10 @@ classifiers = [
 ]
 dependencies = [
     "numpy>=2.2.6",
+    "omegaconf>=2.3.0",
+    "pandas>=2.3.3",
+    "python-json-logger>=4.0.0",
+    "pyyaml>=6.0.3",
     "toml"
 ]
 description = "This is session handler, configuration and logging handler for Saezlab packages and applications."
@@ -49,7 +53,8 @@ dev = [
     "distlib",
     "pre-commit",
     "bump2version",
-    "twine"
+    "twine",
+    "ipykernel"
 ]
 docs = [
     "mkdocs-material>=9.6.14",
@@ -59,6 +64,9 @@ docs = [
 security = [
     "bandit"
 ]
+semantic = [
+    "rdflib>=6.0.0"
+]
 tests = [
     "pytest>=6.0",
     "pytest-cov",

saezlab_core/config.py

@@ -0,0 +1,54 @@
+import os
+
+from omegaconf import OmegaConf, DictConfig
+
+__all__ = [
+    'ConfigLoader',
+]
+
+
+class ConfigLoader:
+    """Loader for YAML configuration files with merging and priority logic.
+
+    This class provides a static method to load and merge configuration files from
+    package defaults, user directory, working directory, and an explicit path, returning
+    a single merged DictConfig object.
+    """
+
+    @staticmethod
+    def load(
+        config_path: str,
+        search_paths: list[str] | None = None,
+        default_config: str | None = None,
+    ) -> DictConfig:
+        """Loads and merges YAML configs in priority order.
+
+        1. Package default (if provided)
+        2. User config in home dir (if exists)
+        3. Config in working dir (if exists)
+        4. Explicit config_path (highest priority)
+
+        Returns:
+            DictConfig: The merged configuration object.
+        """
+        configs = []
+        # 1. Package default
+        if default_config and os.path.exists(default_config):
+            configs.append(OmegaConf.load(default_config))
+        # 2. User config
+        user_config = os.path.expanduser('~/.saezlab_core.yaml')
+        if os.path.exists(user_config):
+            configs.append(OmegaConf.load(user_config))
+        # 3. Working dir config
+        cwd_config = os.path.join(os.getcwd(), 'saezlab_core.yaml')
+        if os.path.exists(cwd_config):
+            configs.append(OmegaConf.load(cwd_config))
+        # 4. Explicit config_path
+        if config_path and os.path.exists(config_path):
+            configs.append(OmegaConf.load(config_path))
+        # Merge all configs (later overrides earlier)
+        if configs:
+            merged = OmegaConf.merge(*configs)
+        else:
+            merged = OmegaConf.create({})
+        return merged

saezlab_core/logger.py

@@ -1,89 +1,140 @@
+import os
 import sys
-from typing import Optional
 import logging
-from datetime import datetime
-from logging.handlers import RotatingFileHandler
+from logging.handlers import QueueHandler, QueueListener, RotatingFileHandler
+
+from pythonjsonlogger import jsonlogger
 
 __all__ = [
-    'DATE_FORMAT',
-    'LOG_FORMAT',
     'get_logger',
-    'get_timestamped_log_path',
     'setup_logging',
+    'stop_async_listener',
 ]
 
-# Default format as per the requirements, adding levelname and module name for context.
-LOG_FORMAT = '[%(asctime)s]  [%(levelname)s]  [%(name)s]  %(message)s'
-DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
+_listener = None  # Global reference for QueueListener
 
 
-def setup_logging(
-    level: int = logging.INFO,
-    log_file: Optional[str] = None,
-    max_bytes: int = 10 * 1024 * 1024,  # 10 MB
-    backup_count: int = 5,
-):
-    """Configures the root logger for the application with log rotation.
+def setup_logging(config: dict) -> None:
+    """Set up logging using a DictConfig (OmegaConf) or dict.
 
-    This sets up handlers for console and optional file logging.
-    It uses a standardized format for all log messages and rotates
-    log files when they reach a specified size.
+    Supports rotation, timestamped files, console+file handlers, logger exclusion, and optional JSON logs.
 
-    :param level: The minimum logging level to capture (e.g., logging.INFO).
-    :param log_file: Optional path to a file for log output.
-    :param max_bytes: The maximum size in bytes for a log file before it is rotated.
-    :param backup_count: The number of backup log files to keep.
+    Args:
+        config (dict): Logging configuration dictionary or DictConfig.
     """
-    formatter = logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)
-
+    # Support both DictConfig and dict
+    cfg = config if isinstance(config, dict) else dict(config)
+    log_dir = cfg.get('log_dir', './log')
+    app_name = cfg.get('app_name', 'saezlab_core')
+    log_level = getattr(logging, cfg.get('level', 'INFO').upper(), logging.INFO)
+    log_format = cfg.get(
+        'format', '[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s'
+    )
+    # Support max_megabytes (preferred) or fallback to max_bytes for backward compatibility
+    if 'max_megabytes' in cfg:
+        max_bytes = int(cfg.get('max_megabytes', 10)) * 1024 * 1024
+    else:
+        max_bytes = cfg.get('max_bytes', 10 * 1024 * 1024)
+    backup_count = cfg.get('backup_count', 5)
+    timestamp = cfg.get('timestamp')
+    use_json = cfg.get('json_logs', False)
+    timezone = cfg.get('timezone', 'UTC')
+    import queue
+    from datetime import datetime
+
+    try:
+        from zoneinfo import ZoneInfo
+
+        tzinfo = ZoneInfo(timezone)
+    except ImportError:
+        # For Python <3.9, fallback to UTC
+        import pytz
+
+        tzinfo = pytz.timezone(timezone) if timezone != 'UTC' else None
+    if not timestamp:
+        timestamp = datetime.now(tz=tzinfo).strftime('%Y-%m-%d')
+    async_logging = cfg.get('async_logging', False)
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir, exist_ok=True)
+    log_file = os.path.join(log_dir, f'{app_name}_{timestamp}.log')
+
+    # Custom formatter to inject timezone-aware asctime
+    class TZFormatter(logging.Formatter):
+        def __init__(
+            self,
+            fmt: str | None = None,
+            datefmt: str | None = None,
+            tz: object = None,
+        ) -> None:
+            super().__init__(fmt=fmt, datefmt=datefmt)
+            self.tz = tz
+
+        def formatTime(
+            self, record: logging.LogRecord, datefmt: str | None = None
+        ) -> str:
+            dt = datetime.fromtimestamp(record.created, tz=self.tz)
+            if datefmt:
+                return dt.strftime(datefmt)
+            return dt.isoformat()
+
+    if use_json:
+        json_format = '%(asctime)s %(levelname)s %(name)s %(message)s'
+        formatter = jsonlogger.JsonFormatter(json_format)
+        formatter.formatTime = (
+            lambda record, datefmt=None: TZFormatter().formatTime(
+                record, datefmt
+            )
+        )
+    else:
+        formatter = TZFormatter(log_format, tz=tzinfo)
     handlers = []
-
-    # Console handler
     console_handler = logging.StreamHandler(sys.stdout)
     console_handler.setFormatter(formatter)
-    handlers.append(console_handler)
-
-    # Rotating file handler (if a path is provided)
-    if log_file:
-        # Use RotatingFileHandler for automatic log rotation.
-        file_handler = RotatingFileHandler(
-            log_file, maxBytes=max_bytes, backupCount=backup_count
-        )
-        file_handler.setFormatter(formatter)
-        handlers.append(file_handler)
-
-    # The `force=True` argument removes any existing handlers
-    # on the root logger, ensuring our configuration is the only one.
-    logging.basicConfig(level=level, handlers=handlers, force=True)
-
-    # Set up a hook for unhandled exceptions to be logged automatically.
-    # This addresses the "Log traceback" requirement.
-    def handle_exception(exc_type, exc_value, exc_traceback):
-        if issubclass(exc_type, KeyboardInterrupt):
-            sys.__excepthook__(exc_type, exc_value, exc_traceback)
-            return
-        logging.getLogger().critical(
-            'Unhandled exception', exc_info=(exc_type, exc_value, exc_traceback)
+    file_handler = RotatingFileHandler(
+        log_file, maxBytes=max_bytes, backupCount=backup_count
+    )
+    file_handler.setFormatter(formatter)
+
+    global _listener
+    if async_logging:
+        log_queue = queue.Queue(-1)
+        queue_handler = QueueHandler(log_queue)
+        handlers = [queue_handler]
+        _listener = QueueListener(log_queue, console_handler, file_handler)
+        _listener.start()
+    else:
+        handlers = [console_handler, file_handler]
+
+    logging.basicConfig(level=log_level, handlers=handlers, force=True)
+
+    # Exclude or set log level for specified loggers
+    exclude_loggers = cfg.get('exclude_loggers', [])
+    for logger_name in exclude_loggers:
+        logger = logging.getLogger(logger_name)
+        logger.setLevel(logging.WARNING)
+        logger.propagate = (
+            False  # Prevents messages from being passed to the root logger
         )
 
-    sys.excepthook = handle_exception
 
+def stop_async_listener() -> None:
+    """Stop the async QueueListener if running (flushes all logs)."""
+    global _listener
+    if _listener is not None:
+        _listener.stop()
+        _listener = None
 
-def get_timestamped_log_path(log_dir: str, app_name: str) -> str:
-    """Generates a timestamped log file path.
 
-    :param log_dir: The directory where the log file should be stored.
-    :param app_name: The base name for the log file.
-    :return: A string representing the full path to the log file.
-    """
-    timestamp = datetime.now().strftime('%Y-%m-%d')
-    return f'{log_dir}/{app_name}_{timestamp}.log'
+def get_logger(name: str) -> logging.Logger:
+    """Get a logger for a given component/module.
 
+    Usage:
+        log = get_logger(__name__) or get_logger('my_component').
 
-def get_logger(name: str) -> logging.Logger:
-    """Returns a logger instance for the given name.
+    Args:
+        name (str): The logger name (usually __name__ or a component name).
 
-    This is the primary function that developers will use to get a
-    pre-configured logger within their modules.
+    Returns:
+        logging.Logger: The logger instance.
     """
     return logging.getLogger(name)