From 9c08d1fb1cb527abf688a4258fc430e392ed2711 Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Thu, 10 Jul 2025 19:17:35 +0200 Subject: [PATCH 1/7] chore(logging): Refacto get_global_file_logger --- airbyte/logs.py | 81 ++++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/airbyte/logs.py b/airbyte/logs.py index cef871f0..4d9372b7 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -131,42 +131,20 @@ def get_global_file_logger() -> logging.Logger | None: logger.setLevel(logging.INFO) logger.propagate = False - if AIRBYTE_LOGGING_ROOT is None: - # No temp directory available, so return None + handlers = _get_global_handlers() + if len(handlers) == 0: return None - # Else, configure the logger to write to a file - # Remove any existing handlers for handler in logger.handlers: logger.removeHandler(handler) - yyyy_mm_dd: str = ab_datetime_now().strftime("%Y-%m-%d") - folder = AIRBYTE_LOGGING_ROOT / yyyy_mm_dd - try: - folder.mkdir(parents=True, exist_ok=True) - except Exception: - warn_once( - f"Failed to create logging directory at '{folder!s}'.", - with_stack=False, - ) - return None - - logfile_path = folder / f"airbyte-log-{str(ulid.ULID())[2:11]}.log" - print(f"Writing PyAirbyte logs to file: {logfile_path!s}") - - file_handler = logging.FileHandler( - filename=logfile_path, - encoding="utf-8", - ) - if AIRBYTE_STRUCTURED_LOGGING: - # Create a formatter and set it for the handler + # Create a formatter and set it for the handlers formatter = logging.Formatter("%(message)s") - file_handler.setFormatter(formatter) - - # Add the file handler to the logger - logger.addHandler(file_handler) + for handler in handlers: + handler.setFormatter(formatter) + logger.addHandler(handler) # Configure structlog structlog.configure( @@ -187,15 +165,15 @@ def get_global_file_logger() -> logging.Logger | None: # Create a logger return structlog.get_logger("airbyte") - # Create and configure file handler - file_handler.setFormatter( - logging.Formatter( - fmt="%(asctime)s - %(levelname)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) + # Configure handlers + formatter = logging.Formatter( + fmt="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", ) + for handler in handlers: + handler.setFormatter(formatter) + logger.addHandler(handler) - logger.addHandler(file_handler) return logger @@ -344,3 +322,36 @@ def new_passthrough_file_logger(connector_name: str) -> logging.Logger: logger.addHandler(file_handler) return logger + + +def _get_global_handlers() -> list[logging.Handler]: + handlers: list[logging.Handler] = [] + handlers.append(_get_global_file_handler()) + + return [h for h in handlers if h is not None] + + +def _get_global_file_handler() -> logging.FileHandler | None: + if AIRBYTE_LOGGING_ROOT is None: + # No temp directory available, so return None + return None + + yyyy_mm_dd: str = ab_datetime_now().strftime("%Y-%m-%d") + folder = AIRBYTE_LOGGING_ROOT / yyyy_mm_dd + try: + folder.mkdir(parents=True, exist_ok=True) + except Exception: + warn_once( + f"Failed to create logging directory at '{folder!s}'.", + with_stack=False, + ) + return None + + logfile_path = folder / f"airbyte-log-{str(ulid.ULID())[2:11]}.log" + print(f"Writing PyAirbyte logs to file: {logfile_path!s}") + + return logging.FileHandler( + filename=logfile_path, + encoding="utf-8", + ) + From 05e836b3ed890bcd7449a5b386ceac44a9663ab8 Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Thu, 10 Jul 2025 19:18:42 +0200 Subject: [PATCH 2/7] feat(logging): Support multiple logging behavior for get_global_file_logger --- airbyte/logs.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/airbyte/logs.py b/airbyte/logs.py index 4d9372b7..01facd33 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -7,13 +7,21 @@ PyAirbyte supports structured JSON logging, which is disabled by default. To enable structured logging in JSON, set `AIRBYTE_STRUCTURED_LOGGING` to `True`. + +PyAirbyte supports different logging behaviors controlled by the `AIRBYTE_LOGGING_BEHAVIOR` +environment variable: +- `FILE_ONLY`: Write logs only to files (default) +- `CONSOLE_ONLY`: Write logs only to stdout/stderr +- `FILE_AND_CONSOLE`: Write logs to both files and stdout/stderr """ from __future__ import annotations +import enum import logging import os import platform +import sys import tempfile import warnings from functools import lru_cache @@ -25,11 +33,28 @@ from airbyte_cdk.utils.datetime_helpers import ab_datetime_now +class LoggingBehavior(enum.Enum): + """Enumeration for PyAirbyte logging behavior.""" + + FILE_ONLY = "FILE_ONLY" + CONSOLE_ONLY = "CONSOLE_ONLY" + FILE_AND_CONSOLE = "FILE_AND_CONSOLE" + + def _str_to_bool(value: str) -> bool: """Convert a string value of an environment values to a boolean value.""" return bool(value) and value.lower() not in {"", "0", "false", "f", "no", "n", "off"} +def _parse_logging_behavior(value: str) -> LoggingBehavior: + """Parse logging behavior from environment variable string.""" + default_logging_behavior = LoggingBehavior.FILE_ONLY + try: + return LoggingBehavior(value.upper()) + except ValueError: + return default_logging_behavior + + AIRBYTE_STRUCTURED_LOGGING: bool = _str_to_bool( os.getenv( key="AIRBYTE_STRUCTURED_LOGGING", @@ -42,6 +67,22 @@ def _str_to_bool(value: str) -> bool: not set, the default value is `False`. """ +AIRBYTE_LOGGING_BEHAVIOR: LoggingBehavior = _parse_logging_behavior( + os.getenv( + key="AIRBYTE_LOGGING_BEHAVIOR", + default="FILE_ONLY", + ) +) +"""The logging behavior for PyAirbyte. + +This value is read from the `AIRBYTE_LOGGING_BEHAVIOR` environment variable. Valid values are: +- `FILE_ONLY`: Write logs only to files (default) +- `CONSOLE_ONLY`: Write logs only to stdout/stderr +- `FILE_AND_CONSOLE`: Write logs to both files and stdout/stderr + +If an invalid value is provided, the default `FILE_ONLY` behavior is used. +""" + _warned_messages: set[str] = set() @@ -125,7 +166,7 @@ def _get_logging_root() -> Path | None: def get_global_file_logger() -> logging.Logger | None: """Return the global logger for PyAirbyte. - This logger is configured to write logs to the console and to a file in the log directory. + This logger is configured based on the AIRBYTE_LOGGING_BEHAVIOR setting. """ logger = logging.getLogger("airbyte") logger.setLevel(logging.INFO) @@ -325,8 +366,14 @@ def new_passthrough_file_logger(connector_name: str) -> logging.Logger: def _get_global_handlers() -> list[logging.Handler]: - handlers: list[logging.Handler] = [] - handlers.append(_get_global_file_handler()) + handlers: list[logging.Handler | None] = [] + match AIRBYTE_LOGGING_BEHAVIOR: + case LoggingBehavior.FILE_ONLY: + handlers.append(_get_global_file_handler()) + case LoggingBehavior.CONSOLE_ONLY: + handlers.append(_get_console_handler()) + case LoggingBehavior.FILE_AND_CONSOLE: + handlers.extend([_get_global_file_handler(), _get_console_handler()]) return [h for h in handlers if h is not None] @@ -355,3 +402,6 @@ def _get_global_file_handler() -> logging.FileHandler | None: encoding="utf-8", ) + +def _get_console_handler() -> logging.StreamHandler: + return logging.StreamHandler(sys.stdout) From 7803c29b7a19ba58cff865198a6de10d28a71922 Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Thu, 10 Jul 2025 19:25:30 +0200 Subject: [PATCH 3/7] feat(logging): Support multiple logging behavior for get_global_stats_logger --- airbyte/logs.py | 53 +++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/airbyte/logs.py b/airbyte/logs.py index 01facd33..a536b129 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -256,38 +256,19 @@ def get_global_stats_logger() -> structlog.BoundLogger: cache_logger_on_first_use=True, ) - logfile_path: Path | None = get_global_stats_log_path() - if AIRBYTE_LOGGING_ROOT is None or logfile_path is None: - # No temp directory available, so return no-op logger without handlers + handlers = _get_global_stats_handlers() + if len(handlers) == 0: return structlog.get_logger("airbyte.stats") - print(f"Writing PyAirbyte performance stats to file: {logfile_path!s}") - # Remove any existing handlers for handler in logger.handlers: logger.removeHandler(handler) - folder = AIRBYTE_LOGGING_ROOT - try: - folder.mkdir(parents=True, exist_ok=True) - except Exception: - warn_once( - f"Failed to create logging directory at '{folder!s}'.", - with_stack=False, - ) - return structlog.get_logger("airbyte.stats") - - file_handler = logging.FileHandler( - filename=logfile_path, - encoding="utf-8", - ) - # Create a formatter and set it for the handler formatter = logging.Formatter("%(message)s") - file_handler.setFormatter(formatter) - - # Add the file handler to the logger - logger.addHandler(file_handler) + for handler in handlers: + handler.setFormatter(formatter) + logger.addHandler(handler) # Create a logger return structlog.get_logger("airbyte.stats") @@ -378,6 +359,18 @@ def _get_global_handlers() -> list[logging.Handler]: return [h for h in handlers if h is not None] +def _get_global_stats_handlers() -> list[logging.Handler]: + handlers: list[logging.Handler | None] = [] + match AIRBYTE_LOGGING_BEHAVIOR: + case LoggingBehavior.FILE_ONLY: + handlers.append(_get_global_stats_file_handler()) + case LoggingBehavior.CONSOLE_ONLY: + handlers.append(_get_console_handler()) + case LoggingBehavior.FILE_AND_CONSOLE: + handlers.extend([_get_global_stats_file_handler(), _get_console_handler()]) + return [h for h in handlers if h is not None] + + def _get_global_file_handler() -> logging.FileHandler | None: if AIRBYTE_LOGGING_ROOT is None: # No temp directory available, so return None @@ -403,5 +396,17 @@ def _get_global_file_handler() -> logging.FileHandler | None: ) +def _get_global_stats_file_handler() -> logging.FileHandler | None: + logfile_path: Path | None = get_global_stats_log_path() + if AIRBYTE_LOGGING_ROOT is None or logfile_path is None: + return None + + print(f"Writing PyAirbyte performance stats to file: {logfile_path!s}") + return logging.FileHandler( + filename=logfile_path, + encoding="utf-8", + ) + + def _get_console_handler() -> logging.StreamHandler: return logging.StreamHandler(sys.stdout) From f8d41828eb7b4e39cd366cacd9d2c8dbbb84324b Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Thu, 10 Jul 2025 19:30:52 +0200 Subject: [PATCH 4/7] chore(logging): move function get_global_stats_log_path --- airbyte/logs.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/airbyte/logs.py b/airbyte/logs.py index a536b129..1865706f 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -218,24 +218,6 @@ def get_global_file_logger() -> logging.Logger | None: return logger -def get_global_stats_log_path() -> Path | None: - """Return the path to the performance log file.""" - if AIRBYTE_LOGGING_ROOT is None: - return None - - folder = AIRBYTE_LOGGING_ROOT - try: - folder.mkdir(parents=True, exist_ok=True) - except Exception: - warn_once( - f"Failed to create logging directory at '{folder!s}'.", - with_stack=False, - ) - return None - - return folder / "airbyte-stats.log" - - @lru_cache def get_global_stats_logger() -> structlog.BoundLogger: """Create a stats logger for performance metrics.""" @@ -410,3 +392,21 @@ def _get_global_stats_file_handler() -> logging.FileHandler | None: def _get_console_handler() -> logging.StreamHandler: return logging.StreamHandler(sys.stdout) + + +def get_global_stats_log_path() -> Path | None: + """Return the path to the performance log file.""" + if AIRBYTE_LOGGING_ROOT is None: + return None + + folder = AIRBYTE_LOGGING_ROOT + try: + folder.mkdir(parents=True, exist_ok=True) + except Exception: + warn_once( + f"Failed to create logging directory at '{folder!s}'.", + with_stack=False, + ) + return None + + return folder / "airbyte-stats.log" From f8f276366acb4a6084850b221fea231848ca26fa Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Thu, 10 Jul 2025 19:37:41 +0200 Subject: [PATCH 5/7] feat(logging): Support multiple logging behavior for new_passthrough_file_logger --- airbyte/logs.py | 73 +++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/airbyte/logs.py b/airbyte/logs.py index 1865706f..1220cbd6 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -264,37 +264,20 @@ def new_passthrough_file_logger(connector_name: str) -> logging.Logger: # Prevent logging to stderr by stopping propagation to the root logger logger.propagate = False - if AIRBYTE_LOGGING_ROOT is None: - # No temp directory available, so return a basic logger + handlers = _get_passthrough_handlers(connector_name) + if len(handlers) == 0: return logger - # Else, configure the logger to write to a file - # Remove any existing handlers for handler in logger.handlers: logger.removeHandler(handler) - folder = AIRBYTE_LOGGING_ROOT / connector_name - folder.mkdir(parents=True, exist_ok=True) - - # Create a file handler - global_logger = get_global_file_logger() - logfile_path = folder / f"{connector_name}-log-{str(ulid.ULID())[2:11]}.log" - logfile_msg = f"Writing `{connector_name}` logs to file: {logfile_path!s}" - print(logfile_msg) - if global_logger: - global_logger.info(logfile_msg) - - file_handler = logging.FileHandler(logfile_path) - file_handler.setLevel(logging.INFO) - if AIRBYTE_STRUCTURED_LOGGING: # Create a formatter and set it for the handler formatter = logging.Formatter("%(message)s") - file_handler.setFormatter(formatter) - - # Add the file handler to the logger - logger.addHandler(file_handler) + for handler in handlers: + handler.setFormatter(formatter) + logger.addHandler(handler) # Configure structlog structlog.configure( @@ -316,15 +299,14 @@ def new_passthrough_file_logger(connector_name: str) -> logging.Logger: return structlog.get_logger(f"airbyte.{connector_name}") # Else, write logs in plain text - - file_handler.setFormatter( - logging.Formatter( - fmt="%(asctime)s - %(levelname)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) + formatter = logging.Formatter( + fmt="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", ) + for handler in handlers: + handler.setFormatter(formatter) + logger.addHandler(handler) - logger.addHandler(file_handler) return logger @@ -353,6 +335,18 @@ def _get_global_stats_handlers() -> list[logging.Handler]: return [h for h in handlers if h is not None] +def _get_passthrough_handlers(connector_name: str) -> list[logging.Handler]: + handlers: list[logging.Handler | None] = [] + match AIRBYTE_LOGGING_BEHAVIOR: + case LoggingBehavior.FILE_ONLY: + handlers.append(_get_passthrough_file_handler(connector_name)) + case LoggingBehavior.CONSOLE_ONLY: + handlers.append(_get_console_handler()) + case LoggingBehavior.FILE_AND_CONSOLE: + handlers.extend([_get_passthrough_file_handler(connector_name), _get_console_handler()]) + return [h for h in handlers if h is not None] + + def _get_global_file_handler() -> logging.FileHandler | None: if AIRBYTE_LOGGING_ROOT is None: # No temp directory available, so return None @@ -390,6 +384,27 @@ def _get_global_stats_file_handler() -> logging.FileHandler | None: ) +def _get_passthrough_file_handler(connector_name: str) -> logging.FileHandler | None: + if AIRBYTE_LOGGING_ROOT is None: + return None + + folder = AIRBYTE_LOGGING_ROOT / connector_name + folder.mkdir(parents=True, exist_ok=True) + + # Create a file handler + global_logger = get_global_file_logger() + logfile_path = folder / f"{connector_name}-log-{str(ulid.ULID())[2:11]}.log" + logfile_msg = f"Writing `{connector_name}` logs to file: {logfile_path!s}" + print(logfile_msg) + if global_logger: + global_logger.info(logfile_msg) + + file_handler = logging.FileHandler(logfile_path) + file_handler.setLevel(logging.INFO) + + return file_handler + + def _get_console_handler() -> logging.StreamHandler: return logging.StreamHandler(sys.stdout) From ceab0268306fe3003cb1e6c70b10721215b5e5d0 Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Thu, 10 Jul 2025 23:42:04 +0200 Subject: [PATCH 6/7] fix(logging): remove all handlers properly --- airbyte/logs.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/airbyte/logs.py b/airbyte/logs.py index 1220cbd6..199f769e 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -176,9 +176,8 @@ def get_global_file_logger() -> logging.Logger | None: if len(handlers) == 0: return None - # Remove any existing handlers - for handler in logger.handlers: - logger.removeHandler(handler) + # We are going to set our own handlers. + _remove_all_handlers(logger) if AIRBYTE_STRUCTURED_LOGGING: # Create a formatter and set it for the handlers @@ -242,9 +241,8 @@ def get_global_stats_logger() -> structlog.BoundLogger: if len(handlers) == 0: return structlog.get_logger("airbyte.stats") - # Remove any existing handlers - for handler in logger.handlers: - logger.removeHandler(handler) + # We are going to set our own handlers. + _remove_all_handlers(logger) # Create a formatter and set it for the handler formatter = logging.Formatter("%(message)s") @@ -268,9 +266,8 @@ def new_passthrough_file_logger(connector_name: str) -> logging.Logger: if len(handlers) == 0: return logger - # Remove any existing handlers - for handler in logger.handlers: - logger.removeHandler(handler) + # We are going to set our own handlers. + _remove_all_handlers(logger) if AIRBYTE_STRUCTURED_LOGGING: # Create a formatter and set it for the handler @@ -425,3 +422,8 @@ def get_global_stats_log_path() -> Path | None: return None return folder / "airbyte-stats.log" + + +def _remove_all_handlers(logger: logging.Logger) -> None: + """Remove all handlers from a logger.""" + logger.handlers.clear() From 752671c1b5a4bf1853b19ed6be02197d65404d7f Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Thu, 10 Jul 2025 23:42:22 +0200 Subject: [PATCH 7/7] chore(logging): add unit tests --- tests/unit_tests/test_logs.py | 468 ++++++++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100644 tests/unit_tests/test_logs.py diff --git a/tests/unit_tests/test_logs.py b/tests/unit_tests/test_logs.py new file mode 100644 index 00000000..44098a05 --- /dev/null +++ b/tests/unit_tests/test_logs.py @@ -0,0 +1,468 @@ +from __future__ import annotations + +import logging +import warnings + +import pytest +from unittest.mock import patch + +from airbyte.logs import ( + LoggingBehavior, + _parse_logging_behavior, + get_global_file_logger, + get_global_stats_logger, + new_passthrough_file_logger, +) + +connector_name = "test_connector" + + +def clear_logger_caches(): + get_global_file_logger.cache_clear() + get_global_stats_logger.cache_clear() + logging.getLogger("airbyte").handlers.clear() + logging.getLogger("airbyte.stats").handlers.clear() + logging.getLogger(f"airbyte.{connector_name}").handlers.clear() + + +class TestParseLoggingBehavior: + @pytest.mark.parametrize( + "input_value,expected", + [ + ("FILE_ONLY", LoggingBehavior.FILE_ONLY), + ("file_only", LoggingBehavior.FILE_ONLY), + ("File_Only", LoggingBehavior.FILE_ONLY), + ("CONSOLE_ONLY", LoggingBehavior.CONSOLE_ONLY), + ("console_only", LoggingBehavior.CONSOLE_ONLY), + ("Console_Only", LoggingBehavior.CONSOLE_ONLY), + ("FILE_AND_CONSOLE", LoggingBehavior.FILE_AND_CONSOLE), + ("file_and_console", LoggingBehavior.FILE_AND_CONSOLE), + ("File_And_Console", LoggingBehavior.FILE_AND_CONSOLE), + ("", LoggingBehavior.FILE_ONLY), + ("FILE", LoggingBehavior.FILE_ONLY), + ("INVALID", LoggingBehavior.FILE_ONLY), + ("CONSOLE", LoggingBehavior.FILE_ONLY), + ("FILE_ONLY_EXTRA", LoggingBehavior.FILE_ONLY), + ], + ) + def test_logging_behavior_parsing(self, input_value, expected): + result = _parse_logging_behavior(input_value) + assert result == expected + + +class TestGetGlobalFileLogger: + """Test the get_global_file_logger function.""" + + def setup_method(self): + clear_logger_caches() + + def test_logger_creation_with_defaults(self): + with ( + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = get_global_file_logger() + + assert logger is not None + assert logger.name == "airbyte" + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], logging.FileHandler) + assert ( + logger.handlers[0].formatter._fmt + == "%(asctime)s - %(levelname)s - %(message)s" + ) + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,formatter_fmt,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, "%(message)s", [logging.FileHandler]), + ( + LoggingBehavior.FILE_ONLY, + False, + "%(asctime)s - %(levelname)s - %(message)s", + [logging.FileHandler], + ), + ( + LoggingBehavior.CONSOLE_ONLY, + True, + "%(message)s", + [logging.StreamHandler], + ), + ( + LoggingBehavior.CONSOLE_ONLY, + False, + "%(asctime)s - %(levelname)s - %(message)s", + [logging.StreamHandler], + ), + ( + LoggingBehavior.FILE_AND_CONSOLE, + True, + "%(message)s", + [logging.FileHandler, logging.StreamHandler], + ), + ( + LoggingBehavior.FILE_AND_CONSOLE, + False, + "%(asctime)s - %(levelname)s - %(message)s", + [logging.FileHandler, logging.StreamHandler], + ), + ], + ) + def test_logger_creation_with_structured_logging( + self, + airbyte_logging_behavior, + airbyte_structured_logging, + formatter_fmt, + handlers, + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = get_global_file_logger() + assert logger is not None + assert logger.name == "airbyte" + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + assert logger.handlers[idx].formatter._fmt == formatter_fmt + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, []), + (LoggingBehavior.FILE_ONLY, False, []), + (LoggingBehavior.CONSOLE_ONLY, True, [logging.StreamHandler]), + (LoggingBehavior.CONSOLE_ONLY, False, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, True, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, False, [logging.StreamHandler]), + ], + ) + def test_directory_creation_failure( + self, airbyte_logging_behavior, airbyte_structured_logging, handlers + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("pathlib.Path.mkdir", side_effect=OSError("Permission denied")), + patch("pathlib.Path.exists", return_value=False), + warnings.catch_warnings(), + ): + warnings.simplefilter("ignore", UserWarning) + logger = get_global_file_logger() + if len(handlers) == 0: + assert logger is None + else: + assert logger is not None + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, []), + (LoggingBehavior.FILE_ONLY, False, []), + (LoggingBehavior.CONSOLE_ONLY, True, [logging.StreamHandler]), + (LoggingBehavior.CONSOLE_ONLY, False, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, True, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, False, [logging.StreamHandler]), + ], + ) + def test_logger_creation_with_logging_root_none( + self, airbyte_logging_behavior, airbyte_structured_logging, handlers + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("airbyte.logs.AIRBYTE_LOGGING_ROOT", None), + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = get_global_file_logger() + if len(handlers) == 0: + assert logger is None + else: + assert logger is not None + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + + +class TestGetGlobalStatsLogger: + """Test the get_global_stats_logger function.""" + + expected_fmt = "%(message)s" + + def setup_method(self): + clear_logger_caches() + + def test_logger_creation_with_defaults(self): + with ( + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = get_global_stats_logger() + + assert logger is not None + assert logger.name == "airbyte.stats" + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], logging.FileHandler) + assert logger.handlers[0].formatter._fmt == self.expected_fmt + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, [logging.FileHandler]), + (LoggingBehavior.FILE_ONLY, False, [logging.FileHandler]), + (LoggingBehavior.CONSOLE_ONLY, True, [logging.StreamHandler]), + (LoggingBehavior.CONSOLE_ONLY, False, [logging.StreamHandler]), + ( + LoggingBehavior.FILE_AND_CONSOLE, + True, + [logging.FileHandler, logging.StreamHandler], + ), + ( + LoggingBehavior.FILE_AND_CONSOLE, + False, + [logging.FileHandler, logging.StreamHandler], + ), + ], + ) + def test_logger_creation_with_structured_logging( + self, airbyte_logging_behavior, airbyte_structured_logging, handlers + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = get_global_stats_logger() + assert logger is not None + assert logger.name == "airbyte.stats" + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + assert logger.handlers[idx].formatter._fmt == self.expected_fmt + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, []), + (LoggingBehavior.FILE_ONLY, False, []), + (LoggingBehavior.CONSOLE_ONLY, True, [logging.StreamHandler]), + (LoggingBehavior.CONSOLE_ONLY, False, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, True, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, False, [logging.StreamHandler]), + ], + ) + def test_directory_creation_failure( + self, airbyte_logging_behavior, airbyte_structured_logging, handlers + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("pathlib.Path.mkdir", side_effect=OSError("Permission denied")), + patch("pathlib.Path.exists", return_value=False), + warnings.catch_warnings(), + ): + warnings.simplefilter("ignore", UserWarning) + logger = get_global_stats_logger() + assert logger is not None + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, []), + (LoggingBehavior.FILE_ONLY, False, []), + (LoggingBehavior.CONSOLE_ONLY, True, [logging.StreamHandler]), + (LoggingBehavior.CONSOLE_ONLY, False, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, True, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, False, [logging.StreamHandler]), + ], + ) + def test_logger_creation_with_logging_root_none( + self, airbyte_logging_behavior, airbyte_structured_logging, handlers + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("airbyte.logs.AIRBYTE_LOGGING_ROOT", None), + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = get_global_stats_logger() + assert logger is not None + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + + +class TestNewPassthroughFileLogger: + """Test the new_passthrough_file_logger function.""" + + def setup_method(self): + clear_logger_caches() + + def test_logger_creation_with_defaults(self): + with ( + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = new_passthrough_file_logger(connector_name) + + assert logger is not None + assert logger.name == f"airbyte.{connector_name}" + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], logging.FileHandler) + assert ( + logger.handlers[0].formatter._fmt + == "%(asctime)s - %(levelname)s - %(message)s" + ) + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,formatter_fmt,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, "%(message)s", [logging.FileHandler]), + ( + LoggingBehavior.FILE_ONLY, + False, + "%(asctime)s - %(levelname)s - %(message)s", + [logging.FileHandler], + ), + ( + LoggingBehavior.CONSOLE_ONLY, + True, + "%(message)s", + [logging.StreamHandler], + ), + ( + LoggingBehavior.CONSOLE_ONLY, + False, + "%(asctime)s - %(levelname)s - %(message)s", + [logging.StreamHandler], + ), + ( + LoggingBehavior.FILE_AND_CONSOLE, + True, + "%(message)s", + [logging.FileHandler, logging.StreamHandler], + ), + ( + LoggingBehavior.FILE_AND_CONSOLE, + False, + "%(asctime)s - %(levelname)s - %(message)s", + [logging.FileHandler, logging.StreamHandler], + ), + ], + ) + def test_logger_creation_with_structured_logging( + self, + airbyte_logging_behavior, + airbyte_structured_logging, + formatter_fmt, + handlers, + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = new_passthrough_file_logger(connector_name) + assert logger is not None + assert logger.name == f"airbyte.{connector_name}" + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + assert logger.handlers[idx].formatter._fmt == formatter_fmt + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, None), + (LoggingBehavior.FILE_ONLY, False, None), + (LoggingBehavior.CONSOLE_ONLY, True, [logging.StreamHandler]), + (LoggingBehavior.CONSOLE_ONLY, False, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, True, None), + (LoggingBehavior.FILE_AND_CONSOLE, False, None), + ], + ) + def test_directory_creation_failure( + self, airbyte_logging_behavior, airbyte_structured_logging, handlers + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("pathlib.Path.mkdir", side_effect=OSError("Permission denied")), + patch("pathlib.Path.exists", return_value=False), + ): + if handlers is None: + with pytest.raises(OSError): + new_passthrough_file_logger(connector_name) + else: + logger = new_passthrough_file_logger(connector_name) + assert logger is not None + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type) + + @pytest.mark.parametrize( + "airbyte_logging_behavior,airbyte_structured_logging,handlers", + [ + (LoggingBehavior.FILE_ONLY, True, []), + (LoggingBehavior.FILE_ONLY, False, []), + (LoggingBehavior.CONSOLE_ONLY, True, [logging.StreamHandler]), + (LoggingBehavior.CONSOLE_ONLY, False, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, True, [logging.StreamHandler]), + (LoggingBehavior.FILE_AND_CONSOLE, False, [logging.StreamHandler]), + ], + ) + def test_logger_creation_with_logging_root_none( + self, airbyte_logging_behavior, airbyte_structured_logging, handlers + ): + with ( + patch("airbyte.logs.AIRBYTE_LOGGING_BEHAVIOR", airbyte_logging_behavior), + patch( + "airbyte.logs.AIRBYTE_STRUCTURED_LOGGING", airbyte_structured_logging + ), + patch("airbyte.logs.AIRBYTE_LOGGING_ROOT", None), + patch("pathlib.Path.mkdir"), + patch("pathlib.Path.exists", return_value=True), + patch("builtins.open", create=True), + ): + logger = new_passthrough_file_logger(connector_name) + assert logger is not None + assert len(logger.handlers) == len(handlers) + for idx, handler_type in enumerate(handlers): + assert isinstance(logger.handlers[idx], handler_type)