Merge pull request #244 from MITLibraries/TIMX-459-update-logging

TIMX 459 - update logging

Author: ghukill

Pipfile

@@ -5,7 +5,7 @@ name = "pypi"
 
 [packages]
 attrs = "*"
-beautifulsoup4 = "*"
+beautifulsoup4 = "==4.12.3"
 click = "*"
 jsonlines = "*"
 lxml = "*"

Pipfile.lock

@@ -53,6 +53,7 @@ WORKSPACE=### Set to `dev` for local development, this will be set to `stage` an
 
 ```shell
 ETL_VERSION=### Version number of the TIMDEX ETL infrastructure.  This can be used to align application behavior with the requirements of other applications in the TIMDEX ETL pipeline.
+WARNING_ONLY_LOGGERS=### Comma-seperated list of logger names to set as WARNING only, e.g. 'botocore,charset_normalizer,smart_open' 
 ```
 
 ## CLI commands

README.md

@@ -51,6 +51,12 @@ fixable = ["E", "F", "I", "Q"]
 [tool.ruff.lint.flake8-annotations]
 mypy-init-return = true
 
+[tool.ruff.lint.flake8-builtins]
+builtins-allowed-modules = [
+    "json", # allows for transmogrifier/sources/json directory
+    "xml", # allows for transmogrifier/sources/xml directory
+]
+
 [tool.ruff.lint.flake8-pytest-style]
 fixture-parentheses = false
 

pyproject.toml

@@ -66,7 +66,7 @@ def main(
 ) -> None:
     start_time = perf_counter()
     root_logger = logging.getLogger()
-    logger.info(configure_logger(root_logger, verbose))
+    logger.info(configure_logger(root_logger, verbose=verbose))
     logger.info(configure_sentry())
     logger.info("Running transform for source %s", source)
 

transmogrifier/cli.py

@@ -132,23 +132,40 @@
 }
 
 
-def configure_logger(logger: logging.Logger, verbose: bool) -> str:  # noqa: FBT001
+def configure_logger(
+    root_logger: logging.Logger,
+    *,
+    verbose: bool = False,
+    warning_only_loggers: str | None = None,
+) -> str:
+    """Configure application via passed application root logger.
+
+    If verbose=True, 3rd party libraries can be quite chatty.  For convenience, they can
+    be set to WARNING level by either passing a comma seperated list of logger names to
+    'warning_only_loggers' or by setting the env var WARNING_ONLY_LOGGERS.
+    """
     if verbose:
-        logging.basicConfig(
-            format="%(asctime)s %(levelname)s %(name)s.%(funcName)s() line %(lineno)d: "
-            "%(message)s",
+        root_logger.setLevel(logging.DEBUG)
+        logging_format = (
+            "%(asctime)s %(levelname)s %(name)s.%(funcName)s() "
+            "line %(lineno)d: %(message)s"
         )
-        logger.setLevel(logging.DEBUG)
-        for handler in logging.root.handlers:
-            handler.addFilter(logging.Filter("transmogrifier"))
     else:
-        logging.basicConfig(
-            format="%(asctime)s %(levelname)s %(name)s.%(funcName)s(): %(message)s"
-        )
-        logger.setLevel(logging.INFO)
+        root_logger.setLevel(logging.INFO)
+        logging_format = "%(asctime)s %(levelname)s %(name)s.%(funcName)s(): %(message)s"
+
+    warning_only_loggers = os.getenv("WARNING_ONLY_LOGGERS", warning_only_loggers)
+    if warning_only_loggers:
+        for name in warning_only_loggers.split(","):
+            logging.getLogger(name).setLevel(logging.WARNING)
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter(logging_format))
+    root_logger.addHandler(handler)
+
     return (
-        f"Logger '{logger.name}' configured with level="
-        f"{logging.getLevelName(logger.getEffectiveLevel())}"
+        f"Logger '{root_logger.name}' configured with level="
+        f"{logging.getLevelName(root_logger.getEffectiveLevel())}"
     )