Merge pull request #2733 from bcgov/DEP-161-mask-sensitive-info

DEP-161: Mask sensitive information in API logs

Author: NatSquared

CHANGELOG.MD

@@ -1,3 +1,12 @@
+## February 2, 2026
+
+- **Security** Implemented automatic sensitive data masking in API logs. [🎟️ DEP-161](https://citz-gdx.atlassian.net/browse/DEP-161)
+  - Added logging filter to automatically mask passwords, tokens, API keys, and database credentials
+  - Replaced print() statements with logger calls to ensure masking coverage
+  - Added linting rules (pylint + flake8) to prevent print statements and enforce logging
+  - Updated logging configuration to apply masking app-wide
+  - See `met_api.utils.logging_masker` for implementation details
+
 ## January 28, 2026
 
 - Added documentation for enabling and logging into the Sysdig monitoring platform for OpenShift projects. [🎟️ DEP-160](https://citz-gdx.atlassian.net/browse/DEP-160)

met-api/.pylintrc

@@ -33,7 +33,7 @@ ignore-patterns=^\.#
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+init-hook='import sys; sys.path.append(".")'
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
@@ -46,7 +46,8 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python module names) to load,
 # usually to register additional checkers.
-load-plugins=
+load-plugins=pylint_flask,
+             pylint_plugins.no_print_checker
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -90,7 +91,9 @@ disable=raw-checker-failed,
         suppressed-message,
         useless-suppression,
         deprecated-pragma,
-        use-symbolic-message-instead
+        use-symbolic-message-instead,
+        C0301,
+        W0511
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -192,7 +195,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=Error
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # class is considered mixin if its name matches the mixin-class-rgx option.
@@ -213,13 +216,13 @@ ignore-on-opaque-inference=yes
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local,scoped_session
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis). It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=flask_sqlalchemy,sqlalchemy,SQLAlchemy,alembic,scoped_session
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.
@@ -290,7 +293,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=120
 
 # Maximum number of lines in a module.
 max-module-lines=1000
@@ -319,7 +322,7 @@ ignore-imports=no
 ignore-signatures=no
 
 # Minimum lines number of a similarity.
-min-similarity-lines=4
+min-similarity-lines=15
 
 
 [STRING]
@@ -574,7 +577,7 @@ max-returns=6
 max-statements=50
 
 # Minimum number of public methods for a class (see R0903).
-min-public-methods=2
+min-public-methods=1
 
 
 [EXCEPTIONS]

met-api/Makefile

@@ -68,7 +68,7 @@ install-dev: ## Install local application
 ci: lint flake8 test ## CI flow
 
 pylint: ## Linting with pylint
-	. venv/bin/activate && pylint --rcfile=setup.cfg  src/$(PROJECT_NAME)
+	. venv/bin/activate && pylint --rcfile=.pylintrc src/$(PROJECT_NAME)
 
 flake8: ## Linting with flake8
 	. venv/bin/activate && flake8 src/$(PROJECT_NAME) tests

met-api/pylint_plugins/README.md

@@ -0,0 +1,59 @@
+# Pylint Plugins
+
+This directory contains custom Pylint checkers for the MET API.
+
+## no_print_checker
+
+**Purpose:** Prevents use of `print()` statements which bypass sensitive data masking.
+
+**Why:** Print statements output directly to stdout/stderr and bypass the logging system's `SensitiveDataFilter`. This creates a security risk where passwords, tokens, API keys, and other credentials could be exposed in logs.
+
+**What it does:**
+- Flags any use of `print()` function with error code **W9001**
+- Exception: Allows print in `config.py` for startup messages before logging is initialized
+
+**Usage:**
+Instead of:
+```python
+print(f'User ID: {user_id}')
+print(f'Database URL: {db_url}')
+```
+
+Use:
+```python
+from flask import current_app
+current_app.logger.info('User ID: %s', user_id)
+
+# or
+import logging
+logger = logging.getLogger(__name__)
+logger.info('Database URL: %s', db_url)
+```
+
+**Configuration:**
+The checker is automatically loaded via `.pylintrc`:
+```ini
+load-plugins=pylint_plugins.no_print_checker
+```
+
+**Testing:**
+```bash
+# Test with a file containing print statements
+pylint --rcfile=.pylintrc path/to/file.py
+
+# Should show:
+# W9001: Print statement found - use logging to ensure sensitive data masking
+```
+
+## Adding New Checkers
+
+1. Create a new file in this directory (e.g., `my_checker.py`)
+2. Implement a class that inherits from `pylint.checkers.BaseChecker`
+3. Define your messages and visit methods
+4. Add a `register()` function
+5. Update `.pylintrc` to load your plugin:
+   ```ini
+   load-plugins=pylint_plugins.no_print_checker,pylint_plugins.my_checker
+   ```
+
+See [Pylint documentation](https://pylint.readthedocs.io/en/latest/development_guide/how_tos/custom_checkers.html) for more details.

met-api/pylint_plugins/__init__.py

@@ -0,0 +1 @@
+"""Pylint plugins for MET API code quality checks."""

met-api/pylint_plugins/no_print_checker.py

@@ -0,0 +1,38 @@
+"""Pylint plugin to check for print statements.
+
+Print statements bypass logging masking and pose a security risk by potentially
+exposing sensitive data (passwords, tokens, API keys, etc.) in logs.
+
+Use current_app.logger or logging.getLogger(__name__) instead.
+"""
+
+from astroid import nodes
+from pylint.checkers import BaseChecker
+
+
+class NoPrintChecker(BaseChecker):
+    """Check for print statements which bypass logging masking."""
+
+    name = "no-print"
+    msgs = {
+        "W9001": (
+            "Print statement found - use logging to ensure sensitive data masking",
+            "print-statement-found",
+            "Print statements bypass the logging masking system and may expose "
+            "sensitive data (passwords, tokens, API keys). Use current_app.logger "
+            "or logging.getLogger(__name__) instead.",
+        ),
+    }
+
+    def visit_call(self, node: nodes.Call) -> None:
+        """Check if the node is a print function call."""
+        if isinstance(node.func, nodes.Name) and node.func.name == "print":
+            # Allow print in config.py for startup messages before logging is initialized
+            if node.root().file and "config.py" in node.root().file:
+                return
+            self.add_message("print-statement-found", node=node)
+
+
+def register(linter):
+    """Register the checker."""
+    linter.register_checker(NoPrintChecker(linter))

met-api/setup.cfg

@@ -38,9 +38,10 @@ ignore = I001, I003, I004, I100, I101, I201, I202, E126, W504
 exclude = .git,*migrations*
 max-line-length = 120
 docstring-min-length=10
+# Disable print statement warning in config file - logger is not initialized yet
 per-file-ignores =
     */__init__.py:F401
-    */config.py:N802
+    */config.py:N802,T201
 
 [pycodestyle]
 max-line-length = 120

met-api/src/met_api/__init__.py

@@ -3,7 +3,9 @@
 This module is for the initiation of the flask app.
 """
 
+import builtins
 import os
+import warnings
 
 import secure
 from flask import Flask, current_app, g, request
@@ -18,6 +20,38 @@
 from met_api.utils import constants
 from met_api.utils.cache import cache
 from met_api.utils.roles import Role
+from met_api.utils.logging_masker import setup_logging_masking
+
+
+# Store reference to original print function
+_original_print = builtins.print
+
+
+class PrintDeprecationWarning(DeprecationWarning):
+    """Custom warning category for deprecated print() usage."""
+
+
+def _deprecated_print(*args, **kwargs):
+    """
+    Issue a deprecation warning when print() is used.
+
+    Using print() bypasses the logging masking system and may expose sensitive data.
+    Use logging instead: current_app.logger.info(), logger.debug(), etc.
+    """
+    warnings.warn(
+        'print() is deprecated in this application. '
+        'Use logging instead (e.g., current_app.logger.info()) to ensure '
+        'sensitive data is automatically masked. '
+        'See met_api/utils/logging_masker.py for details.',
+        PrintDeprecationWarning,
+        stacklevel=2
+    )
+    # Still call original print to not break existing code during transition
+    _original_print(*args, **kwargs)
+
+
+# Replace built-in print with deprecated version
+builtins.print = _deprecated_print
 
 # Security Response headers
 csp = (
@@ -53,6 +87,17 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'development')):
     # Configure app from config.py
     app.config.from_object(get_named_config(run_mode))
 
+    # Replace built-in print with deprecated version AFTER config loads
+    # (config.py prints during initialization before logging is set up)
+    warnings.filterwarnings('default', category=PrintDeprecationWarning)
+    builtins.print = _deprecated_print
+
+    # Set up logging with sensitive data masking
+    # Automatically masks passwords, tokens, API keys, database credentials
+    # Applied to both Flask app logger and root logger for full coverage
+    setup_logging_masking(app.logger)
+    setup_logging_masking()
+
     CORS(app, origins=app.config['CORS_ORIGINS'], supports_credentials=True)
 
     # Register blueprints

met-api/src/met_api/config.py

@@ -85,9 +85,6 @@ def __init__(self) -> None:
         Performs more advanced configuration logic that is not possible
         in the normal class definition.
         """
-        # If extending this class, call super().__init__() in your constructor.
-        print(f'SQLAlchemy URL: {self.SQLALCHEMY_DATABASE_URI}')
-
         # apply configs to _Config in the format that flask_jwt_oidc expects
         # this flattens the JWT_CONFIG dict into individual attributes
         for key, value in self.JWT_CONFIG.items():

met-api/src/met_api/logging.conf

@@ -7,6 +7,11 @@ keys=console
 [formatters]
 keys=simple
 
+[filters]
+# Sensitive data filter automatically masks credentials in logs
+# See met_api.utils.logging_masker for pattern details
+keys=sensitive_data
+
 [logger_root]
 level=DEBUG
 handlers=console
@@ -28,7 +33,11 @@ class=StreamHandler
 level=DEBUG
 formatter=simple
 args=(sys.stdout,)
+filters=sensitive_data # All other loggers inherit this filter via root logger's console handler
 
 [formatter_simple]
 format=%(asctime)s - %(name)s - %(levelname)s in %(module)s:%(filename)s:%(lineno)d - %(funcName)s: %(message)s
-datefmt=
+datefmt=
+
+[filter_sensitive_data]
+class=met_api.utils.logging_masker.SensitiveDataFilter