diff --git a/CHANGELOG.MD b/CHANGELOG.MD index a23d15ba5..af7660181 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,3 +1,12 @@ +## February 2, 2026 + +- **Security** Implemented automatic sensitive data masking in API logs. [🎟️ DEP-161](https://citz-gdx.atlassian.net/browse/DEP-161) + - Added logging filter to automatically mask passwords, tokens, API keys, and database credentials + - Replaced print() statements with logger calls to ensure masking coverage + - Added linting rules (pylint + flake8) to prevent print statements and enforce logging + - Updated logging configuration to apply masking app-wide + - See `met_api.utils.logging_masker` for implementation details + ## January 28, 2026 - Added documentation for enabling and logging into the Sysdig monitoring platform for OpenShift projects. [🎟️ DEP-160](https://citz-gdx.atlassian.net/browse/DEP-160) diff --git a/met-api/.pylintrc b/met-api/.pylintrc index 3d6bdaf39..c0e41f9ad 100644 --- a/met-api/.pylintrc +++ b/met-api/.pylintrc @@ -33,7 +33,7 @@ ignore-patterns=^\.# # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). -#init-hook= +init-hook='import sys; sys.path.append(".")' # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use. @@ -46,7 +46,8 @@ limit-inference-results=100 # List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. -load-plugins= +load-plugins=pylint_flask, + pylint_plugins.no_print_checker # Pickle collected data for later comparisons. persistent=yes @@ -90,7 +91,9 @@ disable=raw-checker-failed, suppressed-message, useless-suppression, deprecated-pragma, - use-symbolic-message-instead + use-symbolic-message-instead, + C0301, + W0511 # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -192,7 +195,7 @@ contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=Error # Tells whether missing members accessed in mixin class should be ignored. A # class is considered mixin if its name matches the mixin-class-rgx option. @@ -213,13 +216,13 @@ ignore-on-opaque-inference=yes # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local +ignored-classes=optparse.Values,thread._local,_thread._local,scoped_session # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=flask_sqlalchemy,sqlalchemy,SQLAlchemy,alembic,scoped_session # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. @@ -290,7 +293,7 @@ indent-after-paren=4 indent-string=' ' # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=120 # Maximum number of lines in a module. max-module-lines=1000 @@ -319,7 +322,7 @@ ignore-imports=no ignore-signatures=no # Minimum lines number of a similarity. -min-similarity-lines=4 +min-similarity-lines=15 [STRING] @@ -574,7 +577,7 @@ max-returns=6 max-statements=50 # Minimum number of public methods for a class (see R0903). -min-public-methods=2 +min-public-methods=1 [EXCEPTIONS] diff --git a/met-api/Makefile b/met-api/Makefile index bc83615bb..11afcba9b 100644 --- a/met-api/Makefile +++ b/met-api/Makefile @@ -68,7 +68,7 @@ install-dev: ## Install local application ci: lint flake8 test ## CI flow pylint: ## Linting with pylint - . venv/bin/activate && pylint --rcfile=setup.cfg src/$(PROJECT_NAME) + . venv/bin/activate && pylint --rcfile=.pylintrc src/$(PROJECT_NAME) flake8: ## Linting with flake8 . venv/bin/activate && flake8 src/$(PROJECT_NAME) tests diff --git a/met-api/pylint_plugins/README.md b/met-api/pylint_plugins/README.md new file mode 100644 index 000000000..44f7b26fc --- /dev/null +++ b/met-api/pylint_plugins/README.md @@ -0,0 +1,59 @@ +# Pylint Plugins + +This directory contains custom Pylint checkers for the MET API. + +## no_print_checker + +**Purpose:** Prevents use of `print()` statements which bypass sensitive data masking. + +**Why:** Print statements output directly to stdout/stderr and bypass the logging system's `SensitiveDataFilter`. This creates a security risk where passwords, tokens, API keys, and other credentials could be exposed in logs. + +**What it does:** +- Flags any use of `print()` function with error code **W9001** +- Exception: Allows print in `config.py` for startup messages before logging is initialized + +**Usage:** +Instead of: +```python +print(f'User ID: {user_id}') +print(f'Database URL: {db_url}') +``` + +Use: +```python +from flask import current_app +current_app.logger.info('User ID: %s', user_id) + +# or +import logging +logger = logging.getLogger(__name__) +logger.info('Database URL: %s', db_url) +``` + +**Configuration:** +The checker is automatically loaded via `.pylintrc`: +```ini +load-plugins=pylint_plugins.no_print_checker +``` + +**Testing:** +```bash +# Test with a file containing print statements +pylint --rcfile=.pylintrc path/to/file.py + +# Should show: +# W9001: Print statement found - use logging to ensure sensitive data masking +``` + +## Adding New Checkers + +1. Create a new file in this directory (e.g., `my_checker.py`) +2. Implement a class that inherits from `pylint.checkers.BaseChecker` +3. Define your messages and visit methods +4. Add a `register()` function +5. Update `.pylintrc` to load your plugin: + ```ini + load-plugins=pylint_plugins.no_print_checker,pylint_plugins.my_checker + ``` + +See [Pylint documentation](https://pylint.readthedocs.io/en/latest/development_guide/how_tos/custom_checkers.html) for more details. diff --git a/met-api/pylint_plugins/__init__.py b/met-api/pylint_plugins/__init__.py new file mode 100644 index 000000000..0ba6205bb --- /dev/null +++ b/met-api/pylint_plugins/__init__.py @@ -0,0 +1 @@ +"""Pylint plugins for MET API code quality checks.""" diff --git a/met-api/pylint_plugins/no_print_checker.py b/met-api/pylint_plugins/no_print_checker.py new file mode 100644 index 000000000..eafcfc313 --- /dev/null +++ b/met-api/pylint_plugins/no_print_checker.py @@ -0,0 +1,38 @@ +"""Pylint plugin to check for print statements. + +Print statements bypass logging masking and pose a security risk by potentially +exposing sensitive data (passwords, tokens, API keys, etc.) in logs. + +Use current_app.logger or logging.getLogger(__name__) instead. +""" + +from astroid import nodes +from pylint.checkers import BaseChecker + + +class NoPrintChecker(BaseChecker): + """Check for print statements which bypass logging masking.""" + + name = "no-print" + msgs = { + "W9001": ( + "Print statement found - use logging to ensure sensitive data masking", + "print-statement-found", + "Print statements bypass the logging masking system and may expose " + "sensitive data (passwords, tokens, API keys). Use current_app.logger " + "or logging.getLogger(__name__) instead.", + ), + } + + def visit_call(self, node: nodes.Call) -> None: + """Check if the node is a print function call.""" + if isinstance(node.func, nodes.Name) and node.func.name == "print": + # Allow print in config.py for startup messages before logging is initialized + if node.root().file and "config.py" in node.root().file: + return + self.add_message("print-statement-found", node=node) + + +def register(linter): + """Register the checker.""" + linter.register_checker(NoPrintChecker(linter)) diff --git a/met-api/setup.cfg b/met-api/setup.cfg index de7acede6..9af9c0694 100644 --- a/met-api/setup.cfg +++ b/met-api/setup.cfg @@ -38,9 +38,10 @@ ignore = I001, I003, I004, I100, I101, I201, I202, E126, W504 exclude = .git,*migrations* max-line-length = 120 docstring-min-length=10 +# Disable print statement warning in config file - logger is not initialized yet per-file-ignores = */__init__.py:F401 - */config.py:N802 + */config.py:N802,T201 [pycodestyle] max-line-length = 120 diff --git a/met-api/src/met_api/__init__.py b/met-api/src/met_api/__init__.py index 69d42d13b..0b4adb3a3 100644 --- a/met-api/src/met_api/__init__.py +++ b/met-api/src/met_api/__init__.py @@ -3,7 +3,9 @@ This module is for the initiation of the flask app. """ +import builtins import os +import warnings import secure from flask import Flask, current_app, g, request @@ -18,6 +20,38 @@ from met_api.utils import constants from met_api.utils.cache import cache from met_api.utils.roles import Role +from met_api.utils.logging_masker import setup_logging_masking + + +# Store reference to original print function +_original_print = builtins.print + + +class PrintDeprecationWarning(DeprecationWarning): + """Custom warning category for deprecated print() usage.""" + + +def _deprecated_print(*args, **kwargs): + """ + Issue a deprecation warning when print() is used. + + Using print() bypasses the logging masking system and may expose sensitive data. + Use logging instead: current_app.logger.info(), logger.debug(), etc. + """ + warnings.warn( + 'print() is deprecated in this application. ' + 'Use logging instead (e.g., current_app.logger.info()) to ensure ' + 'sensitive data is automatically masked. ' + 'See met_api/utils/logging_masker.py for details.', + PrintDeprecationWarning, + stacklevel=2 + ) + # Still call original print to not break existing code during transition + _original_print(*args, **kwargs) + + +# Replace built-in print with deprecated version +builtins.print = _deprecated_print # Security Response headers csp = ( @@ -53,6 +87,17 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'development')): # Configure app from config.py app.config.from_object(get_named_config(run_mode)) + # Replace built-in print with deprecated version AFTER config loads + # (config.py prints during initialization before logging is set up) + warnings.filterwarnings('default', category=PrintDeprecationWarning) + builtins.print = _deprecated_print + + # Set up logging with sensitive data masking + # Automatically masks passwords, tokens, API keys, database credentials + # Applied to both Flask app logger and root logger for full coverage + setup_logging_masking(app.logger) + setup_logging_masking() + CORS(app, origins=app.config['CORS_ORIGINS'], supports_credentials=True) # Register blueprints diff --git a/met-api/src/met_api/config.py b/met-api/src/met_api/config.py index 64fb5e387..4782fc7ef 100644 --- a/met-api/src/met_api/config.py +++ b/met-api/src/met_api/config.py @@ -85,9 +85,6 @@ def __init__(self) -> None: Performs more advanced configuration logic that is not possible in the normal class definition. """ - # If extending this class, call super().__init__() in your constructor. - print(f'SQLAlchemy URL: {self.SQLALCHEMY_DATABASE_URI}') - # apply configs to _Config in the format that flask_jwt_oidc expects # this flattens the JWT_CONFIG dict into individual attributes for key, value in self.JWT_CONFIG.items(): diff --git a/met-api/src/met_api/logging.conf b/met-api/src/met_api/logging.conf index c711f6780..52fb384fc 100644 --- a/met-api/src/met_api/logging.conf +++ b/met-api/src/met_api/logging.conf @@ -7,6 +7,11 @@ keys=console [formatters] keys=simple +[filters] +# Sensitive data filter automatically masks credentials in logs +# See met_api.utils.logging_masker for pattern details +keys=sensitive_data + [logger_root] level=DEBUG handlers=console @@ -28,7 +33,11 @@ class=StreamHandler level=DEBUG formatter=simple args=(sys.stdout,) +filters=sensitive_data # All other loggers inherit this filter via root logger's console handler [formatter_simple] format=%(asctime)s - %(name)s - %(levelname)s in %(module)s:%(filename)s:%(lineno)d - %(funcName)s: %(message)s -datefmt= \ No newline at end of file +datefmt= + +[filter_sensitive_data] +class=met_api.utils.logging_masker.SensitiveDataFilter \ No newline at end of file diff --git a/met-api/src/met_api/models/engagement.py b/met-api/src/met_api/models/engagement.py index 9c404b7a7..b5e5009ec 100644 --- a/met-api/src/met_api/models/engagement.py +++ b/met-api/src/met_api/models/engagement.py @@ -5,6 +5,7 @@ from __future__ import annotations +import logging from datetime import datetime from typing import List, Optional @@ -197,7 +198,7 @@ def close_engagements_due(cls) -> List[Engagement]: def publish_scheduled_engagements_due(cls) -> List[Engagement]: """Update scheduled engagements to published.""" datetime_due = datetime.utcnow() - print('Publish due date (UTC) ------------------------', datetime_due) + logging.getLogger(__name__).debug('Publish due date (UTC): %s', datetime_due) update_fields = { 'status_id': Status.Published.value, 'published_date': datetime.utcnow(), diff --git a/met-api/src/met_api/resources/staff_user.py b/met-api/src/met_api/resources/staff_user.py index ecb18d14f..fb4b4c682 100644 --- a/met-api/src/met_api/resources/staff_user.py +++ b/met-api/src/met_api/resources/staff_user.py @@ -198,7 +198,7 @@ def get(user_id): if user_id == 'me': user_data = TokenInfo.get_user_data() user_id = user_data.get('external_id') - print('User ID: ', user_id) + current_app.logger.debug('User ID: %s', user_id) user_roles = current_app.config['JWT_ROLE_CALLBACK'](g.jwt_oidc_token_info) if Role.SUPER_ADMIN.value in user_roles: return TenantService.get_all(), HTTPStatus.OK diff --git a/met-api/src/met_api/resources/tenant.py b/met-api/src/met_api/resources/tenant.py index d2585a22a..19d24017f 100644 --- a/met-api/src/met_api/resources/tenant.py +++ b/met-api/src/met_api/resources/tenant.py @@ -15,7 +15,7 @@ from http import HTTPStatus -from flask import request +from flask import current_app, request from flask_cors import cross_origin from flask_restx import Namespace, Resource from marshmallow import ValidationError @@ -61,7 +61,7 @@ def post(): request_json = request.get_json() valid_format, errors = schema_utils.validate(request_json, 'tenant') if not valid_format: - print(errors) + current_app.logger.error('Tenant validation errors: %s', errors) return {'message': schema_utils.serialize(errors)}, HTTPStatus.BAD_REQUEST tenant = TenantSchema().load(request_json) diff --git a/met-api/src/met_api/services/cdogs_api_service.py b/met-api/src/met_api/services/cdogs_api_service.py index 57d415cab..c89b36294 100644 --- a/met-api/src/met_api/services/cdogs_api_service.py +++ b/met-api/src/met_api/services/cdogs_api_service.py @@ -69,7 +69,6 @@ def upload_template(self, template_file_path): template = {'template': ('template', file_handle, 'multipart/form-data')} current_app.logger.info('Uploading template %s', template_file_path) - print('Uploading template %s', template_file_path) response = self._post_upload_template(headers, url, template) if response.status_code == HTTPStatus.OK: @@ -77,7 +76,6 @@ def upload_template(self, template_file_path): raise ValueError('Data not found') current_app.logger.info('Returning new hash %s', response.headers['X-Template-Hash']) - print('Returning new hash %s', response.headers['X-Template-Hash']) return response.headers['X-Template-Hash'] response_json = json.loads(response.content) @@ -86,7 +84,6 @@ def upload_template(self, template_file_path): match = re.findall(r"Hash '(.*?)'", response_json['detail']) if match: current_app.logger.info('Template already hashed with code %s', match[0]) - print('Template already hashed with code %s', match[0]) return match[0] raise ValueError('Data not found') diff --git a/met-api/src/met_api/services/engagement_service.py b/met-api/src/met_api/services/engagement_service.py index 91c39f37b..36db65f39 100644 --- a/met-api/src/met_api/services/engagement_service.py +++ b/met-api/src/met_api/services/engagement_service.py @@ -147,10 +147,10 @@ def publish_scheduled_engagements(): engagements = EngagementModel.publish_scheduled_engagements_due() if not engagements: - print('There are no engagements scheduled for publication') + current_app.logger.info('There are no engagements scheduled for publication') return None - print('Engagements published: ', engagements) + current_app.logger.info('Engagements published: %s', engagements) for engagement in engagements: email_util.publish_to_email_queue( SourceType.ENGAGEMENT.value, @@ -158,7 +158,7 @@ def publish_scheduled_engagements(): SourceAction.PUBLISHED.value, True, ) - print('Engagements published added to email queue: ', engagement.id) + current_app.logger.info('Engagement published added to email queue: %s', engagement.id) return engagements @staticmethod diff --git a/met-api/src/met_api/services/rest_service.py b/met-api/src/met_api/services/rest_service.py index 00b987b17..8e47ee105 100644 --- a/met-api/src/met_api/services/rest_service.py +++ b/met-api/src/met_api/services/rest_service.py @@ -22,6 +22,7 @@ from requests.exceptions import ConnectionError as ReqConnectionError from met_api.utils.enums import AuthHeaderType, ContentType +from met_api.utils.logging_masker import mask_dict class RestService: @@ -53,7 +54,8 @@ def _invoke(rest_method, endpoint, token=None, # pylint: disable=too-many-argum data = json.dumps(data) current_app.logger.debug(f'Endpoint : {endpoint}') - current_app.logger.debug(f'headers : {headers}') + # Mask Authorization headers and other sensitive keys before logging + current_app.logger.debug(f'headers : {mask_dict(headers)}') response = None try: invoke_rest_method = getattr(requests, rest_method) diff --git a/met-api/src/met_api/utils/__init__.py b/met-api/src/met_api/utils/__init__.py new file mode 100644 index 000000000..8bd052dd9 --- /dev/null +++ b/met-api/src/met_api/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright © 2021 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility modules for MET API.""" diff --git a/met-api/src/met_api/utils/datetime.py b/met-api/src/met_api/utils/datetime.py index 71fed53b7..c95a9abc2 100644 --- a/met-api/src/met_api/utils/datetime.py +++ b/met-api/src/met_api/utils/datetime.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """Datetime object helper.""" +from datetime import datetime + import pytz from flask import current_app -from datetime import datetime - def local_datetime(): """Get the local (Pacific Timezone) datetime.""" diff --git a/met-api/src/met_api/utils/filter_types.py b/met-api/src/met_api/utils/filter_types.py index 802b5bb53..7ca676f94 100644 --- a/met-api/src/met_api/utils/filter_types.py +++ b/met-api/src/met_api/utils/filter_types.py @@ -1,7 +1,7 @@ """Filters used to filter by metadata contents in various ways.""" +from sqlalchemy import func from met_api.models.engagement_metadata import EngagementMetadata, MetadataTaxonFilterType -from sqlalchemy import func def list_match_all(query, values): @@ -29,10 +29,7 @@ def list_match_any(query, values): return metadata_subq -""" -Provide a mapping from each filter type to the function that should be used -to filter by that type. -""" +# Mapping from each filter type to the function that should be used to filter by that type filter_map = { MetadataTaxonFilterType.CHIPS_ALL: list_match_all, MetadataTaxonFilterType.CHIPS_ANY: list_match_any, diff --git a/met-api/src/met_api/utils/logging_masker.py b/met-api/src/met_api/utils/logging_masker.py new file mode 100644 index 000000000..e51634d03 --- /dev/null +++ b/met-api/src/met_api/utils/logging_masker.py @@ -0,0 +1,345 @@ +# Copyright © 2021 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Logging masker utility for sensitive data. + +This module provides utilities to mask sensitive information in logs, +including passwords, tokens, API keys, and other credentials. + +USAGE: + Automatic masking is enabled app-wide via logging configuration. + No code changes needed for existing logging statements. + + For explicit masking: + from met_api.utils.logging_masker import mask_url, mask_dict + + # Mask URLs + safe_url = mask_url(database_url) + + # Mask dictionaries + safe_headers = mask_dict(request.headers) + +WHAT GETS MASKED: + - Database connection strings (PostgreSQL, MySQL, MongoDB, Redis) + - Authentication tokens (Bearer, JWT) + - API keys and AWS credentials + - Passwords and secrets + - Private keys in PEM format + +ADDING NEW PATTERNS: + Add to SensitiveDataFilter.DEFAULT_PATTERNS list with: + - 'pattern': compiled regex pattern + - 'replacement': replacement string with capture groups + - 'description': human-readable description +""" + +import logging +import re +from typing import Any, Dict, List, Pattern + + +class SensitiveDataFilter(logging.Filter): + """ + Logging filter that masks sensitive data in log messages. + + This filter uses regex patterns to identify and mask sensitive information + such as passwords, tokens, API keys, and other credentials before they + are written to logs. + """ + + # Mask string to replace sensitive data + MASK = '***REDACTED***' + DEFAULT_REPLACEMENT = r'\1' + MASK + r'\3' + + # Regex patterns for common sensitive data + DEFAULT_PATTERNS: List[Dict[str, Any]] = [ + # Database connection strings (postgresql, mysql, etc.) + { + 'pattern': re.compile( + r'(postgresql|mysql|mongodb|redis)://[^:]+:([^@]+)@', + re.IGNORECASE + ), + 'replacement': r'\1://***REDACTED***:***REDACTED***@', + 'description': 'Database connection string with credentials' + }, + # Bearer tokens in Authorization headers + { + 'pattern': re.compile( + r'(Authorization:\s*Bearer\s+)([A-Z0-9\-_\.]+)', + re.IGNORECASE + ), + 'replacement': r'\1***REDACTED***', + 'description': 'Bearer token in Authorization header' + }, + # Generic Authorization header values + { + 'pattern': re.compile( + r"('Authorization':\s*')([^']+)(')", + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'Authorization header value' + }, + # API keys (various formats) + { + 'pattern': re.compile( + r'(api[_-]?key\s*[:=]\s*["\']?)([A-Z0-9_\-]{20,})(["\']?)', + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'API key' + }, + # AWS/S3 access keys + { + 'pattern': re.compile( + r'(access[_-]?key[_-]?id\s*[:=]\s*["\']?)([A-Z0-9]{20})(["\']?)', + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'AWS access key' + }, + # AWS/S3 secret keys + { + 'pattern': re.compile( + r'(secret[_-]?access[_-]?key\s*[:=]\s*["\']?)([A-Z0-9/+=]{40})(["\']?)', + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'AWS secret key' + }, + # Generic secret/password patterns + { + 'pattern': re.compile( + r'(password\s*[:=]\s*["\']?)([^"\'\s,}]{3,})(["\'\s,}])', + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'Password field (key=value format)' + }, + { + 'pattern': re.compile( + r'(\bwith\s+password\s*:\s*)([^\s,}]+)', + re.IGNORECASE + ), + 'replacement': r'\1***REDACTED***', + 'description': 'Password in sentence' + }, + { + 'pattern': re.compile( + r'(secret\s*[:=]\s*["\']?)([^"\'\s,}]{3,})(["\'\s,}])', + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'Secret field' + }, + # JWT tokens (three base64url segments separated by dots) + { + 'pattern': re.compile( + r'\b(eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b' + ), + 'replacement': r'***REDACTED_JWT***', + 'description': 'JWT token' + }, + # Generic tokens (token=value or "token": "value") + { + 'pattern': re.compile( + r'(token["\']?\s*[:=]\s*["\']?)([A-Z0-9_\-\.]{20,})(["\']?)', + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'Generic token' + }, + # Client secrets + { + 'pattern': re.compile( + r'(client[_-]?secret\s*[:=]\s*["\']?)([^"\'\s,}]{10,})(["\'\s,}])', + re.IGNORECASE + ), + 'replacement': DEFAULT_REPLACEMENT, + 'description': 'Client secret' + }, + # Private keys (PEM format) + { + 'pattern': re.compile( + r'-----BEGIN[A-Z\s]+PRIVATE KEY-----[A-Za-z0-9+/=\s]+-----END[A-Z\s]+PRIVATE KEY-----', + re.MULTILINE + ), + 'replacement': r'***REDACTED_PRIVATE_KEY***', + 'description': 'Private key in PEM format' + }, + ] + + def __init__(self, name: str = '', custom_patterns: List[Dict[str, Any]] = None): + """ + Initialize the filter. + + Args: + name: Name of the filter + custom_patterns: Optional custom patterns to use instead of defaults + """ + super().__init__(name) + self.patterns = custom_patterns if custom_patterns is not None else self.DEFAULT_PATTERNS + + def filter(self, record: logging.LogRecord) -> bool: + """ + Filter log record to mask sensitive data. + + Args: + record: The log record to filter + + Returns: + True (always allows the record through after masking) + """ + # Mask the message + record.msg = self.mask_sensitive_data(str(record.msg)) + + # Mask args if present + if record.args: + if isinstance(record.args, dict): + record.args = { + k: self.mask_sensitive_data(str(v)) + for k, v in record.args.items() + } + elif isinstance(record.args, tuple): + record.args = tuple( + self.mask_sensitive_data(str(arg)) + for arg in record.args + ) + + return True + + def mask_sensitive_data(self, text: str) -> str: + """ + Mask sensitive data in a string using defined patterns. + + Args: + text: The text to mask + + Returns: + The text with sensitive data masked + """ + if not text or not isinstance(text, str): + return text + + masked_text = text + for pattern_config in self.patterns: + pattern: Pattern = pattern_config['pattern'] + replacement: str = pattern_config['replacement'] + masked_text = pattern.sub(replacement, masked_text) + + return masked_text + + +def mask_dict(data: Dict[str, Any], sensitive_keys: List[str] = None) -> Dict[str, Any]: + """ + Mask sensitive keys in a dictionary. + + Args: + data: Dictionary to mask + sensitive_keys: List of keys to mask (case-insensitive) + + Returns: + Dictionary with sensitive values masked + """ + if sensitive_keys is None: + sensitive_keys = [ + 'password', 'secret', 'token', 'api_key', 'apikey', + 'access_key', 'secret_key', 'client_secret', + 'authorization', 'auth', 'credentials' + ] + + if not isinstance(data, dict): + return data + + masked_data = {} + for key, value in data.items(): + # Check if key contains sensitive keywords + if any(sensitive in key.lower() for sensitive in sensitive_keys): + masked_data[key] = '***REDACTED***' + elif isinstance(value, dict): + masked_data[key] = mask_dict(value, sensitive_keys) + elif isinstance(value, list): + masked_data[key] = [ + mask_dict(item, sensitive_keys) if isinstance(item, dict) else item + for item in value + ] + else: + masked_data[key] = value + + return masked_data + + +def mask_url(url: str) -> str: + """ + Mask credentials in a URL. + + Args: + url: URL string that may contain credentials + + Returns: + URL with credentials masked + """ + # Pattern to match URLs with credentials + pattern = re.compile( + r'((?:https?|ftp|postgresql|mysql|mongodb|redis)://)' + r'([^:]+):([^@]+)@' + r'(.+)', + re.IGNORECASE + ) + + match = pattern.match(url) + if match: + protocol, _username, _password, rest = match.groups() + return f'{protocol}***REDACTED***:***REDACTED***@{rest}' + + return url + + +def setup_logging_masking(logger: logging.Logger = None) -> None: + """ + Set up sensitive data masking for a logger. + + If no logger is provided, sets up masking for the root logger. + + Args: + logger: Logger to add the filter to (defaults to root logger) + """ + if logger is None: + logger = logging.getLogger() + + # Add the filter if not already present + if not any(isinstance(f, SensitiveDataFilter) for f in logger.filters): + logger.addFilter(SensitiveDataFilter()) + + +def get_masked_config_string(config_obj: Any) -> str: + """ + Get a string representation of a config object with sensitive data masked. + + Args: + config_obj: Configuration object to stringify + + Returns: + Masked string representation of the config + """ + config_dict = {} + for attr in dir(config_obj): + if not attr.startswith('_'): + value = getattr(config_obj, attr, None) + if not callable(value): + config_dict[attr] = value + + masked_dict = mask_dict(config_dict) + return str(masked_dict) diff --git a/met-api/src/met_api/utils/notification.py b/met-api/src/met_api/utils/notification.py index 46b2005cf..6fb8b5561 100644 --- a/met-api/src/met_api/utils/notification.py +++ b/met-api/src/met_api/utils/notification.py @@ -23,8 +23,7 @@ def get_tenant_site_url(tenant_id, path=''): if tenant is None: raise ValueError(f'Tenant with id {tenant_id} not found.') return site_url + f'/{tenant.short_name.lower()}' + path - else: - return site_url + path + return site_url + path def send_email(subject, email, html_body, args, template_id): @@ -53,7 +52,8 @@ def send_email(subject, email, html_body, args, template_id): headers={ 'Content-Type': 'application/json', 'Authorization': f'Bearer {service_account_token}'}, - data=json.dumps(payload)) + data=json.dumps(payload), + timeout=30) response.raise_for_status() diff --git a/met-api/src/met_api/utils/roles.py b/met-api/src/met_api/utils/roles.py index 4ec1f6af7..b17eb8759 100644 --- a/met-api/src/met_api/utils/roles.py +++ b/met-api/src/met_api/utils/roles.py @@ -64,3 +64,4 @@ class Role(Enum): MANAGE_METADATA = 'manage_metadata' VIEW_LANGUAGES = 'view_languages' EDIT_LANGUAGES = 'edit_languages' + SYSTEM = 'system' # System user role for EPIC connections diff --git a/met-api/src/met_api/utils/tenant_validator.py b/met-api/src/met_api/utils/tenant_validator.py index ec5f019e2..c3cbb1d4f 100644 --- a/met-api/src/met_api/utils/tenant_validator.py +++ b/met-api/src/met_api/utils/tenant_validator.py @@ -17,7 +17,7 @@ A simple decorator to validate roles with in the tenant. """ from functools import wraps -from typing import Dict, List +from typing import Dict, List, Union from flask import current_app, g @@ -25,7 +25,7 @@ from met_api.utils.roles import Role -def require_role(required_roles: List[str] | str) -> callable: +def require_role(required_roles: Union[List[str], str]) -> callable: """Validate a token for roles and against tenant as well. Args: diff --git a/met-api/tests/unit/utils/test_logging_masker.py b/met-api/tests/unit/utils/test_logging_masker.py new file mode 100644 index 000000000..b1ada4f26 --- /dev/null +++ b/met-api/tests/unit/utils/test_logging_masker.py @@ -0,0 +1,82 @@ +# Copyright © 2021 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for logging masker utilities.""" + +import logging + +from met_api.utils.logging_masker import SensitiveDataFilter, mask_dict, mask_url + + +def test_logging_masker(): + """Test comprehensive logging masker functionality for sensitive data.""" + # Test URL masking + url = 'postgresql://user:secretpassword@localhost:5432/metdb' + masked_url = mask_url(url) + assert 'secretpassword' not in masked_url + assert 'user' not in masked_url + # Count occurrences of redaction to ensure we are masking both username and password + assert masked_url.count('***REDACTED***') == 2 + + # Test dictionary masking with various sensitive keys + data = { + 'username': 'admin', + 'password': 'secret123', + 'api_key': 'myapikey', + 'token': 'mytoken', + 'email': 'test@example.com', + 'config': { + 'secret': 'nested_secret' + } + } + masked_data = mask_dict(data) + assert masked_data['password'] == '***REDACTED***' + assert masked_data['api_key'] == '***REDACTED***' + assert masked_data['token'] == '***REDACTED***' + assert masked_data['config']['secret'] == '***REDACTED***' + assert masked_data['username'] == 'admin' + assert masked_data['email'] == 'test@example.com' + + # Test SensitiveDataFilter with various patterns + filter_obj = SensitiveDataFilter() + + # Test SQLAlchemy URL + log_record = logging.LogRecord( + name='test', level=logging.INFO, pathname='', lineno=0, + msg='SQLAlchemy URL: postgresql://dbuser:complexP@ss123@localhost:5432/metdb', + args=(), exc_info=None + ) + filter_obj.filter(log_record) + assert 'complexP@ss123' not in str(log_record.msg) + assert '***REDACTED***' in str(log_record.msg) + + # Test JWT token + jwt_token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0In0.abc123' + log_record = logging.LogRecord( + name='test', level=logging.INFO, pathname='', lineno=0, + msg=f'Processing token: {jwt_token}', + args=(), exc_info=None + ) + filter_obj.filter(log_record) + assert jwt_token not in str(log_record.msg) + assert '***REDACTED_JWT***' in str(log_record.msg) + + # Test normal content preservation + normal_message = 'Processing user request for engagement ID 123' + log_record = logging.LogRecord( + name='test', level=logging.INFO, pathname='', lineno=0, + msg=normal_message, args=(), exc_info=None + ) + filter_obj.filter(log_record) + assert log_record.msg == normal_message