(feat) Add masking to python middleware

Makefile

@@ -68,17 +68,17 @@ test-webhooks-php-laravel: ## Run webhooks tests against the PHP SDK + Laravel
 
 test-metrics-python-django: ## Run Metrics tests against the Python SDK + Django
 	docker-compose up --build --detach integration_metrics_python_django
-	npm run test:integration-metrics || make cleanup-failure
+	SUPPORTS_HASHING=true npm run test:integration-metrics || make cleanup-failure
 	@make cleanup
 
 test-metrics-python-flask: ## Run Metrics tests against the Python SDK + Flask
 	docker-compose up --build --detach integration_python_flask_metrics
-	npm run test:integration-metrics || make cleanup-failure
+	SUPPORTS_HASHING=true npm run test:integration-metrics || make cleanup-failure
 	@make cleanup
 
 test-webhooks-python-flask: ## Run webhooks tests against the Python SDK + Flask
 	docker-compose up --build --detach integration_python_flask_webhooks
-	npm run test:integration-webhooks || make cleanup-failure
+	SUPPORTS_HASHING=true npm run test:integration-webhooks || make cleanup-failure
 	@make cleanup
 
 ##

packages/python/readme_metrics/PayloadBuilder.py

@@ -9,6 +9,7 @@
 from typing import List, Optional
 from urllib import parse
 from readme_metrics import ResponseInfoWrapper
+from readme_metrics.util import mask
 
 
 class QueryNotFound(Exception):
@@ -135,6 +136,9 @@ def _validate_group(self, group: Optional[dict]):
             )
             return None
 
+        # Mask the id / api_key
+        group["id"] = mask(group["id"])
+
         for field in ["email", "label"]:
             if field not in group:
                 self.logger.warning(
@@ -167,6 +171,11 @@ def _build_request_payload(self, request) -> dict:
             dict: Wrapped request payload
         """
         headers = self.redact_dict(request.headers)
+
+        # Mask the auth header
+        if headers.get("Authentication"):
+            headers["Authentication"] = mask(headers.get("Authentication"))
+
         queryString = parse.parse_qsl(self._get_query_string(request))
 
         content_type = self._get_content_type(headers)
@@ -364,10 +373,6 @@ def _redact_value(val):
 
             return "[REDACTED]"
 
-        # Short-circuit this function if there's no allowlist or denylist
-        if not (self.allowlist or self.denylist):
-            return mapping
-
         result = {}
         for (key, value) in mapping.items():
             if self.denylist and key in self.denylist:
@@ -376,4 +381,5 @@ def _redact_value(val):
                 result[key] = _redact_value(value)
             else:
                 result[key] = value
+
         return result

packages/python/readme_metrics/__init__.py

@@ -1,4 +1,4 @@
 from readme_metrics.MetricsApiConfig import MetricsApiConfig
 from readme_metrics.MetricsMiddleware import MetricsMiddleware
 
-__version__ = "3.1.0"
+__version__ = "3.2.0"

packages/python/readme_metrics/tests/PayloadBuilder_test.py

@@ -4,6 +4,7 @@
 from readme_metrics import MetricsApiConfig
 from readme_metrics import MetricsMiddleware
 from readme_metrics.PayloadBuilder import PayloadBuilder
+from readme_metrics.util import mask
 from .fixtures import Environ
 
 
@@ -183,7 +184,37 @@ def test_grouping_function(self):
         data = payload(metrics.req, metrics.res, str(uuid.uuid4()))
         group = data["group"]
 
-        assert group["id"] == "spam"
+        assert group["id"] == mask("spam")
+        assert group["email"] == "[email protected]"
+        assert group["label"] == "Spam Musubi"
+
+    def test_mask_matches_node(self):
+        config = self.mockMiddlewareConfig(
+            grouping_function=lambda req: {
+                "api_key": "Bearer: a-random-api-key",
+                "email": "[email protected]",
+                "label": "Spam Musubi",
+            }
+        )
+
+        environ = Environ.MockEnviron().getEnvironForRequest(b"", "POST")
+        app = MockApplication("{ 'responseObject': 'value' }")
+
+        metrics = MetricsCoreMock()
+        middleware = MetricsMiddleware(app, config)
+        middleware.metrics_core = metrics
+
+        next(middleware(environ, app.mockStartResponse))
+
+        payload = self.createPayload(config)
+        data = payload(metrics.req, metrics.res, str(uuid.uuid4()))
+        group = data["group"]
+
+        assert (
+            group["id"]
+            == "sha512-7S+L0vUE8Fn6HI3836rtz4b6fVf6H4JFur6SGkOnL3b"
+            + "FpC856+OSZkpIHphZ0ipNO+kUw1ePb5df2iYrNQCpXw==?-key"
+        )
         assert group["email"] == "[email protected]"
         assert group["label"] == "Spam Musubi"
 
@@ -228,7 +259,7 @@ def test_deprecated_id_grouping(self):
         data = payload(metrics.req, metrics.res, str(uuid.uuid4()))
         group = data["group"]
 
-        assert group["id"] == "spam"
+        assert group["id"] == mask("spam")
         assert group["email"] == "[email protected]"
         assert group["label"] == "Spam Musubi"
 
@@ -282,7 +313,7 @@ def test_grouping_function_validation_when_extra_fields_present(self):
         assert isinstance(data, dict)
         assert "group" in data
         assert data["group"] == {
-            "id": "spam",
+            "id": mask("spam"),
             "email": "[email protected]",
             "label": "Spam Musubi",
         }
@@ -311,6 +342,8 @@ def test_uuid_generation(self):
         payload = self.createPayload(config)
         data = payload(metrics.req, metrics.res, str(uuid.uuid4()))
 
+        print(data["_id"])
+
         assert data["_id"] == str(uuid.UUID(data["_id"], version=4))
 
     def test_har_creator(self):
@@ -337,3 +370,25 @@ def test_har_creator(self):
 
         assert post_data["text"] == json.dumps({"ok": 123, "password": 456})
         assert post_data["mimeType"] == "text/plain"
+
+    def test_auth_header_masked(self):
+        config = self.mockMiddlewareConfig(development_mode=True)
+        environ = Environ.MockEnviron().getEnvironForRequest(b"", "GET")
+        app = MockApplication("{ 'responseObject': 'value' }")
+
+        metrics = MetricsCoreMock()
+        middleware = MetricsMiddleware(app, config)
+        middleware.metrics_core = metrics
+
+        next(middleware(environ, app.mockStartResponse))
+
+        payload = self.createPayload(config)
+        data = payload(metrics.req, metrics.res, str(uuid.uuid4()))
+
+        # Pull out the auth header
+        auth_header = ""
+        for header in data["request"]["log"]["entries"][0]["request"]["headers"]:
+            if header["name"] == "Authentication":
+                auth_header = header["value"]
+
+        assert auth_header == mask(environ["HTTP_AUTHENTICATION"])

packages/python/readme_metrics/tests/fixtures/Environ.py

@@ -27,4 +27,5 @@ def getEnvironForRequest(self, jsonByteString, httpRequestMethod):
         stream = io.BytesIO(jsonByteString)
         environ["wsgi.input"] = wsgi.LimitedStream(stream, contentLength)
         environ["CONTENT_LENGTH"] = contentLength
+        environ["HTTP_AUTHENTICATION"] = "Bearer: a-random-api-key"
         return environ

packages/python/readme_metrics/util.py

@@ -1,4 +1,6 @@
 import logging
+import hashlib
+import base64
 from logging import Formatter, StreamHandler
 
 
@@ -10,3 +12,11 @@ def util_build_logger():
     handler.setFormatter(formatter)
     logger.addHandler(handler)
     return logger
+
+
+def mask(data):
+    m = hashlib.sha512()
+    m.update(bytes(data, "ascii"))
+    digest = base64.b64encode(m.digest())
+    opts = data[-4:]
+    return "sha512-" + digest.decode("ascii") + "?" + opts