Feat: dynamic filter regex

Author: mhd999

README.md

@@ -0,0 +1,23 @@
+An AWS Lambda function that is designed to filter logs to remove sensitive data, mask any sensitive information, and then post the resulting data to a separate log group.
+
+
+## Provision the Lambda using Terraform
+```
+module "log-filter-mask" {
+  source = "github.com/mhd999/aws-lambda-log-filter-mask?ref=v0.0.3"
+
+  source_log_group      = "/aws/appsync/apis/abc"
+  destination_log_group = "aws/sensitive/data"
+  sensitive_words       = "password|\"password\"|new_password|\"new_password\""
+  placeholder           = "*****"
+  filter_pattern        = "?password ?new_password"
+}
+```
+## Environment variables
+
+| Variable              | Description | Example                                     |
+| --------------------- | ------------|---------------------------------------------|
+| DESTINATION_LOG_GROUP | Name of AWS cloudwatch destination log group              | aws/sensitive/data
+| SOURCE_LOG_GROUP      | Name of AWS cloudwatch source log group                   | aws/appsync/apis/abc
+| SENSITIVE_WORDS       | The sensitive words in logs to mask                       | password|"password"|new_password|"new_password"
+| PLACEHOLDER           | The value to replace the value of the senstive words with | "*****"

handler.js

@@ -18451,9 +18451,10 @@ const zlib_1 = __webpack_require__(796);
 const crypto_1 = __webpack_require__(113);
 const client_cloudwatch_logs_1 = __webpack_require__(316);
 exports.handler = async (event) => {
-    const placeholder = process.env.PLAEHOLDER || "*****";
+    const placeholder = process.env.PLACEHOLDER || "*****";
     const destinationLogGroupName = process.env.DESTINATION_LOG_GROUP;
     const sourceLogGroupName = process.env.SOURCE_LOG_GROUP;
+    const sensitiveWords = process.env.SENSITIVE_WORDS;
     const payload = await Buffer.from(event.awslogs.data, 'base64');
     function getLogRecord() {
         return new Promise(function (resolve, reject) {
@@ -18476,7 +18477,7 @@ exports.handler = async (event) => {
         return new Promise(function (resolve, reject) {
             const logs = logEvents.map(logEvent => {
                 const logMessage = logEvent.message;
-                const pattern = /("password"|password|new_password|"new_password")\s*:\s*"([^"]*)/g;
+                let pattern = new RegExp(`(${sensitiveWords})\s*:\s*"([^"]*)`, 'g');
                 const maskedMessage = logMessage?.replace(pattern, (match, key, value) => {
                     return `"${key}":"${placeholder}"`;
                 });

main.tf

@@ -3,6 +3,8 @@ variable "aws_region" {
 }
 variable "source_log_group" {}
 variable "destination_log_group" {}
+variable "sensitive_words" {}
+variable "placeholder" {}
 variable "filter_pattern" {
   default = ""
 }
@@ -27,6 +29,14 @@ resource "aws_cloudwatch_log_group" "destination_log_group" {
   retention_in_days = 60
 }
 
+resource "random_string" "lambda_postfix_generator" {
+  length  = 16
+  upper   = true
+  lower   = true
+  numeric = true
+  special = false
+}
+
 resource "aws_lambda_function" "filter_and_mask_logs" {
   filename      = "filter-and-mask-logs.zip"
   function_name = "filter-and-mask-logs"
@@ -39,6 +49,8 @@ resource "aws_lambda_function" "filter_and_mask_logs" {
       SOURCE_LOG_GROUP = data.aws_cloudwatch_log_group.source_log_group.name
       # FILTER_PATTERN         = "password"
       DESTINATION_LOG_GROUP = aws_cloudwatch_log_group.destination_log_group.name
+      SENSITIVE_WORDS       = var.sensitive_words
+      PLACEHOLDER            = var.placeholder
     }
   }
 

src/index.ts

@@ -4,10 +4,10 @@ import { CloudWatchLogsClient, PutLogEventsCommand, CreateLogStreamCommand, Dele
 import { MetadataBearer } from "@aws-sdk/types";
 
 exports.handler = async (event) => {
-  const placeholder = process.env.PLAEHOLDER || "*****";
+  const placeholder = process.env.PLACEHOLDER || "*****";
   const destinationLogGroupName = process.env.DESTINATION_LOG_GROUP;
   const sourceLogGroupName = process.env.SOURCE_LOG_GROUP;
-  // TODO: get sensitive phrases from env var
+  const sensitiveWords = process.env.SENSITIVE_WORDS;
 
   const payload = await Buffer.from(event.awslogs.data, 'base64');
 
@@ -42,7 +42,7 @@ exports.handler = async (event) => {
     return new Promise(function (resolve, reject) {
       const logs = logEvents.map(logEvent => {
         const logMessage = logEvent.message;
-        const pattern = /("password"|password|new_password|"new_password")\s*:\s*"([^"]*)/g;
+        let pattern = new RegExp(`(${sensitiveWords})\s*:\s*"([^"]*)`, 'g');
 
         const maskedMessage = logMessage?.replace(pattern, (match, key, value) => {
           return `"${key}":"${placeholder}"`;
@@ -60,7 +60,6 @@ exports.handler = async (event) => {
   }
 
   const maskedLogEvents: InputLogEvent[] = await maskEvent();
-
   const logStreamName = randomBytes(20).toString('hex');
 
   // create log stream

template.yaml

@@ -13,4 +13,6 @@ Resources:
       Environment:
         Variables:
           DESTINATION_LOG_GROUP:
-          SOURCE_LOG_GROUP: 
+          SOURCE_LOG_GROUP:
+          SENSITIVE_WORDS:
+          PLACEHOLDER: