code refactor based on review

.github/workflows/run-e2e-tests.yaml

@@ -1,7 +1,7 @@
 name: E2E Test Workflow For Firehose
 
 on:
-  push:
+  pull_request:
     branches:
       - main
       - develop
@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        test-case: [test-without-filter, test-with-filter, test-with-invalid-log-group]
+        test-case: [test-without-filter, test-with-filter, test-with-invalid-log-group, test-with-secret-manager-false]
 
     steps:
       - name: Checkout code

e2e_tests/common-scripts.sh

@@ -73,11 +73,10 @@ delete_stack() {
   fi
 }
 
-validate_stack_resources() {
+validate_stack_resources_with_subscription() {
   local stack_name=$1
-  local validate_only_firehose_stack=$2
-  local log_group_name=$3
-  local log_group_filter=$4
+  local log_group_name=$2
+  local log_group_filter=$3
 
 
   log "Validating stack resources for stack: $stack_name"
@@ -100,12 +99,6 @@ validate_stack_resources() {
     exit_with_error "Failed to retrieve Firehose delivery stream ARN for physical ID: $firehose_stream_physical_id"
   fi
 
-  # Skip log group subscription validation if validate_only_firehose_stack is true or log_group_filter is empty
-  if [ "$validate_only_firehose_stack" == "true" ] || [ -z "$log_group_filter" ]; then
-    log "Validated Firehose Stream, skipping log group subscription validation."
-    return
-  fi
-
   subscriptions=$(aws logs describe-subscription-filters --log-group-name "$log_group_name" --query 'subscriptionFilters[*].[destinationArn, filterPattern]' --output text)
 
   # Check if the Firehose delivery stream is subscribed to the log group with the specified filter pattern
@@ -117,105 +110,34 @@ validate_stack_resources() {
 
 }
 
+validate_stack_resources_without_subscription() {
+  local stack_name=$1
+  local log_group_name=$2
 
-exit_with_error() {
-  echo "Error: $1"
-  exit 1
-}
-
-create_log_event() {
-  local log_group_name=$1
-  local log_stream_name=$2
-  local log_message=$3
-
-  log "Creating log event in CloudWatch Log Group: $log_group_name"
-  local log_stream_exists=$(aws logs describe-log-streams --log-group-name "$log_group_name" --log-stream-name-prefix "$log_stream_name" --query "logStreams[?logStreamName=='$log_stream_name'] | length(@)" --output text)
-
-  if [ "$log_stream_exists" -eq 0 ]; then
-    log "Creating log stream: $log_stream_name"
-    aws logs create-log-stream --log-group-name "$log_group_name" --log-stream-name "$log_stream_name"
-  fi
-
-  # Get the current timestamp in milliseconds
-  timestamp=$(($(date +%s) * 1000 + $(date +%N) / 1000000))
-
-  # Put log event
-  aws logs put-log-events \
-    --log-group-name "$log_group_name" \
-    --log-stream-name "$log_stream_name" \
-    --log-events timestamp=$timestamp,message="$log_message"
-
-  log "Log event created successfully."
 
-}
+  log "Validating stack resources for stack: $stack_name"
+  firehose_stream_physical_id=$(aws cloudformation describe-stack-resources \
+                  --stack-name "$stack_name" \
+                  --logical-resource-id "$FIREHOSE_STREAM_LOGICAL_ID" \
+                  --query "StackResources[0].PhysicalResourceId" \
+                  --output text
+  )
 
-validate_logs_in_new_relic() {
-  local user_key=$1
-  local account_id=$2
-  local log_message=$3
-  local common_attributes=$4
-  local should_log_exist=$5
-
-  local nrql_query="SELECT * FROM Log WHERE message LIKE '%$log_message%' SINCE 10 minutes ago"
-  local query='{"query":"query($id: Int!, $nrql: Nrql!) { actor { account(id: $id) { nrql(query: $nrql) { results } } } }","variables":{"id":'$account_id',"nrql":"'$nrql_query'"}}'
-
-  local log_message_exists=false
-
-  sleep_time=$SLEEP_TIME
-
-  for i in {1..5}; do
-    local response=$(curl -s -X POST \
-      -H "Content-Type: application/json" \
-      -H "API-Key: $user_key" \
-      -d "$query" \
-      https://api.newrelic.com/graphql)
-
-    if echo "$response" | grep -q "$log_message"; then
-      log "Log event successfully found in New Relic."
-      log_message_exists=true
-      validate_logs_meta_data "$response" "$common_attributes"
-      break
-    else
-      log "Log event not found in New Relic. Retrying in $sleep_time seconds..."
-      sleep $sleep_time
-      sleep_time=$(( sleep_time * 2 ))
-    fi
-  done
+  # Get the ARN of the Firehose delivery stream using the physical ID
+  firehose_stream_arn=$(aws firehose describe-delivery-stream \
+                  --delivery-stream-name "$firehose_stream_physical_id" \
+                  --query "DeliveryStreamDescription.DeliveryStreamARN" \
+                  --output text
+  )
 
-  if [ "$should_log_exist" == "true" ] && [ "$log_message_exists" == "false" ]; then
-    exit_with_error "Log event not found in New Relic after 10 retries."
-  elif [ "$should_log_exist" == "false" ] && [ "$log_message_exists" == "true" ]; then
-    exit_with_error "Log event should not exist in New Relic, but it was found."
+  # Check firehose_stream_arn is not null
+  if [ -z "$firehose_stream_arn" ] || [ "$firehose_stream_arn" == "None" ]; then
+    exit_with_error "Failed to retrieve Firehose delivery stream ARN for physical ID: $firehose_stream_physical_id"
   fi
-
 }
 
-validate_logs_meta_data (){
-  local response=$1
-  local common_attributes=$2
-
-  # Remove single quotes from common_attributes
-  common_attributes=$(echo "$common_attributes" | sed "s/'//g")
-
-  # Validate common attributes
-  for attribute in $(echo "$common_attributes" | jq -c '.[]'); do
-    attribute_name=$(echo "$attribute" | jq -r '.AttributeName')
-    attribute_value=$(echo "$attribute" | jq -r '.AttributeValue')
-    if ! echo "$response" | grep -q "\"$attribute_name\":\"$attribute_value\""; then
-      exit_with_error "Common attribute $attribute_name with value $attribute_value not found in New Relic logs."
-    fi
-  done
-  log "Common attributes validated successfully."
-
-  # Read default attributes from config file and replace underscores with dots
-  while IFS='=' read -r key value; do
-    if [[ $key == instrumentation_* ]]; then
-      new_key=$(echo "$key" | sed 's/_/./g')
-      if ! echo "$response" | grep -q "\"$new_key\":\"$value\""; then
-        exit_with_error "Entity synthesis attribute $new_key with value $value not found in New Relic logs."
-      fi
-    fi
-  done < config-file.cfg
 
-  log "Entity synthesis parameter validated successfully."
+exit_with_error() {
+  echo "Error: $1"
+  exit 1
 }

e2e_tests/config-file.cfg

@@ -18,8 +18,7 @@ LOG_GROUP_NAME_2=aws-unified-firehose-e2e-test-log-group-2
 # Variable for third test case
 FIREHOSE_STACK_NAME_3=NewRelicFirehoseTestStackWithInvalidLogGroup
 INVALID_LOG_GROUP_NAME=invalid_log_group
-SLEEP_TIME=15
-# Entity synthesis Parameters
-instrumentation_provider=aws
-instrumentation_name=firehose
-instrumentation_version=1.0.0
+# Variable for fourth test case
+FIREHOSE_STACK_NAME_4=NewRelicFirehoseTestStackWithSecretsManagerFalse
+LOG_GROUP_NAME_4=aws-unified-firehose-e2e-test-log-group-3
+SLEEP_TIME=15

e2e_tests/entity_synthesis_param.cfg

@@ -0,0 +1,4 @@
+# Entity synthesis Parameters
+instrumentation_provider=aws
+instrumentation_name=firehose
+instrumentation_version=1.0.0

e2e_tests/firehose_e2e_tests.sh

@@ -2,6 +2,7 @@
 
 source config-file.cfg
 source common-scripts.sh
+source log_validation.sh
 
 # Test Case 1: 
 # Create a Firehose stack without any subscription filter pattern
@@ -27,8 +28,8 @@ COMMON_ATTRIBUTES=$(<common_attribute.json)
   validate_stack_deployment_status "$FIREHOSE_STACK_NAME_1"
 
   # Validate the stack resources
+  validate_stack_resources_without_subscription "$FIREHOSE_STACK_NAME_1" "$LOG_GROUP_NAME_1"
 
-  validate_stack_resources "$FIREHOSE_STACK_NAME_1" "false" "$LOG_GROUP_NAME_1" ""
   # Generate a UUID and create a dynamic log message
   UUID=$(uuidgen)
   LOG_MESSAGE="RequestId: $UUID hello world"
@@ -70,7 +71,7 @@ COMMON_ATTRIBUTES=$(<common_attribute.json)
   validate_stack_deployment_status "$FIREHOSE_STACK_NAME_2"
 
   # Validate the stack resources
-  validate_stack_resources "$FIREHOSE_STACK_NAME_2" "false" "$LOG_GROUP_NAME_2" "$LOG_GROUP_FILTER_PATTERN"
+  validate_stack_resources_with_subscription "$FIREHOSE_STACK_NAME_2" "$LOG_GROUP_NAME_2" "$LOG_GROUP_FILTER_PATTERN"
 
   # Generate a UUID and create a dynamic log message with the filter pattern
   UUID=$(uuidgen)
@@ -119,6 +120,42 @@ LOG_GROUP_INVALID_JSON=$(<invalid_log_group.json)
   # Delete the Firehose stack
   delete_stack "$FIREHOSE_STACK_NAME_3"
 }
+
+test_with_store_secret_manager_false() {
+  local template_file=$TEMPLATE_FILE_FULL_PATH
+
+cat <<EOF > log_group.json
+'[{"LogGroupName":"$LOG_GROUP_NAME_4"}]'
+EOF
+LOG_GROUP_JSON_4=$(<log_group.json)
+
+cat <<EOF > common_attribute.json
+'[{"AttributeName":"$COMMON_ATTRIBUTE_KEY","AttributeValue":"$COMMON_ATTRIBUTE_VALUE"}]'
+EOF
+COMMON_ATTRIBUTES=$(<common_attribute.json)
+
+  # Deploy the Firehose stack
+  deploy_firehose_stack "$template_file" "$FIREHOSE_STACK_NAME_4" "$NEW_RELIC_LICENSE_KEY" "$NEW_RELIC_REGION" "$NEW_RELIC_ACCOUNT_ID" "false" "$LOG_GROUP_JSON_4" "$COMMON_ATTRIBUTES"
+
+  # Validate the status of the Firehose stack
+  validate_stack_deployment_status "$FIREHOSE_STACK_NAME_4"
+
+  # Validate the stack resources
+  validate_stack_resources_without_subscription "$FIREHOSE_STACK_NAME_4" "$LOG_GROUP_NAME_1"
+
+  # Generate a UUID and create a dynamic log message
+  UUID=$(uuidgen)
+  LOG_MESSAGE="RequestId: $UUID hello world"
+
+  # Create a log event in CloudWatch Logs
+  create_log_event "$LOG_GROUP_NAME_1" "$LOG_STREAM_NAME" "$LOG_MESSAGE"
+
+  # Validate logs in New Relic
+  validate_logs_in_new_relic "$NEW_RELIC_USER_KEY" "$NEW_RELIC_ACCOUNT_ID" "$LOG_MESSAGE" "$COMMON_ATTRIBUTES" "true"
+
+  # Delete the Firehose stack
+  delete_stack "$FIREHOSE_STACK_NAME_4"
+}
 
 
 
@@ -129,20 +166,27 @@ BUILD_DIR="$BUILD_DIR_BASE/$BASE_NAME"
 sam build --template-file "../$TEMPLATE_FILE_NAME" --build-dir "$BUILD_DIR"
 sam package --s3-bucket "$S3_BUCKET" --template-file "$BUILD_DIR/template.yaml" --output-template-file "$BUILD_DIR/$TEMPLATE_FILE_NAME"
 
-
-Run the test cases
-case $1 in
-  test-without-filter)
-    test_logs_without_filter_pattern 
-    ;;
-  test-with-filter)
-    test_logs_with_filter_pattern 
-    ;;
-  test-with-invalid-log-group)
-    test_logs_with_invalid_log_group 
-    ;;
-  *)
-    echo "Invalid test case specified."
-    exit 1
-    ;;
-esac
+test_logs_without_filter_pattern &
+test_logs_with_filter_pattern &
+test_logs_with_invalid_log_group &
+test_with_store_secret_manager_false &
+
+# Run the test cases
+# case $1 in
+#   test-without-filter)
+#     test_logs_without_filter_pattern 
+#     ;;
+#   test-with-filter)
+#     test_logs_with_filter_pattern 
+#     ;;
+#   test-with-invalid-log-group)
+#     test_logs_with_invalid_log_group 
+#     ;;
+#   test-with-secret-manager-false)
+#     test_with_store_secret_manager_false 
+#     ;;    
+#   *)
+#     echo "Invalid test case specified."
+#     exit 1
+#     ;;
+# esac