Skip to content

Commit 3b8077b

Browse files
hrai-nrhrai-nr
committed
code refactor based on review
1 parent a192385 commit 3b8077b

File tree

6 files changed

+195
-129
lines changed

6 files changed

+195
-129
lines changed

‎.github/workflows/run-e2e-tests.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: E2E Test Workflow For Firehose
22

33
on:
4-
push:
4+
pull_request:
55
branches:
66
- main
77
- develop
@@ -15,7 +15,7 @@ jobs:
1515

1616
strategy:
1717
matrix:
18-
test-case: [test-without-filter, test-with-filter, test-with-invalid-log-group]
18+
test-case: [test-without-filter, test-with-filter, test-with-invalid-log-group, test-with-secret-manager-false]
1919

2020
steps:
2121
- name: Checkout code

‎e2e_tests/common-scripts.sh

Lines changed: 25 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,10 @@ delete_stack() {
7373
fi
7474
}
7575

76-
validate_stack_resources() {
76+
validate_stack_resources_with_subscription() {
7777
local stack_name=$1
78-
local validate_only_firehose_stack=$2
79-
local log_group_name=$3
80-
local log_group_filter=$4
78+
local log_group_name=$2
79+
local log_group_filter=$3
8180

8281

8382
log "Validating stack resources for stack: $stack_name"
@@ -100,12 +99,6 @@ validate_stack_resources() {
10099
exit_with_error "Failed to retrieve Firehose delivery stream ARN for physical ID: $firehose_stream_physical_id"
101100
fi
102101

103-
# Skip log group subscription validation if validate_only_firehose_stack is true or log_group_filter is empty
104-
if [ "$validate_only_firehose_stack" == "true" ] || [ -z "$log_group_filter" ]; then
105-
log "Validated Firehose Stream, skipping log group subscription validation."
106-
return
107-
fi
108-
109102
subscriptions=$(aws logs describe-subscription-filters --log-group-name "$log_group_name" --query 'subscriptionFilters[*].[destinationArn, filterPattern]' --output text)
110103

111104
# Check if the Firehose delivery stream is subscribed to the log group with the specified filter pattern
@@ -117,105 +110,34 @@ validate_stack_resources() {
117110

118111
}
119112

113+
validate_stack_resources_without_subscription() {
114+
local stack_name=$1
115+
local log_group_name=$2
120116

121-
exit_with_error() {
122-
echo "Error: $1"
123-
exit 1
124-
}
125-
126-
create_log_event() {
127-
local log_group_name=$1
128-
local log_stream_name=$2
129-
local log_message=$3
130-
131-
log "Creating log event in CloudWatch Log Group: $log_group_name"
132-
local log_stream_exists=$(aws logs describe-log-streams --log-group-name "$log_group_name" --log-stream-name-prefix "$log_stream_name" --query "logStreams[?logStreamName=='$log_stream_name'] | length(@)" --output text)
133-
134-
if [ "$log_stream_exists" -eq 0 ]; then
135-
log "Creating log stream: $log_stream_name"
136-
aws logs create-log-stream --log-group-name "$log_group_name" --log-stream-name "$log_stream_name"
137-
fi
138-
139-
# Get the current timestamp in milliseconds
140-
timestamp=$(($(date +%s) * 1000 + $(date +%N) / 1000000))
141-
142-
# Put log event
143-
aws logs put-log-events \
144-
--log-group-name "$log_group_name" \
145-
--log-stream-name "$log_stream_name" \
146-
--log-events timestamp=$timestamp,message="$log_message"
147-
148-
log "Log event created successfully."
149117

150-
}
118+
log "Validating stack resources for stack: $stack_name"
119+
firehose_stream_physical_id=$(aws cloudformation describe-stack-resources \
120+
--stack-name "$stack_name" \
121+
--logical-resource-id "$FIREHOSE_STREAM_LOGICAL_ID" \
122+
--query "StackResources[0].PhysicalResourceId" \
123+
--output text
124+
)
151125

152-
validate_logs_in_new_relic() {
153-
local user_key=$1
154-
local account_id=$2
155-
local log_message=$3
156-
local common_attributes=$4
157-
local should_log_exist=$5
158-
159-
local nrql_query="SELECT * FROM Log WHERE message LIKE '%$log_message%' SINCE 10 minutes ago"
160-
local query='{"query":"query($id: Int!, $nrql: Nrql!) { actor { account(id: $id) { nrql(query: $nrql) { results } } } }","variables":{"id":'$account_id',"nrql":"'$nrql_query'"}}'
161-
162-
local log_message_exists=false
163-
164-
sleep_time=$SLEEP_TIME
165-
166-
for i in {1..5}; do
167-
local response=$(curl -s -X POST \
168-
-H "Content-Type: application/json" \
169-
-H "API-Key: $user_key" \
170-
-d "$query" \
171-
https://api.newrelic.com/graphql)
172-
173-
if echo "$response" | grep -q "$log_message"; then
174-
log "Log event successfully found in New Relic."
175-
log_message_exists=true
176-
validate_logs_meta_data "$response" "$common_attributes"
177-
break
178-
else
179-
log "Log event not found in New Relic. Retrying in $sleep_time seconds..."
180-
sleep $sleep_time
181-
sleep_time=$(( sleep_time * 2 ))
182-
fi
183-
done
126+
# Get the ARN of the Firehose delivery stream using the physical ID
127+
firehose_stream_arn=$(aws firehose describe-delivery-stream \
128+
--delivery-stream-name "$firehose_stream_physical_id" \
129+
--query "DeliveryStreamDescription.DeliveryStreamARN" \
130+
--output text
131+
)
184132

185-
if [ "$should_log_exist" == "true" ] && [ "$log_message_exists" == "false" ]; then
186-
exit_with_error "Log event not found in New Relic after 10 retries."
187-
elif [ "$should_log_exist" == "false" ] && [ "$log_message_exists" == "true" ]; then
188-
exit_with_error "Log event should not exist in New Relic, but it was found."
133+
# Check firehose_stream_arn is not null
134+
if [ -z "$firehose_stream_arn" ] || [ "$firehose_stream_arn" == "None" ]; then
135+
exit_with_error "Failed to retrieve Firehose delivery stream ARN for physical ID: $firehose_stream_physical_id"
189136
fi
190-
191137
}
192138

193-
validate_logs_meta_data (){
194-
local response=$1
195-
local common_attributes=$2
196-
197-
# Remove single quotes from common_attributes
198-
common_attributes=$(echo "$common_attributes" | sed "s/'//g")
199-
200-
# Validate common attributes
201-
for attribute in $(echo "$common_attributes" | jq -c '.[]'); do
202-
attribute_name=$(echo "$attribute" | jq -r '.AttributeName')
203-
attribute_value=$(echo "$attribute" | jq -r '.AttributeValue')
204-
if ! echo "$response" | grep -q "\"$attribute_name\":\"$attribute_value\""; then
205-
exit_with_error "Common attribute $attribute_name with value $attribute_value not found in New Relic logs."
206-
fi
207-
done
208-
log "Common attributes validated successfully."
209-
210-
# Read default attributes from config file and replace underscores with dots
211-
while IFS='=' read -r key value; do
212-
if [[ $key == instrumentation_* ]]; then
213-
new_key=$(echo "$key" | sed 's/_/./g')
214-
if ! echo "$response" | grep -q "\"$new_key\":\"$value\""; then
215-
exit_with_error "Entity synthesis attribute $new_key with value $value not found in New Relic logs."
216-
fi
217-
fi
218-
done < config-file.cfg
219139

220-
log "Entity synthesis parameter validated successfully."
140+
exit_with_error() {
141+
echo "Error: $1"
142+
exit 1
221143
}

‎e2e_tests/config-file.cfg

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ LOG_GROUP_NAME_2=aws-unified-firehose-e2e-test-log-group-2
1818
# Variable for third test case
1919
FIREHOSE_STACK_NAME_3=NewRelicFirehoseTestStackWithInvalidLogGroup
2020
INVALID_LOG_GROUP_NAME=invalid_log_group
21-
SLEEP_TIME=15
22-
# Entity synthesis Parameters
23-
instrumentation_provider=aws
24-
instrumentation_name=firehose
25-
instrumentation_version=1.0.0
21+
# Variable for fourth test case
22+
FIREHOSE_STACK_NAME_4=NewRelicFirehoseTestStackWithSecretsManagerFalse
23+
LOG_GROUP_NAME_4=aws-unified-firehose-e2e-test-log-group-3
24+
SLEEP_TIME=15

‎e2e_tests/entity_synthesis_param.cfg

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Entity synthesis Parameters
2+
instrumentation_provider=aws
3+
instrumentation_name=firehose
4+
instrumentation_version=1.0.0

‎e2e_tests/firehose_e2e_tests.sh

Lines changed: 63 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
source config-file.cfg
44
source common-scripts.sh
5+
source log_validation.sh
56

67
# Test Case 1:
78
# Create a Firehose stack without any subscription filter pattern
@@ -27,8 +28,8 @@ COMMON_ATTRIBUTES=$(<common_attribute.json)
2728
validate_stack_deployment_status "$FIREHOSE_STACK_NAME_1"
2829

2930
# Validate the stack resources
31+
validate_stack_resources_without_subscription "$FIREHOSE_STACK_NAME_1" "$LOG_GROUP_NAME_1"
3032

31-
validate_stack_resources "$FIREHOSE_STACK_NAME_1" "false" "$LOG_GROUP_NAME_1" ""
3233
# Generate a UUID and create a dynamic log message
3334
UUID=$(uuidgen)
3435
LOG_MESSAGE="RequestId: $UUID hello world"
@@ -70,7 +71,7 @@ COMMON_ATTRIBUTES=$(<common_attribute.json)
7071
validate_stack_deployment_status "$FIREHOSE_STACK_NAME_2"
7172

7273
# Validate the stack resources
73-
validate_stack_resources "$FIREHOSE_STACK_NAME_2" "false" "$LOG_GROUP_NAME_2" "$LOG_GROUP_FILTER_PATTERN"
74+
validate_stack_resources_with_subscription "$FIREHOSE_STACK_NAME_2" "$LOG_GROUP_NAME_2" "$LOG_GROUP_FILTER_PATTERN"
7475

7576
# Generate a UUID and create a dynamic log message with the filter pattern
7677
UUID=$(uuidgen)
@@ -119,6 +120,42 @@ LOG_GROUP_INVALID_JSON=$(<invalid_log_group.json)
119120
# Delete the Firehose stack
120121
delete_stack "$FIREHOSE_STACK_NAME_3"
121122
}
123+
124+
test_with_store_secret_manager_false() {
125+
local template_file=$TEMPLATE_FILE_FULL_PATH
126+
127+
cat <<EOF > log_group.json
128+
'[{"LogGroupName":"$LOG_GROUP_NAME_4"}]'
129+
EOF
130+
LOG_GROUP_JSON_4=$(<log_group.json)
131+
132+
cat <<EOF > common_attribute.json
133+
'[{"AttributeName":"$COMMON_ATTRIBUTE_KEY","AttributeValue":"$COMMON_ATTRIBUTE_VALUE"}]'
134+
EOF
135+
COMMON_ATTRIBUTES=$(<common_attribute.json)
136+
137+
# Deploy the Firehose stack
138+
deploy_firehose_stack "$template_file" "$FIREHOSE_STACK_NAME_4" "$NEW_RELIC_LICENSE_KEY" "$NEW_RELIC_REGION" "$NEW_RELIC_ACCOUNT_ID" "false" "$LOG_GROUP_JSON_4" "$COMMON_ATTRIBUTES"
139+
140+
# Validate the status of the Firehose stack
141+
validate_stack_deployment_status "$FIREHOSE_STACK_NAME_4"
142+
143+
# Validate the stack resources
144+
validate_stack_resources_without_subscription "$FIREHOSE_STACK_NAME_4" "$LOG_GROUP_NAME_1"
145+
146+
# Generate a UUID and create a dynamic log message
147+
UUID=$(uuidgen)
148+
LOG_MESSAGE="RequestId: $UUID hello world"
149+
150+
# Create a log event in CloudWatch Logs
151+
create_log_event "$LOG_GROUP_NAME_1" "$LOG_STREAM_NAME" "$LOG_MESSAGE"
152+
153+
# Validate logs in New Relic
154+
validate_logs_in_new_relic "$NEW_RELIC_USER_KEY" "$NEW_RELIC_ACCOUNT_ID" "$LOG_MESSAGE" "$COMMON_ATTRIBUTES" "true"
155+
156+
# Delete the Firehose stack
157+
delete_stack "$FIREHOSE_STACK_NAME_4"
158+
}
122159

123160

124161

@@ -129,20 +166,27 @@ BUILD_DIR="$BUILD_DIR_BASE/$BASE_NAME"
129166
sam build --template-file "../$TEMPLATE_FILE_NAME" --build-dir "$BUILD_DIR"
130167
sam package --s3-bucket "$S3_BUCKET" --template-file "$BUILD_DIR/template.yaml" --output-template-file "$BUILD_DIR/$TEMPLATE_FILE_NAME"
131168

132-
133-
Run the test cases
134-
case $1 in
135-
test-without-filter)
136-
test_logs_without_filter_pattern
137-
;;
138-
test-with-filter)
139-
test_logs_with_filter_pattern
140-
;;
141-
test-with-invalid-log-group)
142-
test_logs_with_invalid_log_group
143-
;;
144-
*)
145-
echo "Invalid test case specified."
146-
exit 1
147-
;;
148-
esac
169+
test_logs_without_filter_pattern &
170+
test_logs_with_filter_pattern &
171+
test_logs_with_invalid_log_group &
172+
test_with_store_secret_manager_false &
173+
174+
# Run the test cases
175+
# case $1 in
176+
# test-without-filter)
177+
# test_logs_without_filter_pattern
178+
# ;;
179+
# test-with-filter)
180+
# test_logs_with_filter_pattern
181+
# ;;
182+
# test-with-invalid-log-group)
183+
# test_logs_with_invalid_log_group
184+
# ;;
185+
# test-with-secret-manager-false)
186+
# test_with_store_secret_manager_false
187+
# ;;
188+
# *)
189+
# echo "Invalid test case specified."
190+
# exit 1
191+
# ;;
192+
# esac

0 commit comments

Comments
 (0)