Skip to content

Commit 8dd806c

Browse files
whabanksjzbahraismcmurtry
authored
Batch save ses receipts - V2 (#2518)
* Batch save ses receipts * Batch save ses receipts * Remove json.load call as we handle that in the ses_to_sqs_email_callbacks lambda * Fix tests * Fix one more test * Add logging when an uncaught exception occurs - Added doc strings - Improve clarity of existing comments * Fix typo * Add some debug logging for testing in staging * Handle json loads in celery and fix retry bug - Moved the json loading out of the lambda to improve lambda performance. It is now handled here in the celery task - Fixed an issue where the task retry args were getting wrapped in a list when the arg was already a list, causing retries to fail processing * Revert json loading change * Fix test * refactor process_ses_results * fix type annotation * Move retry logic further down, correcting redis counts * handle the existing messages in the queue when the deploy happens --------- Co-authored-by: Jumana B <[email protected]> Co-authored-by: Stephen McMurtry <[email protected]>
1 parent a2831c4 commit 8dd806c

File tree

6 files changed

+406
-153
lines changed

6 files changed

+406
-153
lines changed

‎app/aws/mocks.py‎

Lines changed: 72 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,70 @@ def sns_s3_callback(filename, message_id="some-message-id"):
1212
)
1313

1414

15+
def generate_ses_notification_callbacks(references, with_complaints=False):
16+
messages = []
17+
18+
for i, ref in enumerate(references):
19+
ses_message_body = {
20+
"delivery": {
21+
"processingTimeMillis": 2003,
22+
"recipients": ["[email protected]"],
23+
"remoteMtaIp": "123.123.123.123",
24+
"reportingMTA": "a7-32.smtp-out.eu-west-1.amazonses.com",
25+
"smtpResponse": "250 2.6.0 Message received",
26+
"timestamp": "2017-11-17T12:14:03.646Z",
27+
},
28+
"mail": {
29+
"commonHeaders": {
30+
"from": ["TEST <[email protected]>"],
31+
"subject": "lambda test",
32+
"to": ["[email protected]"],
33+
},
34+
"destination": ["[email protected]"],
35+
"headers": [
36+
{"name": "From", "value": "TEST <[email protected]>"},
37+
{"name": "To", "value": "[email protected]"},
38+
{"name": "Subject", "value": "lambda test"},
39+
{"name": "MIME-Version", "value": "1.0"},
40+
{
41+
"name": "Content-Type",
42+
"value": 'multipart/alternative; boundary="----=_Part_617203_1627511946.1510920841645"',
43+
},
44+
],
45+
"headersTruncated": False,
46+
"messageId": ref,
47+
"sendingAccountId": "12341234",
48+
"source": '"TEST" <[email protected]>',
49+
"sourceArn": "arn:aws:ses:eu-west-1:12341234:identity/notify.works",
50+
"sourceIp": "0.0.0.1",
51+
"timestamp": "2017-11-17T12:14:01.643Z",
52+
},
53+
"notificationType": "Delivery",
54+
}
55+
if with_complaints and i % 2 == 1:
56+
ses_message_body["notificationType"] = "Complaint"
57+
ses_message_body["complaint"] = {
58+
"complaintFeedbackType": "abuse",
59+
"complainedRecipients": [{"emailAddress": "[email protected]"}],
60+
"timestamp": "2018-06-05T13:59:58.000Z",
61+
"feedbackId": "ses_feedback_id",
62+
}
63+
messages.append(ses_message_body)
64+
return {
65+
"Type": "Notification",
66+
"MessageId": "8e83c020-1234-1234-1234-92a8ee9baa0a",
67+
"TopicArn": "arn:aws:sns:eu-west-1:12341234:ses_notifications",
68+
"Subject": None,
69+
"Messages": json.loads(json.dumps(messages)),
70+
"Timestamp": "2017-11-17T12:14:03.710Z",
71+
"SignatureVersion": "1",
72+
"Signature": "[REDACTED]",
73+
"SigningCertUrl": "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem",
74+
"UnsubscribeUrl": "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]",
75+
"MessageAttributes": {},
76+
}
77+
78+
1579
def ses_notification_callback(reference):
1680
ses_message_body = {
1781
"delivery": {
@@ -55,7 +119,7 @@ def ses_notification_callback(reference):
55119
"MessageId": "8e83c020-1234-1234-1234-92a8ee9baa0a",
56120
"TopicArn": "arn:aws:sns:eu-west-1:12341234:ses_notifications",
57121
"Subject": None,
58-
"Message": json.dumps(ses_message_body),
122+
"Messages": [ses_message_body],
59123
"Timestamp": "2017-11-17T12:14:03.710Z",
60124
"SignatureVersion": "1",
61125
"Signature": "[REDACTED]",
@@ -88,7 +152,7 @@ def ses_complaint_callback_malformed_message_id():
88152
"Type": "Notification",
89153
"Timestamp": "2018-06-05T14:00:15.952Z",
90154
"Subject": None,
91-
"Message": '{"notificationType":"Complaint","complaint":{"complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id"},"mail":{"timestamp":"2018-06-05T14:00:15.950Z","source":"\\"Some Service\\" <someservicenotifications.service.gov.uk>","sourceArn":"arn:identity/notifications.service.gov.uk","sourceIp":"52.208.24.161","sendingAccountId":"888450439860","badMessageId":"ref1","destination":["[email protected]"]}}', # noqa
155+
"Messages": '{"notificationType":"Complaint","complaint":{"complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id"},"mail":{"timestamp":"2018-06-05T14:00:15.950Z","source":"\\"Some Service\\" <someservicenotifications.service.gov.uk>","sourceArn":"arn:identity/notifications.service.gov.uk","sourceIp":"52.208.24.161","sendingAccountId":"888450439860","badMessageId":"ref1","destination":["[email protected]"]}}', # noqa
92156
"SigningCertUrl": "https://sns.pem",
93157
}
94158

@@ -107,7 +171,7 @@ def ses_complaint_callback_with_missing_complaint_type():
107171
"Type": "Notification",
108172
"Timestamp": "2018-06-05T14:00:15.952Z",
109173
"Subject": None,
110-
"Message": '{"notificationType":"Complaint","complaint":{"complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id"},"mail":{"timestamp":"2018-06-05T14:00:15.950Z","source":"\\"Some Service\\" <someservicenotifications.service.gov.uk>","sourceArn":"arn:identity/notifications.service.gov.uk","sourceIp":"52.208.24.161","sendingAccountId":"888450439860","messageId":"ref1","destination":["[email protected]"]}}', # noqa
174+
"Messages": '{"notificationType":"Complaint","complaint":{"complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id"},"mail":{"timestamp":"2018-06-05T14:00:15.950Z","source":"\\"Some Service\\" <someservicenotifications.service.gov.uk>","sourceArn":"arn:identity/notifications.service.gov.uk","sourceIp":"52.208.24.161","sendingAccountId":"888450439860","messageId":"ref1","destination":["[email protected]"]}}', # noqa
111175
"SigningCertUrl": "https://sns.pem",
112176
}
113177

@@ -126,7 +190,9 @@ def ses_complaint_callback():
126190
"Type": "Notification",
127191
"Timestamp": "2018-06-05T14:00:15.952Z",
128192
"Subject": None,
129-
"Message": '{"notificationType":"Complaint","complaint":{"complaintFeedbackType": "abuse", "complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id"},"mail":{"timestamp":"2018-06-05T14:00:15.950Z","source":"\\"Some Service\\" <someservicenotifications.service.gov.uk>","sourceArn":"arn:identity/notifications.service.gov.uk","sourceIp":"52.208.24.161","sendingAccountId":"888450439860","messageId":"ref1","destination":["[email protected]"]}}', # noqa
193+
"Messages": json.loads(
194+
'[{"notificationType":"Complaint","complaint":{"complaintFeedbackType": "abuse", "complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id"},"mail":{"timestamp":"2018-06-05T14:00:15.950Z","source":"\\"Some Service\\" <someservicenotifications.service.gov.uk>","sourceArn":"arn:identity/notifications.service.gov.uk","sourceIp":"52.208.24.161","sendingAccountId":"888450439860","messageId":"ref1","destination":["[email protected]"]}}]'
195+
), # noqa
130196
"SigningCertUrl": "https://sns.pem",
131197
}
132198

@@ -145,7 +211,7 @@ def ses_complaint_callback_with_subtype(subtype):
145211
"Type": "Notification",
146212
"Timestamp": "2018-06-05T14:00:15.952Z",
147213
"Subject": None,
148-
"Message": '{"notificationType":"Complaint","complaint":{"complaintFeedbackType": "abuse", "complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id", "complaintSubType":"'
214+
"Messages": '{"notificationType":"Complaint","complaint":{"complaintFeedbackType": "abuse", "complainedRecipients":[{"emailAddress":"[email protected]"}],"timestamp":"2018-06-05T13:59:58.000Z","feedbackId":"ses_feedback_id", "complaintSubType":"'
149215
+ subtype
150216
+ '"},"mail":{"timestamp":"2018-06-05T14:00:15.950Z","source":"\\"Some Service\\" <someservicenotifications.service.gov.uk>","sourceArn":"arn:identity/notifications.service.gov.uk","sourceIp":"52.208.24.161","sendingAccountId":"888450439860","messageId":"ref1","destination":["[email protected]"]}}', # noqa
151217
"SigningCertUrl": "https://sns.pem",
@@ -370,7 +436,7 @@ def _ses_bounce_callback(reference, bounce_type, bounce_subtype=None):
370436
"MessageId": "36e67c28-1234-1234-1234-2ea0172aa4a7",
371437
"TopicArn": "arn:aws:sns:eu-west-1:12341234:ses_notifications",
372438
"Subject": None,
373-
"Message": json.dumps(ses_message_body),
439+
"Messages": [ses_message_body],
374440
"Timestamp": "2017-11-17T12:14:05.149Z",
375441
"SignatureVersion": "1",
376442
"Signature": "[REDACTED]", # noqa

0 commit comments

Comments
 (0)