Better nightly performance test (#2075)

Author: sastels

bin/execute_and_publish_performance_test.sh

@@ -1,13 +1,24 @@
 #!/bin/bash
 
+# Setup
 current_time=$(date "+%Y.%m.%d-%H.%M.%S")
 perf_test_aws_s3_bucket=${PERF_TEST_AWS_S3_BUCKET:-notify-performance-test-results-staging}
 perf_test_csv_directory_path=${PERF_TEST_CSV_DIRECTORY_PATH:-/tmp/notify_performance_test}
-
 mkdir -p $perf_test_csv_directory_path/$current_time
 
+# Run old performance test and copy results to S3
 locust --headless --config tests-perf/locust/locust.conf --html $perf_test_csv_directory_path/$current_time/index.html --csv $perf_test_csv_directory_path/$current_time/perf_test
-
 aws s3 cp $perf_test_csv_directory_path/ "s3://$perf_test_aws_s3_bucket" --recursive || exit 1
 
+# Sleep 15 minutes to allow the system to stabilize
+sleep 900
+
+# Run email send rate performance test
+# This configuration should send 10K emails / minute for 10 minutes for 100K emails total.
+# We run this test on Tuesday through Friday (just after midnight UTC) only.
+if [ "$(date +%u)" -ge 2 ] && [ "$(date +%u)" -le 5 ]; then
+    locust --headless --host https://api.staging.notification.cdssandbox.xyz --locustfile tests-perf/locust/send_rate_email.py --users 5 --run-time 10m --spawn-rate 1
+fi
+
+# Cleanup
 rm -rf $perf_test_csv_directory_path/$current_time

tests-perf/locust/README.md

@@ -36,7 +36,7 @@ You should not have to modify the configuration to run the stress-tests locally.
 
 There are two ways to run Locust, with the UI or headless.
 
-### Add the following to your .env file (ask a coworker):
+### Add the following to your .env file (see 1Password):
 
 ```
 PERF_TEST_AUTH_HEADER =
@@ -67,6 +67,13 @@ locust -f .\locust-notifications.py --headless --users=5500 --spawn-rate=200 --r
 
 You can also modify the *locust.config* file to enable the headless mode and define the necessary users, spawn rate and run time.
 
+## Email send rate test
+
+We also max out the email send rate by sending 2000 x 5 emails per minute for 10 minutes. This can be run manually with the command
+```
+locust --headless --host https://api.staging.notification.cdssandbox.xyz --locustfile tests-perf/locust/send_rate_email.py --users 5 --run-time 10m --spawn-rate 1
+```
+
 ### Performance Testing on AWS
 
 We run Notify performance tests on a daily manner through AWS ECS tasks

tests-perf/locust/send_rate_email.py

@@ -0,0 +1,60 @@
+""" send_rate_email.py
+    isort:skip_file
+"""
+# flake8: noqa
+
+BULK_EMAIL_SIZE = 2000
+
+import os
+import sys
+from datetime import datetime
+from dataclasses import make_dataclass
+
+sys.path.append(os.path.abspath(os.path.join("..", "tests_smoke")))
+
+from dotenv import load_dotenv
+from locust import HttpUser, constant_pacing, task
+from tests_smoke.smoke.common import job_line, rows_to_csv  # type: ignore
+
+load_dotenv()
+NotifyApiUserTemplateGroup = make_dataclass('NotifyApiUserTemplateGroup', [
+    'bulk_email_id',
+    'email_id',
+    'email_with_attachment_id',
+    'email_with_link_id',
+    'sms_id',
+])
+
+
+class NotifyApiUser(HttpUser):
+
+    wait_time = constant_pacing(60)  # 60 seconds between each task
+    host = os.getenv("PERF_TEST_DOMAIN", "https://api.staging.notification.cdssandbox.xyz")
+
+    def __init__(self, *args, **kwargs):
+        super(NotifyApiUser, self).__init__(*args, **kwargs)
+
+        self.headers = {"Authorization": os.getenv("PERF_TEST_AUTH_HEADER")}
+        self.email = os.getenv("PERF_TEST_EMAIL", "[email protected]")
+        self.phone_number = os.getenv("PERF_TEST_PHONE_NUMBER", "16135550123")
+        self.template_group = NotifyApiUserTemplateGroup(
+            bulk_email_id=os.getenv("PERF_TEST_BULK_EMAIL_TEMPLATE_ID"),
+            email_id=os.getenv("PERF_TEST_EMAIL_TEMPLATE_ID"),
+            email_with_attachment_id=os.getenv("PERF_TEST_EMAIL_WITH_ATTACHMENT_TEMPLATE_ID"),
+            email_with_link_id=os.getenv("PERF_TEST_EMAIL_WITH_LINK_TEMPLATE_ID"),
+            sms_id=os.getenv("PERF_TEST_SMS_TEMPLATE_ID"),
+        )
+
+    @task(1)
+    def send_bulk_email_notifications(self):
+        """
+        Send BULK_EMAIL_SIZE emails through the /bulk endpoint
+        """
+
+        json = {
+            "name": f"Send rate test {datetime.utcnow().isoformat()}",
+            "template_id": self.template_group.bulk_email_id,
+            "csv": rows_to_csv([["email address", "application_file"], *job_line(self.email, BULK_EMAIL_SIZE)])
+        }
+
+        self.client.post("/v2/notifications/bulk", json=json, headers=self.headers)