Track delivered & failed notification counts (#2338)

* Track delivered & failed notification counts

- Increment daily delivered and failed notification counts for a service upon ses, sns, or pinpoint receipt received

* Bump utils version

* Update waffles version in WAF rule github action

Author: whabanks

.github/workflows/test.yaml

@@ -67,7 +67,7 @@ jobs:
       run: |
         cp -f .env.example .env
     - name: Checks for new endpoints against AWS WAF rules
-      uses: cds-snc/notification-utils/.github/actions/waffles@52.2.9
+      uses: cds-snc/notification-utils/.github/actions/waffles@52.3.6
       with:
         app-loc: '/github/workspace'
         app-libs: '/github/workspace/env/site-packages'

app/__init__.py

@@ -12,6 +12,7 @@
 from flask_migrate import Migrate
 from flask_redis import FlaskRedis
 from notifications_utils import logging, request_helper
+from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit
 from notifications_utils.clients.redis.bounce_rate import RedisBounceRate
 from notifications_utils.clients.redis.redis_client import RedisClient
 from notifications_utils.clients.statsd.statsd_client import StatsdClient
@@ -60,6 +61,7 @@
 flask_redis_publish = FlaskRedis(config_prefix="REDIS_PUBLISH")
 redis_store = RedisClient()
 bounce_rate_client = RedisBounceRate(redis_store)
+annual_limit_client = RedisAnnualLimit(redis_store)
 metrics_logger = MetricsLogger()
 # TODO: Rework instantiation to decouple redis_store.redis_store and pass it in.\
 email_queue = RedisQueue("email")

app/aws/mocks.py

@@ -73,6 +73,10 @@ def ses_soft_bounce_callback(reference, bounce_subtype=None):
     return _ses_bounce_callback(reference, "Transient", bounce_subtype)
 
 
+def ses_unknown_bounce_callback(reference, bounce_subtype=None):
+    return _ses_bounce_callback(reference, "unknown-bounce", bounce_subtype)
+
+
 def ses_complaint_callback_malformed_message_id():
     return {
         "Signature": "bb",

app/celery/process_pinpoint_receipts_tasks.py

@@ -5,7 +5,7 @@
 from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import notify_celery, statsd_client
+from app import annual_limit_client, notify_celery, statsd_client
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.models import (
@@ -43,6 +43,8 @@
 #     }
 
 
+# TODO FF_ANNUAL_LIMIT removal: Temporarily ignore complexity
+# flake8: noqa: C901
 @notify_celery.task(bind=True, name="process-pinpoint-result", max_retries=5, default_retry_delay=300)
 @statsd(namespace="tasks")
 def process_pinpoint_results(self, response):
@@ -111,10 +113,22 @@ def process_pinpoint_results(self, response):
                     f"Provider response: {provider_response}"
                 )
             )
+            # TODO FF_ANNUAL_LIMIT removal
+            if current_app.config["FF_ANNUAL_LIMIT"]:
+                annual_limit_client.increment_sms_failed(notification.service_id)
+                current_app.logger.info(
+                    f"Incremented sms_delivered count in Redis. Service: {notification.service_id} Notification: {notification.id} Current counts: {annual_limit_client.get_all_notification_counts(notification.service_id)}"
+                )
         else:
             current_app.logger.info(
                 f"Pinpoint callback return status of {notification_status} for notification: {notification.id}"
             )
+            # TODO FF_ANNUAL_LIMIT removal
+            if current_app.config["FF_ANNUAL_LIMIT"]:
+                annual_limit_client.increment_sms_delivered(notification.service_id)
+                current_app.logger.info(
+                    f"Incremented sms_delivered count in Redis. Service: {notification.service_id} Notification: {notification.id} Current counts: {annual_limit_client.get_all_notification_counts(notification.service_id)}"
+                )
 
         statsd_client.incr(f"callback.pinpoint.{notification_status}")
 

app/celery/process_ses_receipts_tasks.py

@@ -4,7 +4,7 @@
 from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import bounce_rate_client, notify_celery, statsd_client
+from app import annual_limit_client, bounce_rate_client, notify_celery, statsd_client
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.models import NOTIFICATION_DELIVERED, NOTIFICATION_PERMANENT_FAILURE
@@ -89,10 +89,20 @@ def process_ses_results(self, response):  # noqa: C901
                     notification.id, reference, aws_response_dict["message"]
                 )
             )
+            if current_app.config["FF_ANNUAL_LIMIT"]:
+                annual_limit_client.increment_email_failed(notification.service_id)
+                current_app.logger.info(
+                    f"Incremented email_failed count in Redis. Service: {notification.service_id} Notification: {notification.id} Current counts: {annual_limit_client.get_all_notification_counts(notification.service_id)}"
+                )
         else:
             current_app.logger.info(
                 "SES callback return status of {} for notification: {}".format(notification_status, notification.id)
             )
+            if current_app.config["FF_ANNUAL_LIMIT"]:
+                annual_limit_client.increment_email_delivered(notification.service_id)
+                current_app.logger.info(
+                    f"Incremented email_delivered count in Redis. Service: {notification.service_id} Notification: {notification.id} current counts: {annual_limit_client.get_all_notification_counts(notification.service_id)}"
+                )
 
         statsd_client.incr("callback.ses.{}".format(notification_status))
 

app/celery/process_sns_receipts_tasks.py

@@ -4,7 +4,7 @@
 from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import notify_celery, statsd_client
+from app import annual_limit_client, notify_celery, statsd_client
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.models import (
@@ -19,6 +19,8 @@
 from celery.exceptions import Retry
 
 
+# TODO: FF_ANNUAL_LIMIT removal: Temporarily ignore complexity
+# flake8: noqa: C901
 @notify_celery.task(bind=True, name="process-sns-result", max_retries=5, default_retry_delay=300)
 @statsd(namespace="tasks")
 def process_sns_results(self, response):
@@ -69,8 +71,20 @@ def process_sns_results(self, response):
                     f"Provider response: {sns_message['delivery']['providerResponse']}"
                 )
             )
+            # TODO FF_ANNUAL_LIMIT removal
+            if current_app.config["FF_ANNUAL_LIMIT"]:
+                annual_limit_client.increment_sms_failed(notification.service_id)
+                current_app.logger.info(
+                    f"Incremented sms_failed count in Redis. Service: {notification.service_id} Notification: {notification.id} Current counts: {annual_limit_client.get_all_notification_counts(notification.service_id)}"
+                )
         else:
             current_app.logger.info(f"SNS callback return status of {notification_status} for notification: {notification.id}")
+            # TODO FF_ANNUAL_LIMIT removal
+            if current_app.config["FF_ANNUAL_LIMIT"]:
+                annual_limit_client.increment_sms_delivered(notification.service_id)
+                current_app.logger.info(
+                    f"Incremented sms_delivered count in Redis. Service: {notification.service_id} Notification: {notification.id} Current counts: {annual_limit_client.get_all_notification_counts(notification.service_id)}"
+                )
 
         statsd_client.incr(f"callback.sns.{notification_status}")
 

pyproject.toml

@@ -66,6 +66,7 @@ MarkupSafe = "2.1.5"
 # REVIEW: v2 is using sha512 instead of sha1 by default (in v1)
 itsdangerous = "2.2.0"
 notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "52.3.6" }
+
 # rsa = "4.9  # awscli 1.22.38 depends on rsa<4.8
 typing-extensions = "4.12.2"
 greenlet = "2.0.2"

tests/app/celery/test_process_pinpoint_receipts_tasks.py

@@ -3,7 +3,7 @@
 import pytest
 from freezegun import freeze_time
 
-from app import statsd_client
+from app import annual_limit_client, statsd_client
 from app.aws.mocks import (
     pinpoint_delivered_callback,
     pinpoint_delivered_callback_missing_sms_data,
@@ -28,6 +28,7 @@
     create_service_callback_api,
     save_notification,
 )
+from tests.conftest import set_config
 
 
 @pytest.mark.parametrize(
@@ -278,3 +279,77 @@ def test_process_pinpoint_results_calls_service_callback(sample_template, notify
         mock_send_status.assert_called_once_with(
             [str(notification.id), signed_data, notification.service_id], queue="service-callbacks"
         )
+
+
+@pytest.mark.parametrize(
+    "provider_response",
+    [
+        "Blocked as spam by phone carrier",
+        "Destination is on a blocked list",
+        "Invalid phone number",
+        "Message body is invalid",
+        "Phone carrier has blocked this message",
+        "Phone carrier is currently unreachable/unavailable",
+        "Phone has blocked SMS",
+        "Phone is on a blocked list",
+        "Phone is currently unreachable/unavailable",
+        "Phone number is opted out",
+        "This delivery would exceed max price",
+        "Unknown error attempting to reach phone",
+    ],
+)
+def test_process_pinpoint_results_should_increment_sms_failed_when_delivery_receipt_is_failure(
+    sample_sms_template_with_html,
+    notify_api,
+    mocker,
+    provider_response,
+):
+    mocker.patch("app.annual_limit_client.increment_sms_delivered")
+    mocker.patch("app.annual_limit_client.increment_sms_failed")
+
+    notification = save_notification(
+        create_notification(
+            sample_sms_template_with_html,
+            reference="ref",
+            sent_at=datetime.utcnow(),
+            status=NOTIFICATION_SENT,
+            sent_by="pinpoint",
+        )
+    )
+    # TODO FF_ANNUAL_LIMIT removal
+    with set_config(notify_api, "FF_ANNUAL_LIMIT", True):
+        process_pinpoint_results(pinpoint_failed_callback(reference="ref", provider_response=provider_response))
+        annual_limit_client.increment_sms_failed.assert_called_once_with(notification.service_id)
+        annual_limit_client.increment_sms_delivered.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    "callback",
+    [
+        (pinpoint_delivered_callback),
+        (pinpoint_shortcode_delivered_callback),
+    ],
+)
+def test_process_pinpoint_results_should_increment_sms_delivered_when_delivery_receipt_is_success(
+    sample_sms_template_with_html,
+    notify_api,
+    mocker,
+    callback,
+):
+    mocker.patch("app.annual_limit_client.increment_sms_delivered")
+    mocker.patch("app.annual_limit_client.increment_sms_failed")
+
+    notification = save_notification(
+        create_notification(
+            sample_sms_template_with_html,
+            reference="ref",
+            sent_at=datetime.utcnow(),
+            status=NOTIFICATION_SENT,
+            sent_by="pinpoint",
+        )
+    )
+    # TODO FF_ANNUAL_LIMIT removal
+    with set_config(notify_api, "FF_ANNUAL_LIMIT", True):
+        process_pinpoint_results(callback(reference="ref"))
+        annual_limit_client.increment_sms_delivered.assert_called_once_with(notification.service_id)
+        annual_limit_client.increment_sms_failed.assert_not_called()

tests/app/celery/test_process_ses_receipts_tasks.py

@@ -4,8 +4,8 @@
 import pytest
 from freezegun import freeze_time
 
-from app import bounce_rate_client, signer_complaint, statsd_client
-from app.aws.mocks import ses_complaint_callback
+from app import annual_limit_client, bounce_rate_client, signer_complaint, statsd_client
+from app.aws.mocks import ses_complaint_callback, ses_unknown_bounce_callback
 from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.celery.research_mode_tasks import (
     ses_hard_bounce_callback,
@@ -26,6 +26,7 @@
     NOTIFICATION_SOFT_GENERAL,
     NOTIFICATION_SOFT_MAILBOXFULL,
     NOTIFICATION_SOFT_MESSAGETOOLARGE,
+    NOTIFICATION_UNKNOWN_BOUNCE,
     Complaint,
     Notification,
 )
@@ -41,6 +42,7 @@
     create_service_callback_api,
     save_notification,
 )
+from tests.conftest import set_config
 
 
 def test_process_ses_results(sample_email_template):
@@ -436,3 +438,41 @@ def test_ses_callback_should_not_add_redis_keys_when_delivery_receipt_is_soft_bo
 
         bounce_rate_client.set_sliding_hard_bounce.assert_not_called()
         bounce_rate_client.set_sliding_notifications.assert_not_called()
+
+
+class TestAnnualLimits:
+    def test_ses_callback_should_increment_email_delivered_when_delivery_receipt_is_delivered(
+        self, notify_api, sample_email_template, mocker
+    ):
+        mocker.patch("app.annual_limit_client.increment_email_delivered")
+        mocker.patch("app.annual_limit_client.increment_email_failed")
+
+        # TODO FF_ANNUAL_LIMIT removal
+        with set_config(notify_api, "FF_ANNUAL_LIMIT", True):
+            save_notification(create_notification(template=sample_email_template, reference="ref", status="sending"))
+
+            assert process_ses_results(ses_notification_callback(reference="ref"))
+            annual_limit_client.increment_email_delivered.assert_called_once_with(sample_email_template.service_id)
+            annual_limit_client.increment_email_failed.assert_not_called()
+
+    @pytest.mark.parametrize(
+        "callback, bounce_type",
+        [
+            (ses_hard_bounce_callback, NOTIFICATION_HARD_BOUNCE),
+            (ses_soft_bounce_callback, NOTIFICATION_SOFT_BOUNCE),
+            (ses_unknown_bounce_callback, NOTIFICATION_UNKNOWN_BOUNCE),
+        ],
+    )
+    def test_ses_callback_should_increment_email_failed_when_delivery_receipt_is_failure(
+        self, notify_api, sample_email_template, mocker, callback, bounce_type
+    ):
+        mocker.patch("app.annual_limit_client.increment_email_failed")
+        mocker.patch("app.annual_limit_client.increment_email_delivered")
+
+        # TODO FF_ANNUAL_LIMIT removal
+        with set_config(notify_api, "FF_ANNUAL_LIMIT", True):
+            save_notification(create_notification(template=sample_email_template, reference="ref", status="sending"))
+
+            assert process_ses_results(callback(reference="ref"))
+            annual_limit_client.increment_email_failed.assert_called_once_with(sample_email_template.service_id)
+            annual_limit_client.increment_email_delivered.assert_not_called()