Merge pull request #2119 from govuk-forms/add-alerts-for-bounce-notifications-jobs

Add alerts for bounce notifications jobs

Author: stephencdaly

infra/deployments/forms/health/alerts/failed-bounce-notifications-job-executions.tf

@@ -0,0 +1,29 @@
+resource "aws_cloudwatch_metric_alarm" "failed_bounce_notifications_job_executions" {
+  alarm_name          = "${var.environment}-failed-bounce-notifications-job-executions"
+  alarm_description   = <<EOF
+There are one or more failed job executions for the bounce_notifications queue in forms-runner in the ${var.environment}
+environment that will not automatically be retried.
+
+Follow the runbook to respond to this: https://github.com/govuk-forms/forms-team/wiki/Runbooks#handle-failed-solid-queue-jobs
+
+Only close this Zendesk ticket when we get another Zendesk ticket to tell us that the alarm is in the OK state.
+EOF
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  namespace           = "Forms/Jobs"
+  metric_name         = "FailedJobExecutions"
+  statistic           = "Minimum"
+  period              = 10 * 60 # 10 minutes
+  threshold           = 1
+
+  dimensions = {
+    Environment = "${var.environment}"
+    ServiceName = "forms-runner"
+    QueueName   = "bounce_notifications"
+  }
+
+  treat_missing_data = "breaching"
+
+  alarm_actions = [local.alert_severity.eu_west_2.info]
+  ok_actions    = [local.alert_severity.eu_west_2.info]
+}

infra/deployments/forms/health/alerts/schedule-bounce-notifications-job-not-run.tf

@@ -0,0 +1,32 @@
+resource "aws_cloudwatch_metric_alarm" "schedule_bounce_notifications_job_not_run" {
+  alarm_name          = "${var.environment}-schedule-bounce-notifications-job-not-run"
+  alarm_description   = <<EOF
+    The forms-runner job to schedule sending notification emails to admin users about bounced submissions deliveries has
+    not run in the ${var.environment} environment in the past 25 hours. It is expected that the job is run every day.
+    This job is run by Solid Queue, which is started in the forms-runner-queue-worker ECS task.
+
+    NEXT STEPS:
+    1. Check the Splunk logs and Sentry for any errors running the job.
+    2. Restart the forms-runner-queue-worker ECS tasks and check whether the job starts running.
+    3. Check that there haven't been any bounced submission deliveries that we should have sent notifications for in the time the job hasn't been running. If there are, make sure we send the notifications.
+
+EOF
+  comparison_operator = "LessThanOrEqualToThreshold"
+  evaluation_periods  = 25
+  metric_name         = "Started"
+  namespace           = "Forms/Jobs"
+  period              = 3600
+  statistic           = "SampleCount"
+  threshold           = 0
+
+  dimensions = {
+    Environment = "${var.environment}"
+    ServiceName = "forms-runner"
+    JobName     = "ScheduleBounceNotificationsJob"
+  }
+
+  treat_missing_data = "breaching"
+
+  alarm_actions = [local.alert_severity.eu_west_2.info]
+  ok_actions    = [local.alert_severity.eu_west_2.info]
+}