feat: turn dev alerting off entirely, not just its delivery

alerts_enabled gated the SNS subscription alone, so dev built every alarm
and every AMP rule and simply dropped what they produced. Nobody acts on a
dev page, and a hibernated environment would sit permanently red.

The recording rules and the delivery path stay in both environments: the
dashboards query the first, and keeping the second is what makes turning
alerts back on a one-line flip rather than a wait on the webhook secret's
recovery window.

Costs dev's validation of the alert rule YAML, which AMP does server-side
when the rule group applies — prod is now the first apply to parse a new
rule.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: daithihearn

terraform/README.md

@@ -60,10 +60,20 @@ metric name, and a wildcard also matches the `ALL` aggregate and the managed gro
 which double-counts. `CountedRequests` overlaps `AllowedRequests` for the same reason — a count-mode
 match does not terminate evaluation — so the two never belong in one stack.
 
-Alerts evaluate in every environment but `alerts_enabled` in the env YAML controls whether they are
-delivered, and dev has it off. To rehearse the path end to end: set the `SLACK_WEBHOOK_URL` secret on
-the `dev` GitHub Environment, flip `alerts_enabled` to `true`, scale the service to zero, and confirm
-the no-healthy-targets alarm arrives and then recovers.
+`alerts_enabled` in the env YAML turns alerting on, and dev has it off — no CloudWatch alarms, no AMP
+alert rule group, no Alertmanager definition, and Grafana's Alerting UI is not wired to any of them.
+Nobody acts on a dev page, and dev is hibernated often enough that alarms there would read as
+permanently red. The recording rules are separate and always built, because the dashboards query them.
+
+So is the delivery path — topic, bridge Lambda and webhook secret — which costs nothing idle and is
+what keeps re-enabling a one-line flip rather than a wait on that secret's seven-day recovery window.
+To rehearse end to end: set the `SLACK_WEBHOOK_URL` secret on the `dev` GitHub Environment, flip
+`alerts_enabled` to `true`, deploy, scale the service to zero, and confirm the no-healthy-targets
+alarm arrives and then recovers.
+
+The cost of dev not evaluating the AMP rules is that **prod is now the first apply to parse a new
+one** — `aws_prometheus_rule_group_namespace` is what validates the YAML and its PromQL, server-side.
+Rehearsing in dev before touching `alert_rules_yaml` is the cheap way round it.
 
 Grafana sign-in is Okta SAML, configured from `grafana_okta_saml_metadata_url` and the two
 `grafana_*_okta_groups` lists in the env YAML. Blank the URL, or leave the Admin list empty, and the

terraform/environments/dev/dev.yaml

@@ -37,7 +37,7 @@ grafana_okta_saml_metadata_url: https://vechaineu.okta.com/app/exk1020c50lQfO4EF
 grafana_admin_okta_groups: [APP_AMG_BLOCK_EXPLORER_Admin]
 grafana_editor_okta_groups: [APP_AMG_BLOCK_EXPLORER_Editor]
 
-# Rules and alarms evaluate either way; this only gates Slack delivery.
+# Off entirely: no alarms, no AMP alert rules, no Alertmanager.
 alerts_enabled: false
 
 # WAF. The rate-based IP rule always blocks; the AWS managed groups stay in

terraform/observability-aws/alarms.tf

@@ -54,7 +54,7 @@ locals {
 # Running against desired, so autoscaling never has to be restated here.
 # `breaching` is load-bearing: a service reporting nothing is what this catches.
 resource "aws_cloudwatch_metric_alarm" "ecs_tasks_below_desired" {
-  for_each = local.monitored_services
+  for_each = local.alerts_enabled ? local.monitored_services : {}
 
   alarm_name        = "${local.name}-${each.key}-tasks-below-desired"
   alarm_description = "${each.value} tasks below desired — ECS is running fewer tasks than the service asks for, or has stopped reporting task counts altogether."
@@ -109,7 +109,7 @@ resource "aws_cloudwatch_metric_alarm" "ecs_tasks_below_desired" {
 # Catches RUNNING-but-wedged, which the task-count alarm sits through. Maximum,
 # not Minimum: the metric is per-AZ, so Minimum false-fires below one task per AZ.
 resource "aws_cloudwatch_metric_alarm" "alb_no_healthy_targets" {
-  count = local.alb_arn_suffix == null || local.target_group_arn_suffix == null ? 0 : 1
+  count = local.alerts_enabled && local.alb_arn_suffix != null && local.target_group_arn_suffix != null ? 1 : 0
 
   alarm_name        = "${local.name}-no-healthy-targets"
   alarm_description = "Explorer has no healthy targets — the load balancer has nothing to route to."
@@ -136,6 +136,8 @@ resource "aws_cloudwatch_metric_alarm" "alb_no_healthy_targets" {
 # publishes to the topic whose consumer is broken — so read this in CloudWatch or
 # Grafana rather than expecting it in Slack.
 resource "aws_cloudwatch_metric_alarm" "alert_delivery_failing" {
+  count = local.alerts_enabled ? 1 : 0
+
   alarm_name        = "${local.name}-alert-delivery-failing"
   alarm_description = "Alert delivery is failing — the SNS to Slack bridge is erroring, so rules and alarms are firing into nothing."
 
@@ -156,7 +158,7 @@ resource "aws_cloudwatch_metric_alarm" "alert_delivery_failing" {
 # The load balancer's own 5xx, not the target's: 502/503 with nothing healthy to
 # route to, 504 on the idle timeout.
 resource "aws_cloudwatch_metric_alarm" "alb_elb_5xx" {
-  count = local.alb_arn_suffix == null ? 0 : 1
+  count = local.alerts_enabled && local.alb_arn_suffix != null ? 1 : 0
 
   alarm_name        = "${local.name}-alb-5xx"
   alarm_description = "ALB returning 5xx — 502/503 with no healthy target to route to, or 504 from the idle timeout."
@@ -178,7 +180,7 @@ resource "aws_cloudwatch_metric_alarm" "alb_elb_5xx" {
 # What the app actually returned, counted at the load balancer. The app's own
 # 5xx rule in AMP is the inside view of the same failure and stays alongside it.
 resource "aws_cloudwatch_metric_alarm" "alb_target_5xx" {
-  count = local.alb_arn_suffix == null || local.target_group_arn_suffix == null ? 0 : 1
+  count = local.alerts_enabled && local.alb_arn_suffix != null && local.target_group_arn_suffix != null ? 1 : 0
 
   alarm_name        = "${local.name}-target-5xx"
   alarm_description = "High target 5xx rate — the explorer has been returning more than 1 server error per second for over 5 minutes."
@@ -204,7 +206,7 @@ resource "aws_cloudwatch_metric_alarm" "alb_target_5xx" {
 # Published as a per-minute Sum, so the ceiling is the per-second cap times 60. At
 # the cap Valkey throttles rather than queues, which the app reads as a plain miss.
 resource "aws_cloudwatch_metric_alarm" "cache_ecpu_high" {
-  count = local.cache_name == null ? 0 : 1
+  count = local.alerts_enabled && local.cache_name != null ? 1 : 0
 
   alarm_name        = "${local.name}-cache-ecpu-high"
   alarm_description = "Valkey ECPU usage is high — the shared cache has been above ${format("%.0f", local.saturation_threshold * 100)}% of its configured ECPU ceiling for five minutes, and commands are throttled once it reaches it."
@@ -228,7 +230,7 @@ resource "aws_cloudwatch_metric_alarm" "cache_ecpu_high" {
 # A serverless cache exposes no eviction policy, so entries live their full TTL and
 # the working set is "distinct keys per day" rather than a number anyone picked.
 resource "aws_cloudwatch_metric_alarm" "cache_storage_high" {
-  count = local.cache_name == null ? 0 : 1
+  count = local.alerts_enabled && local.cache_name != null ? 1 : 0
 
   alarm_name        = "${local.name}-cache-storage-high"
   alarm_description = "Valkey storage is high — the shared cache has held more than ${format("%.0f", local.saturation_threshold * 100)}% of its configured storage ceiling for fifteen minutes. Entries expire but are never evicted, so the TTLs are what have to come down."
@@ -251,7 +253,7 @@ resource "aws_cloudwatch_metric_alarm" "cache_storage_high" {
 # Reads either way round: a real flood, or a managed rule group false-positiving
 # on legitimate traffic once it is flipped from Count to Block.
 resource "aws_cloudwatch_metric_alarm" "waf_blocked_requests" {
-  count = local.waf_web_acl_name == null ? 0 : 1
+  count = local.alerts_enabled && local.waf_web_acl_name != null ? 1 : 0
 
   alarm_name        = "${local.name}-waf-blocking-heavily"
   alarm_description = "WAF is blocking heavily — more than 5 requests per second have been blocked for over 10 minutes. Check the WAF log group for whether the source is hostile or a false positive."

terraform/observability-aws/alerts.tf

@@ -158,8 +158,7 @@ resource "aws_lambda_permission" "sns_to_slack_invoke" {
   source_arn    = aws_sns_topic.alerts.arn
 }
 
-# Both pipelines publish to the one topic, so this subscription is the single gate
-# on delivery. Rules and alarms evaluate and show their state without it.
+# The topic, bridge and secret are built either way; only this is gated.
 resource "aws_sns_topic_subscription" "sns_to_slack" {
   count = local.alerts_enabled ? 1 : 0
 
@@ -188,13 +187,18 @@ resource "aws_prometheus_rule_group_namespace" "recording" {
   data         = local.recording_rules_yaml
 }
 
+# Gated, unlike the recording rules above: the dashboards query those.
 resource "aws_prometheus_rule_group_namespace" "alerts" {
+  count = local.alerts_enabled ? 1 : 0
+
   workspace_id = aws_prometheus_workspace.this.id
   name         = "${local.name}-alerts"
   data         = local.alert_rules_yaml
 }
 
 resource "aws_prometheus_alert_manager_definition" "this" {
+  count = local.alerts_enabled ? 1 : 0
+
   workspace_id = aws_prometheus_workspace.this.id
   definition   = local.alertmanager_definition
 }

terraform/observability-aws/locals.tf

@@ -6,7 +6,7 @@ locals {
 
   okta_saml_ready = local.env.grafana_okta_saml_metadata_url != "" && length(local.env.grafana_admin_okta_groups) > 0 && length(local.env.grafana_editor_okta_groups) > 0
 
-  # Delivery only. Off in dev, where nobody acts on a page.
+  # Gates evaluation, not just delivery. See the README.
   alerts_enabled = local.env.alerts_enabled
 
   saturation_threshold = 0.8

terraform/observability-aws/outputs.tf

@@ -44,7 +44,7 @@ output "alerts_sns_topic_arn" {
 }
 
 output "alerts_enabled" {
-  description = "Whether the bridge Lambda is subscribed to the topic. False in dev: rules and alarms evaluate, nothing is delivered."
+  description = "Whether alerting is on at all. False in dev: no CloudWatch alarms, no AMP alert rules, no Alertmanager and no subscription. Recording rules and the delivery path are built either way."
   value       = local.alerts_enabled
 }
 

terraform/observability-grafana/locals.tf

@@ -20,6 +20,9 @@ locals {
   amg_sa_token_secret_version_id = try(data.terraform_remote_state.observability_aws.outputs.amg_service_account_token_secret_version_id, null)
   amp_prometheus_endpoint        = try(data.terraform_remote_state.observability_aws.outputs.amp_prometheus_endpoint, null)
 
+  # Off, there is no rule group or Alertmanager definition to point Grafana at.
+  alerts_enabled = try(data.terraform_remote_state.observability_aws.outputs.alerts_enabled, false)
+
   amg_service_account_token = try(data.aws_secretsmanager_secret_version.amg_sa_token[0].secret_string, null)
 
   alb_arn_suffix          = try(data.terraform_remote_state.edge.outputs.alb_arn_suffix, null)

terraform/observability-grafana/main.tf

@@ -51,18 +51,19 @@ resource "grafana_data_source" "amp" {
   uid  = local.datasource_uid_amp
   url  = local.amp_prometheus_endpoint
 
-  json_data_encoded = jsonencode({
-    httpMethod      = "POST"
-    sigV4Auth       = true
-    sigV4AuthType   = "default"
-    sigV4Region     = var.aws_region
-    manageAlerts    = true
+  json_data_encoded = jsonencode(merge({
+    httpMethod    = "POST"
+    sigV4Auth     = true
+    sigV4AuthType = "default"
+    sigV4Region   = var.aws_region
+    manageAlerts  = local.alerts_enabled
+    }, local.alerts_enabled ? {
     alertmanagerUid = local.datasource_uid_amp_alertmanager
-  })
+  } : {}))
 }
 
 resource "grafana_data_source" "amp_alertmanager" {
-  count = local.observability_ready ? 1 : 0
+  count = local.observability_ready && local.alerts_enabled ? 1 : 0
 
   type = "alertmanager"
   name = "amp-alertmanager-${terraform.workspace}"