Alerting integration (#338)

* accept alerts from alert manager and store/update them

* update sqlalchemy syntax

* add a reconciliation as backup for webhooks

* don't allow to return to firing

when a webhook arrives late after reconciler, we can ignore it

* factor out alert ingestor

* oauth to slack/discord

* get default channel setup working

* alerting: make rules super duper configurable

* include slack/discord integrations and routing in audit log

* add coverage for environment resolution, ensure we're "there"

* preview routing coverage for notifications

* dispatch alerts to slack/discord

* rich preview for alerts

* refactor and tidy

* appease bandit, the right way

* fill in the gaps, close manual actions out

* notify on reaped build job, reconcile terminal notifications to ensure they're sent

* enqueue after commit

* tz stuff

* raise retryable exception

* more safely match ingress names

* cleanup orphaned routes

* validate channel

* paginate channel list for slack

* optimize application environment resolution

* de-jargon alerts

* allow alert reconciler to initiate notifications if last_notified is None

* Allow routing only image/release/deploy failures

* use advisory lock to avoid duplicate notifications

* need to send scope header to mimir

* re-order migrations

Author: ewdurbin

cabotage/celery/tasks/__init__.py

@@ -30,3 +30,12 @@
     refresh_tailscale_oidc_tokens,  # noqa: F401
     teardown_tailscale_operator,  # noqa: F401
 )
+
+from .alerting import reconcile_alerts  # noqa: F401
+
+from .notify import (
+    dispatch_alert_notification,  # noqa: F401
+    dispatch_pipeline_notification,  # noqa: F401
+    reconcile_notifications,  # noqa: F401
+    send_notification,  # noqa: F401
+)

cabotage/celery/tasks/alerting.py

@@ -0,0 +1,112 @@
+"""Celery task to reconcile alerts with Alertmanager.
+
+Polls the Alertmanager v2 API for all currently active alerts, upserts them
+into the local alerts table, and marks any locally-firing alerts that are
+no longer present in Alertmanager as resolved.
+"""
+
+import logging
+from datetime import UTC, datetime
+
+import requests
+from celery import shared_task
+from flask import current_app
+
+from cabotage.server import db
+from cabotage.server.models.projects import Alert
+from cabotage.server.alerting.ingest import (
+    _record_activity,
+    parse_alertmanager_timestamp,
+    upsert_alert,
+)
+from cabotage.celery.tasks.notify import dispatch_alert_notification
+
+log = logging.getLogger(__name__)
+
+
+@shared_task()
+def reconcile_alerts():
+    alertmanager_url = current_app.config.get("ALERTMANAGER_URL")
+    if not alertmanager_url:
+        return
+
+    verify = current_app.config.get("ALERTMANAGER_VERIFY")
+    if verify is None:
+        verify = True
+
+    headers = {"X-Scope-OrgID": current_app.config.get("MIMIR_TENANT_ID")}
+
+    secret = current_app.config.get("ALERTMANAGER_WEBHOOK_SECRET")
+    if secret:
+        headers["Authorization"] = f"Bearer {secret}"
+
+    try:
+        resp = requests.get(
+            f"{alertmanager_url.rstrip('/')}/api/v2/alerts",
+            headers=headers,
+            verify=verify,
+            timeout=30,
+        )
+        resp.raise_for_status()
+    except requests.RequestException:
+        log.exception("Failed to fetch alerts from Alertmanager")
+        return
+
+    active_alerts = resp.json()
+
+    # Track which fingerprints are still active in Alertmanager
+    seen_fingerprints = set()
+    dispatch_ids = []
+
+    for alert_data in active_alerts:
+        labels = alert_data.get("labels", {})
+        fingerprint = alert_data.get("fingerprint", "")
+
+        starts_at = parse_alertmanager_timestamp(alert_data.get("startsAt"))
+        if starts_at:
+            seen_fingerprints.add((fingerprint, starts_at))
+
+        # The v2 API nests status as {"state": "active|suppressed|..."}
+        status = alert_data.get("status", {})
+        if isinstance(status, dict):
+            state = status.get("state", "active")
+        else:
+            state = status
+        am_status = "firing" if state == "active" else state
+
+        _processed, dispatch_id = upsert_alert(
+            fingerprint=fingerprint,
+            status=am_status,
+            alertname=labels.get("alertname", "unknown"),
+            labels=labels,
+            annotations=alert_data.get("annotations", {}),
+            starts_at=starts_at,
+            ends_at=parse_alertmanager_timestamp(alert_data.get("endsAt")),
+            generator_url=alert_data.get("generatorURL"),
+        )
+        if dispatch_id:
+            dispatch_ids.append(dispatch_id)
+
+    # Resolve any locally-firing alerts that are no longer in Alertmanager
+    firing_alerts = Alert.query.filter_by(status="firing").all()
+    now = datetime.now(UTC).replace(tzinfo=None)
+    resolved_count = 0
+    for alert in firing_alerts:
+        if (alert.fingerprint, alert.starts_at) not in seen_fingerprints:
+            alert.status = "resolved"
+            alert.ends_at = now
+            _record_activity("resolved", alert, alert.application)
+            dispatch_ids.append(alert.id)
+            resolved_count += 1
+
+    db.session.commit()
+
+    # Dispatch notifications after commit so workers can see the data
+    for alert_id in dispatch_ids:
+        dispatch_alert_notification.delay(str(alert_id))
+
+    log.info(
+        "Reconciled alerts: %d active from Alertmanager, %d resolved",
+        len(active_alerts),
+        resolved_count,
+    )