feat: add configuration for Konnectivity agent replica alert in GKE

filippolmt · filippolmt · commit 5f3869d23763 · 2026-01-13T15:59:27.000+01:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,11 +8,16 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Changed
+
+- Add konnectivity agent replica alert with a PromQL-based condition that counts pods via `kubernetes_io:container_uptime`.
+- Standardize alert filter/query style for consistency across configuration.
+
 ## [0.10.0] - 2026-01-05
 
 [Compare with previous version](https://github.com/sparkfabrik/terraform-google-services-monitoring/compare/0.9.0...0.10.0)
 
-### Changed
+### Added
 
 - Add `no agent available` to Kyverno log alert filter to capture control plane-to-node connectivity failures via Konnectivity (upstream Kubernetes); commonly seen on GKE (especially private nodes), but not GKE-specific.
 
diff --git a/cloud_sql.tf b/cloud_sql.tf
@@ -69,7 +69,11 @@ resource "google_monitoring_alert_policy" "cloud_sql_cpu_utilization" {
 
   conditions {
     condition_threshold {
-      filter          = "resource.type = \"cloudsql_database\" AND resource.labels.database_id = \"${local.cloud_sql_project}:${each.value.instance}\" AND metric.type = \"cloudsql.googleapis.com/database/cpu/utilization\""
+      filter          = <<-EOT
+        resource.type="cloudsql_database"
+        AND resource.labels.database_id="${local.cloud_sql_project}:${each.value.instance}"
+        AND metric.type="cloudsql.googleapis.com/database/cpu/utilization"
+      EOT
       comparison      = "COMPARISON_GT"
       threshold_value = each.value.threshold
       duration        = each.value.duration
@@ -101,7 +105,11 @@ resource "google_monitoring_alert_policy" "cloud_sql_memory_utilization" {
   conditions {
     display_name = "${local.cloud_sql_project} ${each.value.instance} - Memory utilization ${each.value.severity} ${each.value.threshold * 100}%"
     condition_threshold {
-      filter          = "resource.type = \"cloudsql_database\" AND resource.labels.database_id = \"${local.cloud_sql_project}:${each.value.instance}\" AND metric.type = \"cloudsql.googleapis.com/database/memory/utilization\""
+      filter          = <<-EOT
+        resource.type="cloudsql_database"
+        AND resource.labels.database_id="${local.cloud_sql_project}:${each.value.instance}"
+        AND metric.type="cloudsql.googleapis.com/database/memory/utilization"
+      EOT
       duration        = each.value.duration
       comparison      = "COMPARISON_GT"
       threshold_value = each.value.threshold
@@ -133,7 +141,11 @@ resource "google_monitoring_alert_policy" "cloud_sql_disk_utilization" {
   conditions {
     display_name = "${local.cloud_sql_project} ${each.value.instance} - Disk utilization ${each.value.severity} ${each.value.threshold * 100}%"
     condition_threshold {
-      filter          = "resource.type = \"cloudsql_database\" AND resource.labels.database_id = \"${local.cloud_sql_project}:${each.value.instance}\" AND metric.type = \"cloudsql.googleapis.com/database/disk/utilization\""
+      filter          = <<-EOT
+        resource.type="cloudsql_database"
+        AND resource.labels.database_id="${local.cloud_sql_project}:${each.value.instance}"
+        AND metric.type="cloudsql.googleapis.com/database/disk/utilization"
+      EOT
       duration        = each.value.duration
       comparison      = "COMPARISON_GT"
       threshold_value = each.value.threshold
diff --git a/examples/main.tf b/examples/main.tf
@@ -42,7 +42,7 @@ locals {
 }
 
 module "example" {
-  source  = "github.com/sparkfabrik/terraform-google-services-monitoring?ref=0.9.0"
+  source = "github.com/sparkfabrik/terraform-google-services-monitoring?ref=0.9.0"
 
   notification_channels = var.notification_channels
   project_id            = var.project_id
@@ -55,22 +55,22 @@ module "example" {
     filter_extra = "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\""
   }
   cert_manager = {
-    cluster_name          = "test-cluster"
-    namespace             = "cert-manager"
+    cluster_name = "test-cluster"
+    namespace    = "cert-manager"
   }
 
   typesense = {
     cluster_name = "test-cluster"
     apps = {
       "typesense-app" = {
         uptime_check = {
-          host    = "typesense.example.com"
+          host = "typesense.example.com"
         }
         container_check = {
           enabled   = true
           namespace = "typesense"
           pod_restart = {
-            threshold          = 1
+            threshold = 1
           }
         }
       }
@@ -82,13 +82,13 @@ module "example" {
     apps = {
       "litellm-app" = {
         uptime_check = {
-          host    = "litellm.example.com"
+          host = "litellm.example.com"
         }
         container_check = {
           namespace = "litellm"
           pod_restart = {
-            threshold          = 2
-            duration           = 300
+            threshold            = 2
+            duration             = 300
             notification_prompts = ["CLOSED"]
           }
         }
diff --git a/konnectivity_agent.tf b/konnectivity_agent.tf
@@ -0,0 +1,66 @@
+locals {
+  konnectivity_agent_project = (
+    var.konnectivity_agent.project_id != null
+    ? var.konnectivity_agent.project_id
+    : var.project_id
+  )
+
+  konnectivity_agent_notification_channels = (
+    var.konnectivity_agent.notification_enabled
+    ? (
+      length(var.konnectivity_agent.notification_channels) > 0
+      ? var.konnectivity_agent.notification_channels
+      : var.notification_channels
+    )
+    : []
+  )
+}
+
+resource "google_monitoring_alert_policy" "konnectivity_agent_replicas" {
+  count = var.konnectivity_agent.enabled ? 1 : 0
+
+  project      = local.konnectivity_agent_project
+  display_name = "CRITICAL: Konnectivity agent pod count == 0 (cluster=${var.konnectivity_agent.cluster_name}, namespace=${var.konnectivity_agent.namespace}, deployment=${var.konnectivity_agent.deployment_name})"
+  combiner     = "OR"
+  enabled      = var.konnectivity_agent.enabled
+  user_labels = {
+    severity = "critical"
+  }
+
+  conditions {
+    display_name = "Konnectivity agent pod count == 0"
+
+    condition_prometheus_query_language {
+      query = <<-PROMQL
+        (
+          count(
+            max by (pod_name) (
+              kubernetes_io:container_uptime{
+                monitored_resource="k8s_container",
+                project_id="${local.konnectivity_agent_project}",
+                cluster_name="${var.konnectivity_agent.cluster_name}",
+                namespace_name="${var.konnectivity_agent.namespace}",
+                metadata_system_top_level_controller_name="${var.konnectivity_agent.deployment_name}"
+              }
+            )
+          )
+          or on() vector(0)
+        ) == 0
+      PROMQL
+
+      duration = "${var.konnectivity_agent.duration_seconds}s"
+    }
+  }
+
+  documentation {
+    content   = "CRITICAL: Konnectivity agent has zero ready replicas in kube-system. Investigate immediately."
+    mime_type = "text/markdown"
+  }
+
+  notification_channels = local.konnectivity_agent_notification_channels
+
+  alert_strategy {
+    auto_close           = "${var.konnectivity_agent.auto_close_seconds}s"
+    notification_prompts = var.konnectivity_agent.notification_prompts
+  }
+}
diff --git a/lite_llm.tf b/lite_llm.tf
@@ -74,7 +74,7 @@ resource "google_monitoring_alert_policy" "litellm_pod_restart" {
   notification_channels = local.litellm_notification_channels
 
   alert_strategy {
-    auto_close = "${each.value.pod_restart.auto_close_seconds}s"
+    auto_close           = "${each.value.pod_restart.auto_close_seconds}s"
     notification_prompts = each.value.pod_restart.notification_prompts
   }
 }
diff --git a/ssl_alert.tf b/ssl_alert.tf
@@ -11,7 +11,10 @@ resource "google_monitoring_alert_policy" "ssl_expiring_days" {
   combiner     = "OR"
   conditions {
     condition_threshold {
-      filter          = "metric.type=\"monitoring.googleapis.com/uptime_check/time_until_ssl_cert_expires\" AND resource.type=\"uptime_url\""
+      filter          = <<-EOT
+        metric.type="monitoring.googleapis.com/uptime_check/time_until_ssl_cert_expires"
+        AND resource.type="uptime_url"
+      EOT
       comparison      = "COMPARISON_LT"
       threshold_value = each.value
       duration        = "600s"
diff --git a/typesense.tf b/typesense.tf
@@ -74,7 +74,7 @@ resource "google_monitoring_alert_policy" "typesense_pod_restart" {
   notification_channels = local.typesense_notification_channels
 
   alert_strategy {
-    auto_close = "${each.value.pod_restart.auto_close_seconds}s"
+    auto_close           = "${each.value.pod_restart.auto_close_seconds}s"
     notification_prompts = each.value.pod_restart.notification_prompts
   }
 }
diff --git a/variables.tf b/variables.tf
@@ -99,6 +99,23 @@ variable "cert_manager" {
   })
 }
 
+variable "konnectivity_agent" {
+  description = "Configuration for Konnectivity agent deployment replica alert in GKE. Triggers when there are no available replicas."
+  default     = {}
+  type = object({
+    enabled               = optional(bool, true)
+    cluster_name          = string
+    project_id            = optional(string, null)
+    namespace             = optional(string, "kube-system")
+    deployment_name       = optional(string, "konnectivity-agent")
+    duration_seconds      = optional(number, 60)
+    auto_close_seconds    = optional(number, 3600)
+    notification_enabled  = optional(bool, true)
+    notification_channels = optional(list(string), [])
+    notification_prompts  = optional(list(string), null)
+  })
+}
+
 variable "typesense" {
   description = "Configuration for Typesense monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key)."
   default     = {}
@@ -120,10 +137,10 @@ variable "typesense" {
         enabled   = optional(bool, true)
         namespace = string
         pod_restart = optional(object({
-          threshold          = optional(number, 0)
-          alignment_period   = optional(number, 60)
-          duration           = optional(number, 180)
-          auto_close_seconds = optional(number, 3600)
+          threshold            = optional(number, 0)
+          alignment_period     = optional(number, 60)
+          duration             = optional(number, 180)
+          auto_close_seconds   = optional(number, 3600)
           notification_prompts = optional(list(string), null)
         }), {})
       }), null)
@@ -171,10 +188,10 @@ variable "litellm" {
         enabled   = optional(bool, true)
         namespace = string
         pod_restart = optional(object({
-          threshold          = optional(number, 0)
-          alignment_period   = optional(number, 60)
-          duration           = optional(number, 180)
-          auto_close_seconds = optional(number, 3600)
+          threshold            = optional(number, 0)
+          alignment_period     = optional(number, 60)
+          duration             = optional(number, 180)
+          auto_close_seconds   = optional(number, 3600)
           notification_prompts = optional(list(string), null)
         }), {})
       }), null)

Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ locals {`
`42`	`42`	`}`
`43`	`43`
`44`	`44`	`module "example" {`
`45`		`- source = "github.com/sparkfabrik/terraform-google-services-monitoring?ref=0.9.0"`
	`45`	`+ source = "github.com/sparkfabrik/terraform-google-services-monitoring?ref=0.9.0"`
`46`	`46`
`47`	`47`	`notification_channels = var.notification_channels`
`48`	`48`	`project_id = var.project_id`
`@@ -55,22 +55,22 @@ module "example" {`
`55`	`55`	`filter_extra = "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\""`
`56`	`56`	`}`
`57`	`57`	`cert_manager = {`
`58`		`- cluster_name = "test-cluster"`
`59`		`- namespace = "cert-manager"`
	`58`	`+ cluster_name = "test-cluster"`
	`59`	`+ namespace = "cert-manager"`
`60`	`60`	`}`
`61`	`61`
`62`	`62`	`typesense = {`
`63`	`63`	`cluster_name = "test-cluster"`
`64`	`64`	`apps = {`
`65`	`65`	`"typesense-app" = {`
`66`	`66`	`uptime_check = {`
`67`		`- host = "typesense.example.com"`
	`67`	`+ host = "typesense.example.com"`
`68`	`68`	`}`
`69`	`69`	`container_check = {`
`70`	`70`	`enabled = true`
`71`	`71`	`namespace = "typesense"`
`72`	`72`	`pod_restart = {`
`73`		`- threshold = 1`
	`73`	`+ threshold = 1`
`74`	`74`	`}`
`75`	`75`	`}`
`76`	`76`	`}`
`@@ -82,13 +82,13 @@ module "example" {`
`82`	`82`	`apps = {`
`83`	`83`	`"litellm-app" = {`
`84`	`84`	`uptime_check = {`
`85`		`- host = "litellm.example.com"`
	`85`	`+ host = "litellm.example.com"`
`86`	`86`	`}`
`87`	`87`	`container_check = {`
`88`	`88`	`namespace = "litellm"`
`89`	`89`	`pod_restart = {`
`90`		`- threshold = 2`
`91`		`- duration = 300`
	`90`	`+ threshold = 2`
	`91`	`+ duration = 300`
`92`	`92`	`notification_prompts = ["CLOSED"]`
`93`	`93`	`}`
`94`	`94`	`}`
Original file line number	Diff line number	Diff line change
`@@ -74,7 +74,7 @@ resource "google_monitoring_alert_policy" "litellm_pod_restart" {`
`74`	`74`	`notification_channels = local.litellm_notification_channels`
`75`	`75`
`76`	`76`	`alert_strategy {`
`77`		`- auto_close = "${each.value.pod_restart.auto_close_seconds}s"`
	`77`	`+ auto_close = "${each.value.pod_restart.auto_close_seconds}s"`
`78`	`78`	`notification_prompts = each.value.pod_restart.notification_prompts`
`79`	`79`	`}`
`80`	`80`	`}`