refs platform/board#4051: add Lite LLM monitoring (#12)

Author: FabrizioCafolla

CHANGELOG.md

@@ -8,7 +8,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
-## [0.7.0] - 2025-12-12
+## [0.8.0] - 2025-12-12
+
+[Compare with previous version](https://github.com/sparkfabrik/terraform-google-services-monitoring/compare/0.7.0...0.8.0)
+
+### Added
+
+- refs platform/board#4051: add LiteLLM monitoring
+
+## [0.7.0] - 2025-12-11
 
 [Compare with previous version](https://github.com/sparkfabrik/terraform-google-services-monitoring/compare/0.6.0...0.7.0)
 

README.md

@@ -38,10 +38,11 @@ Supported services:
 | <a name="input_cert_manager"></a> [cert\_manager](#input\_cert\_manager) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. | <pre>object({<br/>    enabled                          = optional(bool, true)<br/>    cluster_name                     = string<br/>    project_id                       = optional(string, null)<br/>    namespace                        = optional(string, "cert-manager")<br/>    notification_enabled             = optional(bool, true)<br/>    notification_channels            = optional(list(string), [])<br/>    logmatch_notification_rate_limit = optional(string, "300s")<br/>    alert_documentation              = optional(string, null)<br/>    auto_close_seconds               = optional(number, 3600)<br/>    filter_extra                     = optional(string, "")<br/>  })</pre> | n/a | yes |
 | <a name="input_cloud_sql"></a> [cloud\_sql](#input\_cloud\_sql) | Configuration for Cloud SQL monitoring alerts. Supports customization of project, auto-close timing, notification channels, and per-instance alert thresholds for CPU, memory, and disk utilization. | <pre>object({<br/>    project_id            = optional(string, null)<br/>    auto_close            = optional(string, "86400s") # default 24h<br/>    notification_enabled  = optional(bool, true)<br/>    notification_channels = optional(list(string), [])<br/>    instances = optional(map(object({<br/>      cpu_utilization = optional(list(object({<br/>        severity         = optional(string, "WARNING"),<br/>        threshold        = optional(number, 0.90)<br/>        alignment_period = optional(string, "120s")<br/>        duration         = optional(string, "300s")<br/>        })), [<br/>        {<br/>          threshold = 0.85,<br/>          duration  = "1200s",<br/>        },<br/>        {<br/>          severity         = "CRITICAL",<br/>          threshold        = 1,<br/>          duration         = "300s",<br/>          alignment_period = "60s",<br/>        }<br/>      ])<br/>      memory_utilization = optional(list(object({<br/>        severity         = optional(string, "WARNING"),<br/>        threshold        = optional(number, 0.90)<br/>        alignment_period = optional(string, "300s")<br/>        duration         = optional(string, "300s")<br/>        })), [<br/>        {<br/>          severity = "WARNING",<br/>        },<br/>        {<br/>          severity  = "CRITICAL",<br/>          threshold = 0.95,<br/>        }<br/>      ])<br/>      disk_utilization = optional(list(object({<br/>        severity         = optional(string, "WARNING"),<br/>        threshold        = optional(number, 0.85)<br/>        alignment_period = optional(string, "300s")<br/>        duration         = optional(string, "600s")<br/>        })), [<br/>        {<br/>          severity = "WARNING",<br/>        },<br/>        {<br/>          severity  = "CRITICAL",<br/>          threshold = 0.95,<br/>        }<br/>      ])<br/>    })), {})<br/>  })</pre> | n/a | yes |
 | <a name="input_kyverno"></a> [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. | <pre>object({<br/>    enabled               = optional(bool, true)<br/>    cluster_name          = string<br/>    project_id            = optional(string, null)<br/>    notification_enabled  = optional(bool, true)<br/>    notification_channels = optional(list(string), [])<br/>    # Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts<br/>    logmatch_notification_rate_limit = optional(string, "300s")<br/>    alert_documentation              = optional(string, null)<br/>    auto_close_seconds               = optional(number, 3600)<br/>    filter_extra                     = optional(string, "")<br/>    namespace                        = optional(string, "kyverno")<br/>  })</pre> | n/a | yes |
+| <a name="input_litellm"></a> [litellm](#input\_litellm) | Configuration for LiteLLM monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key). | <pre>object({<br/>    enabled               = optional(bool, false)<br/>    project_id            = optional(string, null)<br/>    notification_enabled  = optional(bool, true)<br/>    notification_channels = optional(list(string), [])<br/>    cluster_name          = optional(string, null)<br/><br/>    apps = optional(map(object({<br/>      uptime_check = optional(object({<br/>        enabled = optional(bool, true)<br/>        host    = string<br/>        path    = optional(string, "/health/readiness")<br/>      }), null)<br/><br/>      container_check = optional(object({<br/>        enabled   = optional(bool, true)<br/>        namespace = string<br/>        pod_restart = optional(object({<br/>          threshold          = optional(number, 0)<br/>          alignment_period   = optional(number, 60)<br/>          duration           = optional(number, 0)<br/>          auto_close_seconds = optional(number, 3600)<br/>        }), {})<br/>      }), null)<br/>    })), {})<br/>  })</pre> | `{}` | no |
 | <a name="input_notification_channels"></a> [notification\_channels](#input\_notification\_channels) | List of notification channel IDs to notify when an alert is triggered | `list(string)` | `[]` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The Google Cloud project ID where logging exclusions will be created | `string` | n/a | yes |
-| <a name="input_ssl_alert"></a> [ssl\_alert](#input\_ssl\_alert) | Configuration for SSL certificate expiration alerts. Allows customization of project, notification channels, alert thresholds, and user labels. | <pre>object({<br/>    enabled               = optional(bool, false)<br/>    project_id            = optional(string, null)<br/>    notification_enabled  = optional(bool, true)<br/>    notification_channels = optional(list(string), [])<br/>    threshold_days        = optional(list(number), [15, 7])<br/>    user_labels            = optional(map(string), {})<br/>  })</pre> | `{}` | no |
-| <a name="input_typesense"></a> [typesense](#input\_typesense) | Configuration for Typesense monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key). For container checks, the app name corresponds to the Kubernetes 'app' label; for apps with only uptime checks, this correspondence does not apply. | <pre>object({<br/>    enabled               = optional(bool, false)<br/>    project_id            = optional(string, null)<br/>    notification_enabled  = optional(bool, true)<br/>    notification_channels = optional(list(string), [])<br/>    cluster_name          = optional(string, null) # GKE cluster name for container checks<br/><br/>    # Apps configuration - map keyed by app_name<br/>    apps = optional(map(object({<br/>      # Uptime check configuration (optional)<br/>      uptime_check = optional(object({<br/>        enabled = optional(bool, true)<br/>        host    = string<br/>        path    = optional(string, "/readyz")<br/>      }), null)<br/><br/>      # Container check configuration for GKE (optional)<br/>      container_check = optional(object({<br/>        enabled   = optional(bool, true)<br/>        namespace = string<br/>        pod_restart = optional(object({<br/>          threshold          = optional(number, 0)<br/>          alignment_period   = optional(number, 60)<br/>          duration           = optional(number, 0)<br/>          auto_close_seconds = optional(number, 3600)<br/>        }), {})<br/>      }), null)<br/>    })), {})<br/>  })</pre> | `{}` | no |
+| <a name="input_ssl_alert"></a> [ssl\_alert](#input\_ssl\_alert) | Configuration for SSL certificate expiration alerts. Allows customization of project, notification channels, alert thresholds, and user labels. | <pre>object({<br/>    enabled               = optional(bool, false)<br/>    project_id            = optional(string, null)<br/>    notification_enabled  = optional(bool, true)<br/>    notification_channels = optional(list(string), [])<br/>    threshold_days        = optional(list(number), [15, 7])<br/>    user_labels           = optional(map(string), {})<br/>  })</pre> | `{}` | no |
+| <a name="input_typesense"></a> [typesense](#input\_typesense) | Configuration for Typesense monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key). | <pre>object({<br/>    enabled               = optional(bool, false)<br/>    project_id            = optional(string, null)<br/>    notification_enabled  = optional(bool, true)<br/>    notification_channels = optional(list(string), [])<br/>    cluster_name          = optional(string, null) # GKE cluster name for container checks<br/><br/>    # Apps configuration - map keyed by app_name<br/>    apps = optional(map(object({<br/>      # Uptime check configuration (optional)<br/>      uptime_check = optional(object({<br/>        enabled = optional(bool, true)<br/>        host    = string<br/>        path    = optional(string, "/readyz")<br/>      }), null)<br/><br/>      # Container check configuration for GKE (optional)<br/>      container_check = optional(object({<br/>        enabled   = optional(bool, true)<br/>        namespace = string<br/>        pod_restart = optional(object({<br/>          threshold          = optional(number, 0)<br/>          alignment_period   = optional(number, 60)<br/>          duration           = optional(number, 0)<br/>          auto_close_seconds = optional(number, 3600)<br/>        }), {})<br/>      }), null)<br/>    })), {})<br/>  })</pre> | `{}` | no |
 
 ## Outputs
 
@@ -61,13 +62,15 @@ Supported services:
 | [google_monitoring_alert_policy.cloud_sql_disk_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.cloud_sql_memory_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.kyverno_logmatch_alert](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
+| [google_monitoring_alert_policy.litellm_pod_restart](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.ssl_expiring_days](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.typesense_pod_restart](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_litellm_uptime_checks"></a> [litellm\_uptime\_checks](#module\_litellm\_uptime\_checks) | github.com/sparkfabrik/terraform-sparkfabrik-gcp-http-monitoring | 1.0.0 |
 | <a name="module_typesense_uptime_checks"></a> [typesense\_uptime\_checks](#module\_typesense\_uptime\_checks) | github.com/sparkfabrik/terraform-sparkfabrik-gcp-http-monitoring | 1.0.0 |
 
 <!-- END_TF_DOCS -->

lite_llm.tf

@@ -0,0 +1,79 @@
+locals {
+  litellm_project = var.litellm.project_id != null ? var.litellm.project_id : var.project_id
+
+  litellm_notification_channels = var.litellm.notification_enabled ? (length(var.litellm.notification_channels) > 0 ? var.litellm.notification_channels : var.notification_channels) : []
+
+  litellm_uptime_checks = var.litellm.enabled ? {
+    for app_name, config in var.litellm.apps :
+    app_name => config.uptime_check
+    if config.uptime_check != null && try(config.uptime_check.enabled, false)
+  } : {}
+
+  litellm_container_checks = var.litellm.enabled ? {
+    for app_name, config in var.litellm.apps :
+    app_name => config.container_check
+    if config.container_check != null && try(config.container_check.enabled, false)
+  } : {}
+}
+
+module "litellm_uptime_checks" {
+  for_each = local.litellm_uptime_checks
+
+  source                      = "github.com/sparkfabrik/terraform-sparkfabrik-gcp-http-monitoring?ref=1.0.0"
+  gcp_project                 = local.litellm_project
+  uptime_monitoring_host      = each.value.host
+  uptime_monitoring_path      = each.value.path
+  alert_notification_channels = local.litellm_notification_channels
+  alert_threshold_value       = 1
+  uptime_check_period         = "900s"
+}
+
+# Alert: GKE Pod Restarts
+# This alert monitors the restart count of LiteLLM containers in GKE.
+# It triggers when the delta of restarts is greater than the threshold
+# within the specified alignment period.
+resource "google_monitoring_alert_policy" "litellm_pod_restart" {
+  for_each = local.litellm_container_checks
+
+  project      = local.litellm_project
+  display_name = "LiteLLM Pod Restarts (cluster=${var.litellm.cluster_name}, namespace=${each.value.namespace}, app=${each.key})"
+  combiner     = "OR"
+  enabled      = true
+
+  conditions {
+    display_name = "LiteLLM container restart count > ${each.value.pod_restart.threshold}"
+
+    condition_threshold {
+      filter = <<-EOT
+        resource.type="k8s_container"
+        AND resource.labels.project_id="${local.litellm_project}"
+        AND resource.labels.cluster_name="${var.litellm.cluster_name}"
+        AND resource.labels.namespace_name="${each.value.namespace}"
+        AND metric.type="kubernetes.io/container/restart_count"
+      EOT
+
+      comparison      = "COMPARISON_GT"
+      threshold_value = each.value.pod_restart.threshold
+      duration        = "${each.value.pod_restart.duration}s"
+
+      aggregations {
+        alignment_period     = "${each.value.pod_restart.alignment_period}s"
+        per_series_aligner   = "ALIGN_DELTA"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields = [
+          "metadata.user_labels.\"app.kubernetes.io/instance\"",
+        ]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  notification_channels = local.litellm_notification_channels
+
+  alert_strategy {
+    auto_close = "${each.value.pod_restart.auto_close_seconds}s"
+  }
+}

typesense.tf

@@ -1,4 +1,3 @@
-
 locals {
   typesense_project = var.typesense.project_id != null ? var.typesense.project_id : var.project_id
 

variables.tf

@@ -100,7 +100,7 @@ variable "cert_manager" {
 }
 
 variable "typesense" {
-  description = "Configuration for Typesense monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key). For container checks, the app name corresponds to the Kubernetes 'app' label; for apps with only uptime checks, this correspondence does not apply."
+  description = "Configuration for Typesense monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key)."
   default     = {}
   type = object({
     enabled               = optional(bool, false)
@@ -152,6 +152,56 @@ variable "typesense" {
   }
 }
 
+variable "litellm" {
+  description = "Configuration for LiteLLM monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key)."
+  default     = {}
+  type = object({
+    enabled               = optional(bool, false)
+    project_id            = optional(string, null)
+    notification_enabled  = optional(bool, true)
+    notification_channels = optional(list(string), [])
+    cluster_name          = optional(string, null)
+
+    apps = optional(map(object({
+      uptime_check = optional(object({
+        enabled = optional(bool, true)
+        host    = string
+        path    = optional(string, "/health/readiness")
+      }), null)
+
+      container_check = optional(object({
+        enabled   = optional(bool, true)
+        namespace = string
+        pod_restart = optional(object({
+          threshold          = optional(number, 0)
+          alignment_period   = optional(number, 60)
+          duration           = optional(number, 0)
+          auto_close_seconds = optional(number, 3600)
+        }), {})
+      }), null)
+    })), {})
+  })
+
+  validation {
+    condition = alltrue([
+      for app_name, config in var.litellm.apps : (
+        trimspace(app_name) != "" &&
+        (config.uptime_check != null ? try(trimspace(config.uptime_check.host), "") != "" : true) &&
+        (config.container_check != null ? try(trimspace(config.container_check.namespace), "") != "" : true)
+      )
+    ])
+    error_message = "Each app must have a non-empty name (map key). If uptime_check is provided, 'host' must be non-empty. If container_check is provided, 'namespace' must be non-empty."
+  }
+
+  validation {
+    condition = (
+      length([for app_name, config in var.litellm.apps : app_name if config.container_check != null]) == 0 ||
+      try(trimspace(var.litellm.cluster_name), "") != ""
+    )
+    error_message = "When any app has container_check configured, 'cluster_name' must be provided at the litellm level."
+  }
+}
+
 variable "ssl_alert" {
   description = "Configuration for SSL certificate expiration alerts. Allows customization of project, notification channels, alert thresholds, and user labels."
   default     = {}
@@ -161,6 +211,6 @@ variable "ssl_alert" {
     notification_enabled  = optional(bool, true)
     notification_channels = optional(list(string), [])
     threshold_days        = optional(list(number), [15, 7])
-    user_labels            = optional(map(string), {})
+    user_labels           = optional(map(string), {})
   })
 }