diff --git a/CHANGELOG.md b/CHANGELOG.md index df8e671..2abf53f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,20 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Adjust Kyverno log filter to reduce false positives from normal transient errors such as `i/o timeout` and `failed to acquire lease`, including removal of the explicit `failed to acquire lease` condition. - Rename error pattern `list resources failed` to `failed to list resources` for consistency with other error patterns. +### Added + +- Add `error_patterns_exclude` to Kyverno configuration to allow excluding specific error patterns from the default set. +- Add `error_patterns_include` to Kyverno configuration to allow adding custom error patterns to the default set. +- Add validation for `error_patterns_exclude` to ensure only valid default patterns can be excluded. + +### Breaking change + +- The `filter_extra` variable has been removed and replaced with `error_patterns_include` and `error_patterns_exclude`. To migrate: + - If you were using `filter_extra` to add custom error patterns for `jsonPayload.error` matching, use `error_patterns_include` instead. + - If you need to exclude specific default error patterns, use `error_patterns_exclude`. + - **Note:** The new options are specifically designed for error pattern matching against `jsonPayload.error`. + - See [examples/main.tf](examples/main.tf) for usage examples. + ## [0.12.0] - 2026-01-28 [Compare with previous version](https://github.com/sparkfabrik/terraform-google-services-monitoring/compare/0.11.0...0.12.0) diff --git a/README.md b/README.md index 2f092e2..2fc9de3 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Supported services: | [cert\_manager](#input\_cert\_manager) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, null)
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
}) | `{}` | no |
| [cloud\_sql](#input\_cloud\_sql) | Configuration for Cloud SQL monitoring alerts. Supports customization of project, auto-close timing, notification channels, and per-instance alert thresholds for CPU, memory, and disk utilization. | object({
enabled = optional(bool, true)
project_id = optional(string, null)
auto_close = optional(string, "86400s") # default 24h
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
instances = optional(map(object({
cpu_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "120s")
duration = optional(string, "300s")
})), [
{
threshold = 0.85,
duration = "1200s",
},
{
severity = "CRITICAL",
threshold = 1,
duration = "300s",
alignment_period = "60s",
}
])
memory_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "300s")
duration = optional(string, "300s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
disk_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.85)
alignment_period = optional(string, "300s")
duration = optional(string, "600s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
})), {})
}) | `{}` | no |
| [konnectivity\_agent](#input\_konnectivity\_agent) | Configuration for Konnectivity agent deployment replica alert in GKE. Triggers when there are no available replicas. | object({
enabled = optional(bool, true)
cluster_name = optional(string, null)
project_id = optional(string, null)
namespace = optional(string, "kube-system")
deployment_name = optional(string, "konnectivity-agent")
duration_seconds = optional(number, 60)
auto_close_seconds = optional(number, 3600)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
notification_prompts = optional(list(string), null)
}) | `{}` | no |
-| [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. | object({
enabled = optional(bool, true)
cluster_name = optional(string, null)
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
}) | `{}` | no |
+| [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, error pattern inclusions/exclusions for jsonPayload.error matching, and namespace. | object({
enabled = optional(bool, true)
cluster_name = optional(string, null)
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
namespace = optional(string, "kyverno")
# List of error patterns to exclude from the default set.
# Default patterns available for exclusion:
# "internal error", "failed calling webhook", "timeout", "client-side throttling",
# "failed to run warmup", "schema not found", "failed to list resources",
# "failed to watch resource", "context deadline exceeded", "is forbidden",
# "cannot list resource", "cannot watch resource", "RBAC.*denied",
# "failed to start watcher", "leader election lost", "unable to update .*WebhookConfiguration",
# "failed to sync", "dropping request", "failed to load certificate",
# "failed to update lock", "the object has been modified", "no matches for kind",
# "the server could not find the requested resource", "Too Many Requests", "x509",
# "is invalid:", "connection refused", "no agent available", "fatal error", "panic"
error_patterns_exclude = optional(list(string), [])
# List of additional regex error patterns to include (added to default set)
# e.g. ["my custom.*error", "failed to connect.*database"]
error_patterns_include = optional(list(string), [])
}) | `{}` | no |
| [litellm](#input\_litellm) | Configuration for LiteLLM monitoring alerts. Supports uptime checks for HTTP endpoints and container-level alerts (pod restarts) in GKE. Each app is identified by its name (map key). | object({
enabled = optional(bool, false)
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
cluster_name = optional(string, null)
apps = optional(map(object({
uptime_check = optional(object({
enabled = optional(bool, true)
host = string
path = optional(string, "/health/readiness")
}), null)
container_check = optional(object({
enabled = optional(bool, true)
namespace = string
pod_restart = optional(object({
threshold = optional(number, 0)
alignment_period = optional(number, 60)
duration = optional(number, 180)
auto_close_seconds = optional(number, 3600)
notification_prompts = optional(list(string), null)
}), {})
}), null)
})), {})
}) | `{}` | no |
| [notification\_channels](#input\_notification\_channels) | List of notification channel IDs to notify when an alert is triggered | `list(string)` | `[]` | no |
| [project\_id](#input\_project\_id) | The Google Cloud project ID where logging exclusions will be created | `string` | n/a | yes |
diff --git a/cert_manager.tf b/cert_manager.tf
index 7cb7fb7..bc2877d 100644
--- a/cert_manager.tf
+++ b/cert_manager.tf
@@ -8,7 +8,7 @@ locals {
EOT
)
cert_manager_notification_channels = var.cert_manager.notification_enabled ? (length(var.cert_manager.notification_channels) > 0 ? var.cert_manager.notification_channels : var.notification_channels) : []
- cert_manager_cluster_name = var.cert_manager.cluster_name != null ? trimspace(var.cert_manager.cluster_name) : ""
+ cert_manager_cluster_name = var.cert_manager.cluster_name != null ? trimspace(var.cert_manager.cluster_name) : ""
cert_manager_log_filter = local.cert_manager_cluster_name != "" ? (<<-EOT
(
diff --git a/examples/main.tf b/examples/main.tf
index 8f31b2d..3fa851f 100644
--- a/examples/main.tf
+++ b/examples/main.tf
@@ -50,9 +50,18 @@ module "example" {
kyverno = {
cluster_name = "test-cluster"
notification_channels = []
- # Optional filter for log entries, exclude known non-actionable messages
- # e.g., "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\""
- filter_extra = "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\""
+ # Exclude specific error patterns from the default set (only affects jsonPayload.error matching)
+ error_patterns_exclude = [
+ "failed to start watcher",
+ "failed to list resources",
+ ]
+ # Add custom regex error patterns to the default set (matched against jsonPayload.error)
+ # Note: These options only support error pattern matching. Arbitrary log filter conditions
+ # (e.g., negative filters like -textPayload:"...") are not supported.
+ # error_patterns_include = [
+ # "my custom.*error",
+ # "failed to connect.*database",
+ # ]
}
cert_manager = {
cluster_name = "test-cluster"
diff --git a/kyverno.tf b/kyverno.tf
index 132f365..ba37b1d 100644
--- a/kyverno.tf
+++ b/kyverno.tf
@@ -5,7 +5,58 @@ locals {
kyverno_cluster_name = var.kyverno.cluster_name != null ? trimspace(var.kyverno.cluster_name) : ""
- kyverno_log_filter = local.kyverno_cluster_name != "" ? (<<-EOT
+ # Default error patterns for Kyverno log matching
+ kyverno_default_error_patterns = [
+ "internal error",
+ "failed calling webhook",
+ "timeout",
+ "client-side throttling",
+ "failed to run warmup",
+ "schema not found",
+ "failed to list resources",
+ "failed to watch resource",
+ "context deadline exceeded",
+ "is forbidden",
+ "cannot list resource",
+ "cannot watch resource",
+ "RBAC.*denied",
+ "failed to start watcher",
+ "leader election lost",
+ "unable to update .*WebhookConfiguration",
+ "failed to sync",
+ "dropping request",
+ "failed to load certificate",
+ "failed to update lock",
+ "the object has been modified",
+ "no matches for kind",
+ "the server could not find the requested resource",
+ "Too Many Requests",
+ "x509",
+ "is invalid:",
+ "connection refused",
+ "no agent available",
+ "fatal error",
+ "panic",
+ ]
+
+ # Combine default patterns with included patterns, then filter out excluded ones
+ kyverno_all_error_patterns = distinct(concat(
+ local.kyverno_default_error_patterns,
+ var.kyverno.error_patterns_include
+ ))
+
+ kyverno_active_error_patterns = [
+ for pattern in local.kyverno_all_error_patterns :
+ pattern if !contains(var.kyverno.error_patterns_exclude, pattern)
+ ]
+
+ # Build the error patterns filter string
+ kyverno_error_patterns_filter = length(local.kyverno_active_error_patterns) > 0 ? join("\n OR ", [
+ for pattern in local.kyverno_active_error_patterns :
+ "jsonPayload.error=~\"(?i)${pattern}\""
+ ]) : ""
+
+ kyverno_log_filter = local.kyverno_cluster_name != "" && length(local.kyverno_active_error_patterns) > 0 ? (<<-EOT
resource.type="k8s_container"
AND resource.labels.project_id="${local.kyverno_project_id}"
AND resource.labels.cluster_name="${local.kyverno_cluster_name}"
@@ -15,38 +66,8 @@ locals {
OR resource.labels.pod_name=~"kyverno-(admission|background|cleanup|reports)-controller-.*"
)
AND (
- jsonPayload.error=~"(?i)internal error"
- OR jsonPayload.error=~"(?i)failed calling webhook"
- OR jsonPayload.error=~"(?i)timeout"
- OR jsonPayload.error=~"(?i)client-side throttling"
- OR jsonPayload.error=~"(?i)failed to run warmup"
- OR jsonPayload.error=~"(?i)schema not found"
- OR jsonPayload.error=~"(?i)failed to list resources"
- OR jsonPayload.error=~"(?i)failed to watch resource"
- OR jsonPayload.error=~"(?i)context deadline exceeded"
- OR jsonPayload.error=~"(?i)is forbidden"
- OR jsonPayload.error=~"(?i)cannot list resource"
- OR jsonPayload.error=~"(?i)cannot watch resource"
- OR jsonPayload.error=~"(?i)RBAC.*denied"
- OR jsonPayload.error=~"(?i)failed to start watcher"
- OR jsonPayload.error=~"(?i)leader election lost"
- OR jsonPayload.error=~"(?i)unable to update .*WebhookConfiguration"
- OR jsonPayload.error=~"(?i)failed to sync"
- OR jsonPayload.error=~"(?i)dropping request"
- OR jsonPayload.error=~"(?i)failed to load certificate"
- OR jsonPayload.error=~"(?i)failed to update lock"
- OR jsonPayload.error=~"(?i)the object has been modified"
- OR jsonPayload.error=~"(?i)no matches for kind"
- OR jsonPayload.error=~"(?i)the server could not find the requested resource"
- OR jsonPayload.error=~"(?i)Too Many Requests"
- OR jsonPayload.error=~"(?i)x509"
- OR jsonPayload.error=~"(?i)is invalid:"
- OR jsonPayload.error=~"(?i)connection refused"
- OR jsonPayload.error=~"(?i)no agent available"
- OR jsonPayload.error=~"(?i)fatal error"
- OR jsonPayload.error=~"(?i)panic"
+ ${local.kyverno_error_patterns_filter}
)
- ${trimspace(var.kyverno.filter_extra)}
EOT
) : ""
}
@@ -55,6 +76,7 @@ resource "google_monitoring_alert_policy" "kyverno_logmatch_alert" {
count = (
var.kyverno.enabled
&& local.kyverno_cluster_name != ""
+ && length(local.kyverno_active_error_patterns) > 0
) ? 1 : 0
display_name = "Kyverno controllers ERROR logs (namespace=${var.kyverno.namespace})"
diff --git a/modules/http_monitoring/main.tf b/modules/http_monitoring/main.tf
index f3b9eee..6500b29 100644
--- a/modules/http_monitoring/main.tf
+++ b/modules/http_monitoring/main.tf
@@ -1,7 +1,7 @@
locals {
- suffix = var.uptime_monitoring_path != "/" ? var.uptime_monitoring_path : ""
- uptime_monitoring_display_name = var.uptime_monitoring_display_name != "" ? "${var.uptime_monitoring_display_name} - ${var.uptime_monitoring_host}${local.suffix}" : "${var.uptime_monitoring_host}${local.suffix}"
- alert_display_name = var.alert_display_name != "" ? var.alert_display_name : "Failure of uptime check for: ${local.uptime_monitoring_display_name}"
+ suffix = var.uptime_monitoring_path != "/" ? var.uptime_monitoring_path : ""
+ uptime_monitoring_display_name = var.uptime_monitoring_display_name != "" ? "${var.uptime_monitoring_display_name} - ${var.uptime_monitoring_host}${local.suffix}" : "${var.uptime_monitoring_host}${local.suffix}"
+ alert_display_name = var.alert_display_name != "" ? var.alert_display_name : "Failure of uptime check for: ${local.uptime_monitoring_display_name}"
}
resource "google_monitoring_uptime_check_config" "https_uptime" {
diff --git a/variables.tf b/variables.tf
index dcde9b9..3f9982c 100644
--- a/variables.tf
+++ b/variables.tf
@@ -69,7 +69,7 @@ variable "cloud_sql" {
}
variable "kyverno" {
- description = "Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace."
+ description = "Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, error pattern inclusions/exclusions for jsonPayload.error matching, and namespace."
default = {}
type = object({
enabled = optional(bool, true)
@@ -81,8 +81,22 @@ variable "kyverno" {
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
- filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
+ # List of error patterns to exclude from the default set.
+ # Default patterns available for exclusion:
+ # "internal error", "failed calling webhook", "timeout", "client-side throttling",
+ # "failed to run warmup", "schema not found", "failed to list resources",
+ # "failed to watch resource", "context deadline exceeded", "is forbidden",
+ # "cannot list resource", "cannot watch resource", "RBAC.*denied",
+ # "failed to start watcher", "leader election lost", "unable to update .*WebhookConfiguration",
+ # "failed to sync", "dropping request", "failed to load certificate",
+ # "failed to update lock", "the object has been modified", "no matches for kind",
+ # "the server could not find the requested resource", "Too Many Requests", "x509",
+ # "is invalid:", "connection refused", "no agent available", "fatal error", "panic"
+ error_patterns_exclude = optional(list(string), [])
+ # List of additional regex error patterns to include (added to default set)
+ # e.g. ["my custom.*error", "failed to connect.*database"]
+ error_patterns_include = optional(list(string), [])
})
validation {
@@ -92,6 +106,86 @@ variable "kyverno" {
)
error_message = "When 'enabled' is true, 'cluster_name' must be provided and cannot be empty."
}
+
+ validation {
+ condition = alltrue([
+ for pattern in var.kyverno.error_patterns_exclude : contains([
+ "internal error",
+ "failed calling webhook",
+ "timeout",
+ "client-side throttling",
+ "failed to run warmup",
+ "schema not found",
+ "failed to list resources",
+ "failed to watch resource",
+ "context deadline exceeded",
+ "is forbidden",
+ "cannot list resource",
+ "cannot watch resource",
+ "RBAC.*denied",
+ "failed to start watcher",
+ "leader election lost",
+ "unable to update .*WebhookConfiguration",
+ "failed to sync",
+ "dropping request",
+ "failed to load certificate",
+ "failed to update lock",
+ "the object has been modified",
+ "no matches for kind",
+ "the server could not find the requested resource",
+ "Too Many Requests",
+ "x509",
+ "is invalid:",
+ "connection refused",
+ "no agent available",
+ "fatal error",
+ "panic",
+ ], pattern)
+ ])
+ error_message = "error_patterns_exclude contains invalid pattern(s). Only default patterns can be excluded. Check the variable description for the list of valid patterns."
+ }
+
+ validation {
+ condition = (
+ !var.kyverno.enabled ||
+ length(setsubtract(
+ toset(concat([
+ "internal error",
+ "failed calling webhook",
+ "timeout",
+ "client-side throttling",
+ "failed to run warmup",
+ "schema not found",
+ "failed to list resources",
+ "failed to watch resource",
+ "context deadline exceeded",
+ "is forbidden",
+ "cannot list resource",
+ "cannot watch resource",
+ "RBAC.*denied",
+ "failed to start watcher",
+ "leader election lost",
+ "unable to update .*WebhookConfiguration",
+ "failed to sync",
+ "dropping request",
+ "failed to load certificate",
+ "failed to update lock",
+ "the object has been modified",
+ "no matches for kind",
+ "the server could not find the requested resource",
+ "Too Many Requests",
+ "x509",
+ "is invalid:",
+ "connection refused",
+ "no agent available",
+ "fatal error",
+ "panic",
+ ], var.kyverno.error_patterns_include)),
+ toset(var.kyverno.error_patterns_exclude)
+ )) > 0
+ )
+ error_message = "The combination of error_patterns_exclude and error_patterns_include results in no active error patterns. At least one pattern must remain active, otherwise the alert will not be created."
+ }
}
variable "cert_manager" {