From 713b14e9010fb6cd960e2084396749dbaed78495 Mon Sep 17 00:00:00 2001 From: Filippo Date: Fri, 10 Oct 2025 15:00:28 +0200 Subject: [PATCH 1/8] feat: add cert-manager missing issuer log alert configuration and monitoring resource --- README.md | 2 ++ certificate_log_alert.tf | 69 ++++++++++++++++++++++++++++++++++++++++ variables.tf | 18 ++++++++++- 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 certificate_log_alert.tf diff --git a/README.md b/README.md index 18b825f..e8b96ca 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Supported services: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [certificate](#input\_certificate) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = string
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
})
| n/a | yes | | [cloud\_sql](#input\_cloud\_sql) | Configuration for Cloud SQL monitoring alerts. Supports customization of project, auto-close timing, notification channels, and per-instance alert thresholds for CPU, memory, and disk utilization. |
object({
project_id = optional(string, null)
auto_close = optional(string, "86400s") # default 24h
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
instances = optional(map(object({
cpu_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "120s")
duration = optional(string, "300s")
})), [
{
threshold = 0.85,
duration = "1200s",
},
{
severity = "CRITICAL",
threshold = 1,
duration = "300s",
alignment_period = "60s",
}
])
memory_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "300s")
duration = optional(string, "300s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
disk_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.85)
alignment_period = optional(string, "300s")
duration = optional(string, "600s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
})), {})
})
| n/a | yes | | [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. |
object({
enabled = optional(bool, true)
cluster_name = string
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
})
| n/a | yes | | [notification\_channels](#input\_notification\_channels) | List of notification channel IDs to notify when an alert is triggered | `list(string)` | `[]` | no | @@ -50,6 +51,7 @@ Supported services: | Name | Type | |------|------| +| [google_monitoring_alert_policy.certificate_logmatch_alert](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_cpu_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_disk_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_memory_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | diff --git a/certificate_log_alert.tf b/certificate_log_alert.tf new file mode 100644 index 0000000..7e107f6 --- /dev/null +++ b/certificate_log_alert.tf @@ -0,0 +1,69 @@ +locals { + certificate_project_id = var.certificate.project_id != null ? var.certificate.project_id : var.project_id + certificate_alert_documentation = ( + var.certificate.alert_documentation != null + ? var.certificate.alert_documentation + : <<-EOT + cert-manager is reporting that an Issuer or ClusterIssuer resource referenced by a Certificate cannot be found. This may indicate that the Issuer/ClusterIssuer has been deleted or is otherwise unavailable. + EOT + ) + certificate_notification_channels = var.certificate.notification_enabled ? (length(var.certificate.notification_channels) > 0 ? var.certificate.notification_channels : var.notification_channels) : [] + + certificate_log_filter = <<-EOT + ( + ( + resource.type="k8s_container" + AND resource.labels.project_id="${local.certificate_project_id}" + AND resource.labels.cluster_name="${var.certificate.cluster_name}" + AND resource.labels.namespace_name="${var.certificate.namespace}" + ) + OR ( + log_id("events") + AND resource.labels.project_id="${local.certificate_project_id}" + AND resource.labels.cluster_name="${var.certificate.cluster_name}" + AND ( + jsonPayload.involvedObject.namespace="${var.certificate.namespace}" + OR jsonPayload.metadata.namespace="${var.certificate.namespace}" + ) + ) + ) + AND ( + textPayload=~"Referenced \\"(Issuer|ClusterIssuer)\\" not found" + OR jsonPayload.message=~"Referenced \\"(Issuer|ClusterIssuer)\\" not found" + OR jsonPayload.note=~"Referenced \\"(Issuer|ClusterIssuer)\\" not found" + ) + ${trimspace(var.certificate.filter_extra)} + EOT +} + +resource "google_monitoring_alert_policy" "certificate_logmatch_alert" { + count = ( + var.certificate.enabled + && trimspace(var.certificate.cluster_name) != "" + ) ? 1 : 0 + + display_name = "cert-manager missing Issuer/ClusterIssuer (cluster=${var.certificate.cluster_name}, namespace=${var.certificate.namespace})" + combiner = "OR" + enabled = var.certificate.enabled + + conditions { + display_name = "Log match: cert-manager Issuer/ClusterIssuer not found" + condition_matched_log { + filter = local.certificate_log_filter + } + } + + documentation { + content = local.certificate_alert_documentation + mime_type = "text/markdown" + } + + notification_channels = local.certificate_notification_channels + + alert_strategy { + auto_close = "${var.certificate.auto_close_seconds}s" + notification_rate_limit { + period = var.certificate.logmatch_notification_rate_limit + } + } +} diff --git a/variables.tf b/variables.tf index 14a0392..0827445 100644 --- a/variables.tf +++ b/variables.tf @@ -70,7 +70,7 @@ variable "kyverno" { description = "Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace." type = object({ enabled = optional(bool, true) - cluster_name = string + cluster_name = optional(string, null) project_id = optional(string, null) notification_enabled = optional(bool, true) notification_channels = optional(list(string), []) @@ -82,3 +82,19 @@ variable "kyverno" { namespace = optional(string, "kyverno") }) } + +variable "certificate" { + description = "Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting." + type = object({ + enabled = optional(bool, true) + cluster_name = optional(string, null) + project_id = optional(string, null) + namespace = optional(string, "cert-manager") + notification_enabled = optional(bool, true) + notification_channels = optional(list(string), []) + logmatch_notification_rate_limit = optional(string, "300s") + alert_documentation = optional(string, null) + auto_close_seconds = optional(number, 3600) + filter_extra = optional(string, "") + }) +} From 0854d1ee50121e3f531caaa407030509860ec87b Mon Sep 17 00:00:00 2001 From: Filippo Date: Fri, 10 Oct 2025 15:17:59 +0200 Subject: [PATCH 2/8] feat: add cert-manager missing issuer log alert configuration and monitoring resource --- README.md | 6 +-- cert_manager_issuer_log_alert.tf | 69 ++++++++++++++++++++++++++++++++ certificate_log_alert.tf | 69 -------------------------------- cloud-sql.tf => cloud_sql.tf | 0 variables.tf | 6 +-- 5 files changed, 75 insertions(+), 75 deletions(-) create mode 100644 cert_manager_issuer_log_alert.tf delete mode 100644 certificate_log_alert.tf rename cloud-sql.tf => cloud_sql.tf (100%) diff --git a/README.md b/README.md index e8b96ca..c171420 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ Supported services: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [certificate](#input\_certificate) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = string
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
})
| n/a | yes | +| [cert\_manager\_issuer](#input\_cert\_manager\_issuer) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, "")
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
})
| n/a | yes | | [cloud\_sql](#input\_cloud\_sql) | Configuration for Cloud SQL monitoring alerts. Supports customization of project, auto-close timing, notification channels, and per-instance alert thresholds for CPU, memory, and disk utilization. |
object({
project_id = optional(string, null)
auto_close = optional(string, "86400s") # default 24h
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
instances = optional(map(object({
cpu_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "120s")
duration = optional(string, "300s")
})), [
{
threshold = 0.85,
duration = "1200s",
},
{
severity = "CRITICAL",
threshold = 1,
duration = "300s",
alignment_period = "60s",
}
])
memory_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "300s")
duration = optional(string, "300s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
disk_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.85)
alignment_period = optional(string, "300s")
duration = optional(string, "600s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
})), {})
})
| n/a | yes | -| [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. |
object({
enabled = optional(bool, true)
cluster_name = string
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
})
| n/a | yes | +| [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, "")
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
})
| n/a | yes | | [notification\_channels](#input\_notification\_channels) | List of notification channel IDs to notify when an alert is triggered | `list(string)` | `[]` | no | | [project\_id](#input\_project\_id) | The Google Cloud project ID where logging exclusions will be created | `string` | n/a | yes | @@ -51,7 +51,7 @@ Supported services: | Name | Type | |------|------| -| [google_monitoring_alert_policy.certificate_logmatch_alert](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | +| [google_monitoring_alert_policy.cert_manager_issuer_logmatch_alert](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_cpu_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_disk_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_memory_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | diff --git a/cert_manager_issuer_log_alert.tf b/cert_manager_issuer_log_alert.tf new file mode 100644 index 0000000..a65b4fb --- /dev/null +++ b/cert_manager_issuer_log_alert.tf @@ -0,0 +1,69 @@ +locals { + cert_manager_issuer_project_id = var.cert_manager_issuer.project_id != null ? var.cert_manager_issuer.project_id : var.project_id + cert_manager_issuer_alert_documentation = ( + var.cert_manager_issuer.alert_documentation != null + ? var.cert_manager_issuer.alert_documentation + : <<-EOT + cert-manager is reporting that an Issuer or ClusterIssuer resource referenced by a cert_manager_issuer cannot be found. This may indicate that the Issuer/ClusterIssuer has been deleted or is otherwise unavailable. + EOT + ) + cert_manager_issuer_notification_channels = var.cert_manager_issuer.notification_enabled ? (length(var.cert_manager_issuer.notification_channels) > 0 ? var.cert_manager_issuer.notification_channels : var.notification_channels) : [] + + cert_manager_issuer_log_filter = <<-EOT + ( + ( + resource.type="k8s_container" + AND resource.labels.project_id="${local.cert_manager_issuer_project_id}" + AND resource.labels.cluster_name="${var.cert_manager_issuer.cluster_name}" + AND resource.labels.namespace_name="${var.cert_manager_issuer.namespace}" + ) + OR ( + log_id("events") + AND resource.labels.project_id="${local.cert_manager_issuer_project_id}" + AND resource.labels.cluster_name="${var.cert_manager_issuer.cluster_name}" + AND ( + jsonPayload.involvedObject.namespace="${var.cert_manager_issuer.namespace}" + OR jsonPayload.metadata.namespace="${var.cert_manager_issuer.namespace}" + ) + ) + ) + AND ( + textPayload=~"Referenced \"(Issuer|ClusterIssuer)\" not found" + OR jsonPayload.message=~"Referenced \"(Issuer|ClusterIssuer)\" not found" + OR jsonPayload.note=~"Referenced \"(Issuer|ClusterIssuer)\" not found" + ) + ${trimspace(var.cert_manager_issuer.filter_extra)} + EOT +} + +resource "google_monitoring_alert_policy" "cert_manager_issuer_logmatch_alert" { + count = ( + var.cert_manager_issuer.enabled + && trimspace(var.cert_manager_issuer.cluster_name) != "" + ) ? 1 : 0 + + display_name = "cert-manager missing Issuer/ClusterIssuer (cluster=${var.cert_manager_issuer.cluster_name}, namespace=${var.cert_manager_issuer.namespace})" + combiner = "OR" + enabled = var.cert_manager_issuer.enabled + + conditions { + display_name = "Log match: cert-manager Issuer/ClusterIssuer not found" + condition_matched_log { + filter = local.cert_manager_issuer_log_filter + } + } + + documentation { + content = local.cert_manager_issuer_alert_documentation + mime_type = "text/markdown" + } + + notification_channels = local.cert_manager_issuer_notification_channels + + alert_strategy { + auto_close = "${var.cert_manager_issuer.auto_close_seconds}s" + notification_rate_limit { + period = var.cert_manager_issuer.logmatch_notification_rate_limit + } + } +} diff --git a/certificate_log_alert.tf b/certificate_log_alert.tf deleted file mode 100644 index 7e107f6..0000000 --- a/certificate_log_alert.tf +++ /dev/null @@ -1,69 +0,0 @@ -locals { - certificate_project_id = var.certificate.project_id != null ? var.certificate.project_id : var.project_id - certificate_alert_documentation = ( - var.certificate.alert_documentation != null - ? var.certificate.alert_documentation - : <<-EOT - cert-manager is reporting that an Issuer or ClusterIssuer resource referenced by a Certificate cannot be found. This may indicate that the Issuer/ClusterIssuer has been deleted or is otherwise unavailable. - EOT - ) - certificate_notification_channels = var.certificate.notification_enabled ? (length(var.certificate.notification_channels) > 0 ? var.certificate.notification_channels : var.notification_channels) : [] - - certificate_log_filter = <<-EOT - ( - ( - resource.type="k8s_container" - AND resource.labels.project_id="${local.certificate_project_id}" - AND resource.labels.cluster_name="${var.certificate.cluster_name}" - AND resource.labels.namespace_name="${var.certificate.namespace}" - ) - OR ( - log_id("events") - AND resource.labels.project_id="${local.certificate_project_id}" - AND resource.labels.cluster_name="${var.certificate.cluster_name}" - AND ( - jsonPayload.involvedObject.namespace="${var.certificate.namespace}" - OR jsonPayload.metadata.namespace="${var.certificate.namespace}" - ) - ) - ) - AND ( - textPayload=~"Referenced \\"(Issuer|ClusterIssuer)\\" not found" - OR jsonPayload.message=~"Referenced \\"(Issuer|ClusterIssuer)\\" not found" - OR jsonPayload.note=~"Referenced \\"(Issuer|ClusterIssuer)\\" not found" - ) - ${trimspace(var.certificate.filter_extra)} - EOT -} - -resource "google_monitoring_alert_policy" "certificate_logmatch_alert" { - count = ( - var.certificate.enabled - && trimspace(var.certificate.cluster_name) != "" - ) ? 1 : 0 - - display_name = "cert-manager missing Issuer/ClusterIssuer (cluster=${var.certificate.cluster_name}, namespace=${var.certificate.namespace})" - combiner = "OR" - enabled = var.certificate.enabled - - conditions { - display_name = "Log match: cert-manager Issuer/ClusterIssuer not found" - condition_matched_log { - filter = local.certificate_log_filter - } - } - - documentation { - content = local.certificate_alert_documentation - mime_type = "text/markdown" - } - - notification_channels = local.certificate_notification_channels - - alert_strategy { - auto_close = "${var.certificate.auto_close_seconds}s" - notification_rate_limit { - period = var.certificate.logmatch_notification_rate_limit - } - } -} diff --git a/cloud-sql.tf b/cloud_sql.tf similarity index 100% rename from cloud-sql.tf rename to cloud_sql.tf diff --git a/variables.tf b/variables.tf index 0827445..43b7c73 100644 --- a/variables.tf +++ b/variables.tf @@ -70,7 +70,7 @@ variable "kyverno" { description = "Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace." type = object({ enabled = optional(bool, true) - cluster_name = optional(string, null) + cluster_name = optional(string, "") project_id = optional(string, null) notification_enabled = optional(bool, true) notification_channels = optional(list(string), []) @@ -83,11 +83,11 @@ variable "kyverno" { }) } -variable "certificate" { +variable "cert_manager_issuer" { description = "Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting." type = object({ enabled = optional(bool, true) - cluster_name = optional(string, null) + cluster_name = optional(string, "") project_id = optional(string, null) namespace = optional(string, "cert-manager") notification_enabled = optional(bool, true) From b9d14e2e8670f415fd53b00c4149ef6241faf1dd Mon Sep 17 00:00:00 2001 From: Filippo Date: Mon, 13 Oct 2025 12:37:16 +0200 Subject: [PATCH 3/8] add changelog --- CHANGELOG.md | 9 +++++++++ README.md | 4 +++- examples/main.tf | 9 +++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19f98b9..6d0a6f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.4.0] - 2025-10-13 + +[Compare with previous version](https://github.com/sparkfabrik/terraform-google-services-monitoring/compare/0.3.0...0.4.0) + +### changed + +- Rename tf file from `cloud-sql.tf` to `cloud_sql.tf`. +- Add cert-manager missing issuer alert log. + ## [0.3.0] - 2025-10-07 [Compare with previous version](https://github.com/sparkfabrik/terraform-google-services-monitoring/compare/0.2.0...0.3.0) diff --git a/README.md b/README.md index c171420..8d042b9 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,9 @@ Supported services: - Kyverno - Error logs for admission-controller, background-controller, cleanup-controller, reports-controller - - Metric threshold (optional) + +- cert-manager + - Error logs for cert-manager controller when an Issuer or ClusterIssuer is missing ## Providers diff --git a/examples/main.tf b/examples/main.tf index b91edef..4abd3a9 100644 --- a/examples/main.tf +++ b/examples/main.tf @@ -58,4 +58,13 @@ module "example" { # e.g., "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\"" filter_extra = "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\"" } + cert_manager_issuer = { + cluster_name = "test-cluster" + namespace = "cert-manager" + enabled = true + notification_channels = [] + # Optional filter for log entries, exclude known non-actionable messages + # e.g., "-textPayload:\"some known non-actionable message\"" + filter_extra = "" + } } From e60f1b2f9d5fa6f042fb1566a59dc8c941ec6b25 Mon Sep 17 00:00:00 2001 From: Filippo Date: Mon, 13 Oct 2025 13:04:12 +0200 Subject: [PATCH 4/8] Rename --- CHANGELOG.md | 1 + README.md | 4 +- cert_manager.tf | 69 ++++++++++++++++++++++++++++++ cert_manager_issuer_log_alert.tf | 69 ------------------------------ kyverno_log_alert.tf => kyverno.tf | 0 variables.tf | 2 +- 6 files changed, 73 insertions(+), 72 deletions(-) create mode 100644 cert_manager.tf delete mode 100644 cert_manager_issuer_log_alert.tf rename kyverno_log_alert.tf => kyverno.tf (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d0a6f7..a34555c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### changed - Rename tf file from `cloud-sql.tf` to `cloud_sql.tf`. +- Rename tf file from `kyverno_log_alert.tf` to `kyverno.tf`. - Add cert-manager missing issuer alert log. ## [0.3.0] - 2025-10-07 diff --git a/README.md b/README.md index 8d042b9..c44eb61 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Supported services: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cert\_manager\_issuer](#input\_cert\_manager\_issuer) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, "")
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
})
| n/a | yes | +| [cert\_manager](#input\_cert\_manager) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, "")
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
})
| n/a | yes | | [cloud\_sql](#input\_cloud\_sql) | Configuration for Cloud SQL monitoring alerts. Supports customization of project, auto-close timing, notification channels, and per-instance alert thresholds for CPU, memory, and disk utilization. |
object({
project_id = optional(string, null)
auto_close = optional(string, "86400s") # default 24h
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
instances = optional(map(object({
cpu_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "120s")
duration = optional(string, "300s")
})), [
{
threshold = 0.85,
duration = "1200s",
},
{
severity = "CRITICAL",
threshold = 1,
duration = "300s",
alignment_period = "60s",
}
])
memory_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "300s")
duration = optional(string, "300s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
disk_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.85)
alignment_period = optional(string, "300s")
duration = optional(string, "600s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
})), {})
})
| n/a | yes | | [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, "")
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
})
| n/a | yes | | [notification\_channels](#input\_notification\_channels) | List of notification channel IDs to notify when an alert is triggered | `list(string)` | `[]` | no | @@ -53,7 +53,7 @@ Supported services: | Name | Type | |------|------| -| [google_monitoring_alert_policy.cert_manager_issuer_logmatch_alert](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | +| [google_monitoring_alert_policy.cert_manager_logmatch_alert](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_cpu_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_disk_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | | [google_monitoring_alert_policy.cloud_sql_memory_utilization](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource | diff --git a/cert_manager.tf b/cert_manager.tf new file mode 100644 index 0000000..b09e227 --- /dev/null +++ b/cert_manager.tf @@ -0,0 +1,69 @@ +locals { + cert_manager_project_id = var.cert_manager.project_id != null ? var.cert_manager.project_id : var.project_id + cert_manager_alert_documentation = ( + var.cert_manager.alert_documentation != null + ? var.cert_manager.alert_documentation + : <<-EOT + cert-manager is reporting that an Issuer or ClusterIssuer resource referenced by a cert_manager cannot be found. This may indicate that the Issuer/ClusterIssuer has been deleted or is otherwise unavailable. + EOT + ) + cert_manager_notification_channels = var.cert_manager.notification_enabled ? (length(var.cert_manager.notification_channels) > 0 ? var.cert_manager.notification_channels : var.notification_channels) : [] + + cert_manager_log_filter = <<-EOT + ( + ( + resource.type="k8s_container" + AND resource.labels.project_id="${local.cert_manager_project_id}" + AND resource.labels.cluster_name="${var.cert_manager.cluster_name}" + AND resource.labels.namespace_name="${var.cert_manager.namespace}" + ) + OR ( + log_id("events") + AND resource.labels.project_id="${local.cert_manager_project_id}" + AND resource.labels.cluster_name="${var.cert_manager.cluster_name}" + AND ( + jsonPayload.involvedObject.namespace="${var.cert_manager.namespace}" + OR jsonPayload.metadata.namespace="${var.cert_manager.namespace}" + ) + ) + ) + AND ( + textPayload=~"Referenced \"(Issuer|ClusterIssuer)\" not found" + OR jsonPayload.message=~"Referenced \"(Issuer|ClusterIssuer)\" not found" + OR jsonPayload.note=~"Referenced \"(Issuer|ClusterIssuer)\" not found" + ) + ${trimspace(var.cert_manager.filter_extra)} + EOT +} + +resource "google_monitoring_alert_policy" "cert_manager_logmatch_alert" { + count = ( + var.cert_manager.enabled + && trimspace(var.cert_manager.cluster_name) != "" + ) ? 1 : 0 + + display_name = "cert-manager missing Issuer/ClusterIssuer (cluster=${var.cert_manager.cluster_name}, namespace=${var.cert_manager.namespace})" + combiner = "OR" + enabled = var.cert_manager.enabled + + conditions { + display_name = "Log match: cert-manager Issuer/ClusterIssuer not found" + condition_matched_log { + filter = local.cert_manager_log_filter + } + } + + documentation { + content = local.cert_manager_alert_documentation + mime_type = "text/markdown" + } + + notification_channels = local.cert_manager_notification_channels + + alert_strategy { + auto_close = "${var.cert_manager.auto_close_seconds}s" + notification_rate_limit { + period = var.cert_manager.logmatch_notification_rate_limit + } + } +} diff --git a/cert_manager_issuer_log_alert.tf b/cert_manager_issuer_log_alert.tf deleted file mode 100644 index a65b4fb..0000000 --- a/cert_manager_issuer_log_alert.tf +++ /dev/null @@ -1,69 +0,0 @@ -locals { - cert_manager_issuer_project_id = var.cert_manager_issuer.project_id != null ? var.cert_manager_issuer.project_id : var.project_id - cert_manager_issuer_alert_documentation = ( - var.cert_manager_issuer.alert_documentation != null - ? var.cert_manager_issuer.alert_documentation - : <<-EOT - cert-manager is reporting that an Issuer or ClusterIssuer resource referenced by a cert_manager_issuer cannot be found. This may indicate that the Issuer/ClusterIssuer has been deleted or is otherwise unavailable. - EOT - ) - cert_manager_issuer_notification_channels = var.cert_manager_issuer.notification_enabled ? (length(var.cert_manager_issuer.notification_channels) > 0 ? var.cert_manager_issuer.notification_channels : var.notification_channels) : [] - - cert_manager_issuer_log_filter = <<-EOT - ( - ( - resource.type="k8s_container" - AND resource.labels.project_id="${local.cert_manager_issuer_project_id}" - AND resource.labels.cluster_name="${var.cert_manager_issuer.cluster_name}" - AND resource.labels.namespace_name="${var.cert_manager_issuer.namespace}" - ) - OR ( - log_id("events") - AND resource.labels.project_id="${local.cert_manager_issuer_project_id}" - AND resource.labels.cluster_name="${var.cert_manager_issuer.cluster_name}" - AND ( - jsonPayload.involvedObject.namespace="${var.cert_manager_issuer.namespace}" - OR jsonPayload.metadata.namespace="${var.cert_manager_issuer.namespace}" - ) - ) - ) - AND ( - textPayload=~"Referenced \"(Issuer|ClusterIssuer)\" not found" - OR jsonPayload.message=~"Referenced \"(Issuer|ClusterIssuer)\" not found" - OR jsonPayload.note=~"Referenced \"(Issuer|ClusterIssuer)\" not found" - ) - ${trimspace(var.cert_manager_issuer.filter_extra)} - EOT -} - -resource "google_monitoring_alert_policy" "cert_manager_issuer_logmatch_alert" { - count = ( - var.cert_manager_issuer.enabled - && trimspace(var.cert_manager_issuer.cluster_name) != "" - ) ? 1 : 0 - - display_name = "cert-manager missing Issuer/ClusterIssuer (cluster=${var.cert_manager_issuer.cluster_name}, namespace=${var.cert_manager_issuer.namespace})" - combiner = "OR" - enabled = var.cert_manager_issuer.enabled - - conditions { - display_name = "Log match: cert-manager Issuer/ClusterIssuer not found" - condition_matched_log { - filter = local.cert_manager_issuer_log_filter - } - } - - documentation { - content = local.cert_manager_issuer_alert_documentation - mime_type = "text/markdown" - } - - notification_channels = local.cert_manager_issuer_notification_channels - - alert_strategy { - auto_close = "${var.cert_manager_issuer.auto_close_seconds}s" - notification_rate_limit { - period = var.cert_manager_issuer.logmatch_notification_rate_limit - } - } -} diff --git a/kyverno_log_alert.tf b/kyverno.tf similarity index 100% rename from kyverno_log_alert.tf rename to kyverno.tf diff --git a/variables.tf b/variables.tf index 43b7c73..286954f 100644 --- a/variables.tf +++ b/variables.tf @@ -83,7 +83,7 @@ variable "kyverno" { }) } -variable "cert_manager_issuer" { +variable "cert_manager" { description = "Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting." type = object({ enabled = optional(bool, true) From 6abe7026407346362c9f7fc5a4aac68fb7a198be Mon Sep 17 00:00:00 2001 From: Filippo Date: Mon, 13 Oct 2025 13:08:45 +0200 Subject: [PATCH 5/8] fix example --- README.md | 4 ++-- examples/main.tf | 13 ++++--------- examples/variables.tf | 16 ++++++++++++++++ variables.tf | 4 ++-- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c44eb61..e2738e8 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,9 @@ Supported services: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cert\_manager](#input\_cert\_manager) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, "")
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
})
| n/a | yes | +| [cert\_manager](#input\_cert\_manager) | Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting. |
object({
enabled = optional(bool, true)
cluster_name = string
project_id = optional(string, null)
namespace = optional(string, "cert-manager")
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
})
| n/a | yes | | [cloud\_sql](#input\_cloud\_sql) | Configuration for Cloud SQL monitoring alerts. Supports customization of project, auto-close timing, notification channels, and per-instance alert thresholds for CPU, memory, and disk utilization. |
object({
project_id = optional(string, null)
auto_close = optional(string, "86400s") # default 24h
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
instances = optional(map(object({
cpu_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "120s")
duration = optional(string, "300s")
})), [
{
threshold = 0.85,
duration = "1200s",
},
{
severity = "CRITICAL",
threshold = 1,
duration = "300s",
alignment_period = "60s",
}
])
memory_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.90)
alignment_period = optional(string, "300s")
duration = optional(string, "300s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
disk_utilization = optional(list(object({
severity = optional(string, "WARNING"),
threshold = optional(number, 0.85)
alignment_period = optional(string, "300s")
duration = optional(string, "600s")
})), [
{
severity = "WARNING",
},
{
severity = "CRITICAL",
threshold = 0.95,
}
])
})), {})
})
| n/a | yes | -| [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. |
object({
enabled = optional(bool, true)
cluster_name = optional(string, "")
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
})
| n/a | yes | +| [kyverno](#input\_kyverno) | Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace. |
object({
enabled = optional(bool, true)
cluster_name = string
project_id = optional(string, null)
notification_enabled = optional(bool, true)
notification_channels = optional(list(string), [])
# Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts
logmatch_notification_rate_limit = optional(string, "300s")
alert_documentation = optional(string, null)
auto_close_seconds = optional(number, 3600)
filter_extra = optional(string, "")
namespace = optional(string, "kyverno")
})
| n/a | yes | | [notification\_channels](#input\_notification\_channels) | List of notification channel IDs to notify when an alert is triggered | `list(string)` | `[]` | no | | [project\_id](#input\_project\_id) | The Google Cloud project ID where logging exclusions will be created | `string` | n/a | yes | diff --git a/examples/main.tf b/examples/main.tf index 4abd3a9..92ffa3b 100644 --- a/examples/main.tf +++ b/examples/main.tf @@ -49,22 +49,17 @@ module "example" { project_id = var.project_id cloud_sql = local.cloud_sql kyverno = { - cluster_name = "test-cluster" - enabled = true - use_metric_threshold = true - metric_threshold_count = 5 - notification_channels = [] + cluster_name = "test-cluster" + enabled = true + notification_channels = [] # Optional filter for log entries, exclude known non-actionable messages # e.g., "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\"" filter_extra = "-textPayload:\"stale GroupVersion discovery: metrics.k8s.io/v1beta1\"" } - cert_manager_issuer = { + cert_manager = { cluster_name = "test-cluster" namespace = "cert-manager" enabled = true notification_channels = [] - # Optional filter for log entries, exclude known non-actionable messages - # e.g., "-textPayload:\"some known non-actionable message\"" - filter_extra = "" } } diff --git a/examples/variables.tf b/examples/variables.tf index 2a09651..f00efc8 100644 --- a/examples/variables.tf +++ b/examples/variables.tf @@ -26,3 +26,19 @@ variable "kyverno" { filter_extra = optional(string, "") }) } + +variable "cert_manager" { + description = "Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting." + type = object({ + enabled = optional(bool, true) + cluster_name = optional(string, "") + project_id = optional(string, null) + namespace = optional(string, "cert-manager") + notification_enabled = optional(bool, true) + notification_channels = optional(list(string), []) + logmatch_notification_rate_limit = optional(string, "300s") + alert_documentation = optional(string, null) + auto_close_seconds = optional(number, 3600) + filter_extra = optional(string, "") + }) +} diff --git a/variables.tf b/variables.tf index 286954f..49c0f99 100644 --- a/variables.tf +++ b/variables.tf @@ -70,7 +70,7 @@ variable "kyverno" { description = "Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace." type = object({ enabled = optional(bool, true) - cluster_name = optional(string, "") + cluster_name = string project_id = optional(string, null) notification_enabled = optional(bool, true) notification_channels = optional(list(string), []) @@ -87,7 +87,7 @@ variable "cert_manager" { description = "Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting." type = object({ enabled = optional(bool, true) - cluster_name = optional(string, "") + cluster_name = string project_id = optional(string, null) namespace = optional(string, "cert-manager") notification_enabled = optional(bool, true) From 8395d891fd7503bdb29e32d2009cd50e51b1478f Mon Sep 17 00:00:00 2001 From: Filippo Date: Mon, 13 Oct 2025 13:18:52 +0200 Subject: [PATCH 6/8] remove optional clustername --- cert_manager.tf | 1 + kyverno.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/cert_manager.tf b/cert_manager.tf index b09e227..06871e7 100644 --- a/cert_manager.tf +++ b/cert_manager.tf @@ -40,6 +40,7 @@ resource "google_monitoring_alert_policy" "cert_manager_logmatch_alert" { count = ( var.cert_manager.enabled && trimspace(var.cert_manager.cluster_name) != "" + && var.cert_manager.cluster_name != null ) ? 1 : 0 display_name = "cert-manager missing Issuer/ClusterIssuer (cluster=${var.cert_manager.cluster_name}, namespace=${var.cert_manager.namespace})" diff --git a/kyverno.tf b/kyverno.tf index 048a1b4..45f3d5c 100644 --- a/kyverno.tf +++ b/kyverno.tf @@ -21,6 +21,7 @@ resource "google_monitoring_alert_policy" "kyverno_logmatch_alert" { count = ( var.kyverno.enabled && trimspace(var.kyverno.cluster_name) != "" + && var.kyverno.cluster_name != null ) ? 1 : 0 display_name = "Kyverno controllers ERROR logs (namespace=${var.kyverno.namespace})" From a7046684dbfed5e479a66d0192bc2ada82f99bd9 Mon Sep 17 00:00:00 2001 From: Filippo Date: Mon, 13 Oct 2025 13:34:27 +0200 Subject: [PATCH 7/8] fix example --- examples/variables.tf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/variables.tf b/examples/variables.tf index f00efc8..c06bf12 100644 --- a/examples/variables.tf +++ b/examples/variables.tf @@ -13,17 +13,17 @@ variable "notification_channels" { variable "kyverno" { description = "Configuration for Kyverno monitoring alerts. Allows customization of cluster name, project, notification channels, alert documentation, metric thresholds, auto-close timing, enablement, extra filters, and namespace." type = object({ - enabled = optional(bool, true) - project_id = optional(string, null) - cluster_name = string - namespace = optional(string, "kyverno") - notification_enabled = optional(bool, true) - notification_channels = optional(list(string), []) - alert_documentation = optional(string, null) - metric_threshold_count = optional(number, 2) - metric_lookback_minutes = optional(number, 1) - auto_close_seconds = optional(number, 3600) - filter_extra = optional(string, "") + enabled = optional(bool, true) + cluster_name = string + project_id = optional(string, null) + notification_enabled = optional(bool, true) + notification_channels = optional(list(string), []) + # Rate limit for notifications, e.g. "300s" for 5 minutes, used only for log match alerts + logmatch_notification_rate_limit = optional(string, "300s") + alert_documentation = optional(string, null) + auto_close_seconds = optional(number, 3600) + filter_extra = optional(string, "") + namespace = optional(string, "kyverno") }) } @@ -31,7 +31,7 @@ variable "cert_manager" { description = "Configuration for cert-manager missing issuer log alert. Allows customization of project, cluster, namespace, notification channels, alert documentation, enablement, extra filters, auto-close timing, and notification rate limiting." type = object({ enabled = optional(bool, true) - cluster_name = optional(string, "") + cluster_name = string project_id = optional(string, null) namespace = optional(string, "cert-manager") notification_enabled = optional(bool, true) From 12e25d31bd0724b6db83d447d654f0269689da1c Mon Sep 17 00:00:00 2001 From: Filippo Merante Caparrotta <61194571+filippolmt@users.noreply.github.com> Date: Mon, 13 Oct 2025 13:48:14 +0200 Subject: [PATCH 8/8] Update cert_manager.tf Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- cert_manager.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cert_manager.tf b/cert_manager.tf index 06871e7..a27df28 100644 --- a/cert_manager.tf +++ b/cert_manager.tf @@ -4,7 +4,7 @@ locals { var.cert_manager.alert_documentation != null ? var.cert_manager.alert_documentation : <<-EOT - cert-manager is reporting that an Issuer or ClusterIssuer resource referenced by a cert_manager cannot be found. This may indicate that the Issuer/ClusterIssuer has been deleted or is otherwise unavailable. + cert-manager is reporting that an Issuer or ClusterIssuer resource referenced by a Certificate cannot be found. This may indicate that the Issuer/ClusterIssuer has been deleted or is otherwise unavailable. EOT ) cert_manager_notification_channels = var.cert_manager.notification_enabled ? (length(var.cert_manager.notification_channels) > 0 ? var.cert_manager.notification_channels : var.notification_channels) : []