fix(observability): reject prometheusScrape with no monitors

prometheusScrape.enabled=true with both serviceMonitor.enabled and
podMonitor.enabled false is contradictory: otelCIScrapeEnabled stayed true,
so an idle Target Allocator rendered and CRDs bundled while nothing was
discovered. Add validatePrometheusScrape, invoked from the always-rendered
custom-resource template, to fail loudly with guidance (enable a monitor or
set prometheusScrape.enabled=false). Add a matrix expect_fail assertion.

Author: wenegiemepraise

charts/amazon-cloudwatch-observability/templates/_helpers.tpl

@@ -254,6 +254,20 @@ otelContainerInsights.serviceMonitor.enabled / .podMonitor.enabled if set
 {{- if $v -}}true{{- end -}}
 {{- end -}}
 
+{{/*
+Reject a contradictory scraping config. prometheusScrape.enabled=true with BOTH
+ServiceMonitor and PodMonitor discovery disabled would render an idle Target Allocator
+(and bundle CRDs) that discovers nothing. Fail loudly rather than ship a no-op path.
+Invoked from an always-rendered template so it runs regardless of which agents render.
+*/}}
+{{- define "cloudwatch-agent.validatePrometheusScrape" -}}
+{{- if and .Values.otelContainerInsights.enabled (dig "prometheusScrape" "enabled" true .Values.otelContainerInsights) -}}
+{{- if and (ne (include "cloudwatch-agent.serviceMonitorEnabled" .) "true") (ne (include "cloudwatch-agent.podMonitorEnabled" .) "true") -}}
+{{- fail "otelContainerInsights.prometheusScrape.enabled=true requires at least one of prometheusScrape.serviceMonitor.enabled or prometheusScrape.podMonitor.enabled to be true; enable one, or set prometheusScrape.enabled=false" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Helper function to modify cloudwatch-agent config
 */}}

charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-custom-resource.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.agent.enabled }}
+{{- include "cloudwatch-agent.validatePrometheusScrape" . -}}
 {{- if and (.Values.agent.autoGenerateCert.enabled) (not .Values.agent.certManager.enabled) -}}
 {{- $altNames := list ( printf "%s-service" (include "dcgm-exporter.name" .) ) ( printf "%s-service" (include "neuron-monitor.name" .) ) ( printf "%s-service.%s.svc" (include "dcgm-exporter.name" .) .Release.Namespace ) ( printf "%s-service.%s.svc" (include "neuron-monitor.name" .) .Release.Namespace ) ( printf "%s-service" (include "node-exporter.name" .) ) ( printf "%s-service.%s.svc" (include "node-exporter.name" .) .Release.Namespace ) ( include "kube-state-metrics.name" . ) ( printf "%s.%s.svc" (include "kube-state-metrics.name" .) .Release.Namespace ) -}}
 {{- range $i, $customAgent := .Values.agents }}

charts/amazon-cloudwatch-observability/tests/prometheus_crds_matrix.sh

@@ -53,6 +53,23 @@ expect_count() {
     fi
 }
 
+# expect_fail <desc> <pattern> <render-args...> -- expects helm to error with output matching <pattern>
+expect_fail() {
+    local desc="$1" pattern="$2"; shift 2
+    local out
+    if out="$(render "$@")"; then
+        printf "${R}FAIL${N} %s (expected helm error, render succeeded) [%s]\n" "$desc" "$*"
+        fail_count=$((fail_count+1)); return
+    fi
+    if printf '%s\n' "$out" | grep -qE "$pattern"; then
+        printf "${G}PASS${N} %s (rejected as expected)\n" "$desc"
+        pass_count=$((pass_count+1))
+    else
+        printf "${R}FAIL${N} %s (errored but message missing /%s/) [%s]\n" "$desc" "$pattern" "$*"
+        fail_count=$((fail_count+1))
+    fi
+}
+
 printf "${Y}== CRD bundling gating ==${N}\n"
 # auto (default): bundle only when otelContainerInsights.enabled (and prometheusScrape enabled).
 expect_count "auto + otelCI on  => ServiceMonitor CRD bundled" "$SM_CRD" 1 --set otelContainerInsights.enabled=true
@@ -87,6 +104,14 @@ expect_count "TA absent when scrape off"             'targetAllocator:' 0 --set
 expect_count "SM disabled => serviceMonitorSelector rendered" 'serviceMonitorSelector' 1 --set otelContainerInsights.enabled=true --set otelContainerInsights.prometheusScrape.serviceMonitor.enabled=false
 expect_count "SM disabled => podMonitorSelector absent"       'podMonitorSelector' 0 --set otelContainerInsights.enabled=true --set otelContainerInsights.prometheusScrape.serviceMonitor.enabled=false
 
+printf "\n${Y}== both monitors disabled => rejected ==${N}\n"
+# prometheusScrape.enabled=true with both monitor types off is contradictory (idle TA,
+# bundled CRDs, nothing discovered) and must fail rather than render a no-op path.
+expect_fail "scrape on + both monitors off => render fails" 'requires at least one of' \
+    --set otelContainerInsights.enabled=true \
+    --set otelContainerInsights.prometheusScrape.serviceMonitor.enabled=false \
+    --set otelContainerInsights.prometheusScrape.podMonitor.enabled=false
+
 printf "\n${Y}== per-node allocation strategy default ==${N}\n"
 expect_count "otelCI on => TA allocationStrategy per-node"        'allocationStrategy: "per-node"' 1 --set otelContainerInsights.enabled=true
 expect_count "override => TA allocationStrategy consistent-hashing" 'allocationStrategy: "consistent-hashing"' 1 --set otelContainerInsights.enabled=true --set otelContainerInsights.prometheusScrape.allocationStrategy=consistent-hashing