[amazon-cloudwatch-observability] refactor: single-switch SM/PM scraping under otelContainerInsights.prometheusScrape

charts/amazon-cloudwatch-observability/crds/cloudwatch.aws.amazon.com_amazoncloudwatchagents.yaml

@@ -6405,9 +6405,10 @@ spec:
                   allocationStrategy:
                     description: |-
                       AllocationStrategy determines which strategy the target allocator should use for allocation.
-                      The current option is consistent-hashing.
+                      The options are consistent-hashing and per-node.
                     enum:
                     - consistent-hashing
+                    - per-node
                     type: string
                   enabled:
                     description: Enabled indicates whether to use a target allocation

charts/amazon-cloudwatch-observability/templates/_helpers.tpl

@@ -5,6 +5,33 @@ Expand the name of the chart.
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Whether to bundle the community ServiceMonitor/PodMonitor CRDs. Honours
+.Values.otelContainerInsights.prometheusScrape.crds.install:
+  "always" => true; "never" => empty;
+  "auto" (default) => true only when otelContainerInsights.enabled AND
+  otelContainerInsights.prometheusScrape.enabled are both true.
+Returns the string "true" when CRDs should be rendered, empty otherwise.
+*/}}
+{{- define "amazon-cloudwatch-observability.prometheusCRDsEnabled" -}}
+{{- $install := (dig "prometheusScrape" "crds" "install" "auto" .Values.otelContainerInsights) -}}
+{{- /* Back-compat: honor the legacy top-level prometheusCRDs.install if set (deprecated). */ -}}
+{{- if hasKey .Values "prometheusCRDs" -}}
+{{- $install = (dig "install" $install .Values.prometheusCRDs) -}}
+{{- end -}}
+{{- $scrapeEnabled := (dig "prometheusScrape" "enabled" true .Values.otelContainerInsights) -}}
+{{- if eq $install "always" -}}
+true
+{{- else if eq $install "never" -}}
+{{- else if eq $install "auto" -}}
+{{- if and .Values.otelContainerInsights.enabled $scrapeEnabled -}}
+true
+{{- end -}}
+{{- else -}}
+{{- fail (printf "prometheusCRDs.install must be one of \"auto\", \"always\", or \"never\", got: %s" $install) -}}
+{{- end -}}
+{{- end -}}
+
 {{- define "amazon-cloudwatch-observability.common.tolerations" -}}
 {{- $tolerations := .context.Values.tolerations }}
 {{- if .component }}
@@ -191,6 +218,56 @@ Logic:
 {{- end -}}
 {{- end -}}
 
+{{/*
+Returns "true" when otelContainerInsights-driven ServiceMonitor/PodMonitor scraping
+applies to the given agent. True when otelContainerInsights is enabled, the agent is
+the configured targetAgent, and at least one of serviceMonitor/podMonitor is enabled.
+Accepts a dict with "agentName" (string) and "context" (root context $).
+*/}}
+{{- define "cloudwatch-agent.otelCIScrapeEnabled" -}}
+{{- $ctx := .context -}}
+{{- $agentName := .agentName -}}
+{{- if and $ctx.Values.otelContainerInsights.enabled (eq $agentName $ctx.Values.otelContainerInsights.targetAgent) (dig "prometheusScrape" "enabled" true $ctx.Values.otelContainerInsights) -}}
+true
+{{- end -}}
+{{- end -}}
+
+{{/*
+Whether ServiceMonitor / PodMonitor discovery is enabled. Honors the legacy
+otelContainerInsights.serviceMonitor.enabled / .podMonitor.enabled if set
+(deprecated), otherwise otelContainerInsights.prometheusScrape.<monitor>.enabled
+(default true). Return "true" when enabled, empty otherwise.
+*/}}
+{{- define "cloudwatch-agent.serviceMonitorEnabled" -}}
+{{- $v := dig "prometheusScrape" "serviceMonitor" "enabled" true .Values.otelContainerInsights -}}
+{{- if hasKey .Values.otelContainerInsights "serviceMonitor" -}}
+{{- $v = dig "serviceMonitor" "enabled" $v .Values.otelContainerInsights -}}
+{{- end -}}
+{{- if $v -}}true{{- end -}}
+{{- end -}}
+
+{{- define "cloudwatch-agent.podMonitorEnabled" -}}
+{{- $v := dig "prometheusScrape" "podMonitor" "enabled" true .Values.otelContainerInsights -}}
+{{- if hasKey .Values.otelContainerInsights "podMonitor" -}}
+{{- $v = dig "podMonitor" "enabled" $v .Values.otelContainerInsights -}}
+{{- end -}}
+{{- if $v -}}true{{- end -}}
+{{- end -}}
+
+{{/*
+Reject a contradictory scraping config. prometheusScrape.enabled=true with BOTH
+ServiceMonitor and PodMonitor discovery disabled would render an idle Target Allocator
+(and bundle CRDs) that discovers nothing. Fail loudly rather than ship a no-op path.
+Invoked from an always-rendered template so it runs regardless of which agents render.
+*/}}
+{{- define "cloudwatch-agent.validatePrometheusScrape" -}}
+{{- if and .Values.otelContainerInsights.enabled (dig "prometheusScrape" "enabled" true .Values.otelContainerInsights) -}}
+{{- if and (ne (include "cloudwatch-agent.serviceMonitorEnabled" .) "true") (ne (include "cloudwatch-agent.podMonitorEnabled" .) "true") -}}
+{{- fail "otelContainerInsights.prometheusScrape.enabled=true requires at least one of prometheusScrape.serviceMonitor.enabled or prometheusScrape.podMonitor.enabled to be true; enable one, or set prometheusScrape.enabled=false" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Helper function to modify cloudwatch-agent config
 */}}

charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-cluster-scraper-config.tpl

@@ -231,7 +231,8 @@ processors:
         - k8s.job.name
         - k8s.cronjob.name
       labels:
-        # $$$1 is Helm escaping: $$$ → $$ (Helm) → $ (OTel env resolver) → literal $1 backreference
+        # $$$1 -> literal $1 backreference (group 1 = label key). The agent's OTel confmap
+        # resolves it twice (expandconverter + resolver), each collapsing $$->$; Helm leaves it as-is.
         - tag_name: "k8s.pod.label.$$$1"
           key_regex: "(.*)"
           from: pod

charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-config.tpl

@@ -43,6 +43,23 @@ receivers:
             - targets:
                 - ${env:HOST_IP}:10250
 
+{{- if dig "prometheusScrape" "enabled" true .Values.otelContainerInsights }}
+  # ServiceMonitor/PodMonitor scraping via the Target Allocator (prometheusCR discovery).
+  # The Target Allocator deployed for the targetAgent serves the scrape jobs derived from
+  # ServiceMonitor/PodMonitor CRs; this receiver pulls this collector's assigned shard and
+  # routes the series into the v2 OTLP pipeline (-> CloudWatch/Zeus). Requires the Target
+  # Allocator + prometheusCR to be enabled for the targetAgent and the POD_NAME env (set below).
+  prometheus/cw_k8s_ci_v0_prometheuscr:
+    target_allocator:
+      endpoint: https://{{ .Values.otelContainerInsights.targetAgent }}-target-allocator-service:80
+      interval: {{ .Values.otelContainerInsights.metricResolution }}
+      collector_id: ${env:POD_NAME}
+      tls:
+        ca_file: /etc/amazon-cloudwatch-observability-agent-cert/tls-ca.crt
+        cert_file: /etc/amazon-cloudwatch-observability-agent-ta-client-cert/client.crt
+        key_file: /etc/amazon-cloudwatch-observability-agent-ta-client-cert/client.key
+{{- end }}
+
   {{- if .Values.dcgmExporter.enabled }}
   prometheus/cw_k8s_ci_v0_dcgm:
     config:
@@ -329,6 +346,18 @@ processors:
           - set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
           - set(attributes["cloudwatch.pipeline"], "efa")
 
+{{- if dig "prometheusScrape" "enabled" true .Values.otelContainerInsights }}
+  transform/cw_k8s_ci_v0_set_scope_prometheuscr:
+    error_mode: ignore
+    metric_statements:
+      - context: scope
+        statements:
+          - set(scope.schema_url, "")
+          - set(attributes["cloudwatch.source"], "cloudwatch-agent")
+          - set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
+          - set(attributes["cloudwatch.pipeline"], "prometheus-cr")
+{{- end }}
+
   transform/cw_k8s_ci_v0_set_scope_ebs_csi:
     error_mode: ignore
     metric_statements:
@@ -427,6 +456,8 @@ processors:
         - k8s.job.name
         - k8s.cronjob.name
       labels:
+        # $$$1 -> literal $1 backreference (group 1 = label key). The agent's OTel confmap
+        # resolves it twice (expandconverter + resolver), each collapsing $$->$; Helm leaves it as-is.
         - tag_name: "k8s.pod.label.$$$1"
           key_regex: "(.*)"
           from: pod
@@ -829,6 +860,20 @@ service:
       exporters:
         - otlphttp/cw_k8s_ci_v0_metrics_dest
 
+{{- if dig "prometheusScrape" "enabled" true .Values.otelContainerInsights }}
+    metrics/cw_k8s_ci_v0_prometheuscr:
+      receivers: [prometheus/cw_k8s_ci_v0_prometheuscr]
+      processors:
+        - filter/cw_k8s_ci_v0_scrape_metadata
+        - metricstarttime/cw_k8s_ci_v0
+        - transform/cw_k8s_ci_v0_set_cluster_name
+        - transform/cw_k8s_ci_v0_set_scope_prometheuscr
+        - resourcedetection/cw_k8s_ci_v0
+        - batch/cw_k8s_ci_v0_metrics_dest
+      exporters:
+        - otlphttp/cw_k8s_ci_v0_metrics_dest
+{{- end }}
+
     {{- if .Values.dcgmExporter.enabled }}
     metrics/cw_k8s_ci_v0_dcgm:
       receivers: [prometheus/cw_k8s_ci_v0_dcgm]

charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-custom-resource.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.agent.enabled }}
+{{- include "cloudwatch-agent.validatePrometheusScrape" . -}}
 {{- if and (.Values.agent.autoGenerateCert.enabled) (not .Values.agent.certManager.enabled) -}}
 {{- $altNames := list ( printf "%s-service" (include "dcgm-exporter.name" .) ) ( printf "%s-service" (include "neuron-monitor.name" .) ) ( printf "%s-service.%s.svc" (include "dcgm-exporter.name" .) .Release.Namespace ) ( printf "%s-service.%s.svc" (include "neuron-monitor.name" .) .Release.Namespace ) ( printf "%s-service" (include "node-exporter.name" .) ) ( printf "%s-service.%s.svc" (include "node-exporter.name" .) .Release.Namespace ) ( include "kube-state-metrics.name" . ) ( printf "%s.%s.svc" (include "kube-state-metrics.name" .) .Release.Namespace ) -}}
 {{- range $i, $customAgent := .Values.agents }}
@@ -110,21 +111,50 @@ spec:
   {{- else if ne (trimAll " \n\t" $generatedOtelConfig) "{}" }}
   otelConfig: {{ include "cloudwatch-agent.modify-otel-config" (merge (dict "OtelConfig" $generatedOtelConfig) $) }}
   {{- end }}
+  {{- $otelCIScrape := eq (include "cloudwatch-agent.otelCIScrapeEnabled" (dict "agentName" $agent.name "context" $)) "true" }}
+  {{- $taEnabled := or $agent.prometheus.targetAllocator.enabled $otelCIScrape }}
+  {{- $allocationStrategy := $agent.prometheus.targetAllocator.allocationStrategy | default "consistent-hashing" }}
+  {{- if $otelCIScrape }}
+  {{- $allocationStrategy = (dig "prometheusScrape" "allocationStrategy" "per-node" $.Values.otelContainerInsights) }}
+  {{- end }}
   {{- if $agent.prometheus.config }}
   prometheus:
     {{- with $agent.prometheus.config }}
     config:
       {{- toYaml . | nindent 6 }}
     {{- end }}
+  {{- else if $taEnabled }}
+  # The operator builds the Target Allocator config from spec.prometheus and requires
+  # a prometheus.config containing a scrape_configs key (see operator
+  # targetallocator/adapters GetPromConfig). With prometheusCR (ServiceMonitor/
+  # PodMonitor) discovery the static scrape list is empty -- the Target Allocator
+  # discovers targets from ServiceMonitor/PodMonitor CRs.
+  prometheus:
+    config:
+      scrape_configs: []
   {{- end }}
-  {{- if $agent.prometheus.targetAllocator.enabled }}
+  {{- if $taEnabled }}
   targetAllocator:
-    enabled: {{ $agent.prometheus.targetAllocator.enabled | default false }}
+    enabled: true
     image: {{ template "target-allocator.image" (merge $agent.prometheus.targetAllocator.image (dict "region" $.Values.region)) }}
-    allocationStrategy: "consistent-hashing"
-    {{- if $agent.prometheus.targetAllocator.prometheusCR.enabled }}
+    allocationStrategy: {{ $allocationStrategy | quote }}
+    {{- if or $agent.prometheus.targetAllocator.prometheusCR.enabled $otelCIScrape }}
     prometheusCR:
-      enabled: {{ $agent.prometheus.targetAllocator.prometheusCR.enabled | default false }}
+      enabled: true
+      {{- if $otelCIScrape }}
+      scrapeInterval: {{ $.Values.otelContainerInsights.metricResolution | quote }}
+      {{- if ne (include "cloudwatch-agent.serviceMonitorEnabled" $) "true" }}
+      # serviceMonitor disabled: select a label no real ServiceMonitor
+      # carries, so the Target Allocator discovers no ServiceMonitors.
+      serviceMonitorSelector:
+        amazon-cloudwatch-observability.aws/otel-ci-scrape: disabled
+      {{- end }}
+      {{- if ne (include "cloudwatch-agent.podMonitorEnabled" $) "true" }}
+      # podMonitor disabled: select a label no real PodMonitor carries.
+      podMonitorSelector:
+        amazon-cloudwatch-observability.aws/otel-ci-scrape: disabled
+      {{- end }}
+      {{- end }}
     {{- end }}
   {{- end }}
   {{- with $agent.resources }}
@@ -281,6 +311,12 @@ spec:
   {{- if $.Values.otelContainerInsights.enabled }}
   - name: OTEL_CI_VERSION
     value: "1.0.0"
+  # POD_NAME is used as the Target Allocator collector_id by the prometheusCR
+  # receiver (otel-container-insights.config) so each agent pulls its own shard.
+  - name: POD_NAME
+    valueFrom:
+      fieldRef:
+        fieldPath: metadata.name
   {{- end }}
   {{- with $agent.env }}
   {{- . | toYaml | nindent 2 }}