Merge otel-ci-smpm-scrape-path (aws-observability#329) into helm-per-node-allocation

Re-sync aws-observability#330 onto the updated scraping-path branch: inherit the feature-gate ID
fix, co-gated CRD RBAC render-once ClusterRole/Binding, the $$$1 comment
correction, and the prometheuscr scope-processor guard, alongside aws-observability#330's per-node
work.

Author: wenegiemepraise

charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-cluster-scraper-config.tpl

@@ -231,7 +231,8 @@ processors:
         - k8s.job.name
         - k8s.cronjob.name
       labels:
-        # $$$1 is Helm escaping: $$$ → $$ (Helm) → $ (OTel env resolver) → literal $1 backreference
+        # $$$1 -> literal $1 backreference (group 1 = label key). The agent's OTel confmap
+        # resolves it twice (expandconverter + resolver), each collapsing $$->$; Helm leaves it as-is.
         - tag_name: "k8s.pod.label.$$$1"
           key_regex: "(.*)"
           from: pod

charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-config.tpl

@@ -346,6 +346,7 @@ processors:
           - set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
           - set(attributes["cloudwatch.pipeline"], "efa")
 
+{{- if or .Values.otelContainerInsights.serviceMonitor.enabled .Values.otelContainerInsights.podMonitor.enabled }}
   transform/cw_k8s_ci_v0_set_scope_prometheuscr:
     error_mode: ignore
     metric_statements:
@@ -355,6 +356,7 @@ processors:
           - set(attributes["cloudwatch.source"], "cloudwatch-agent")
           - set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
           - set(attributes["cloudwatch.pipeline"], "prometheus-cr")
+{{- end }}
 
   transform/cw_k8s_ci_v0_set_scope_ebs_csi:
     error_mode: ignore
@@ -454,6 +456,8 @@ processors:
         - k8s.job.name
         - k8s.cronjob.name
       labels:
+        # $$$1 -> literal $1 backreference (group 1 = label key). The agent's OTel confmap
+        # resolves it twice (expandconverter + resolver), each collapsing $$->$; Helm leaves it as-is.
         - tag_name: "k8s.pod.label.$$$1"
           key_regex: "(.*)"
           from: pod

charts/amazon-cloudwatch-observability/templates/operator-deployment.yaml

@@ -37,7 +37,7 @@ spec:
         - "--auto-instrumentation-dotnet-image={{ template "auto-instrumentation-dotnet.image" . }}"
         - "--auto-instrumentation-nodejs-image={{ template "auto-instrumentation-nodejs.image" . }}"
         - "--target-allocator-image={{ template "target-allocator.image" (merge .Values.agent.prometheus.targetAllocator.image (dict "region" $.Values.region)) }}"
-        - "--feature-gates=operator.autoinstrumentation.multi-instrumentation,operator.autoinstrumentation.multi-instrumentation.skip-container-validation"
+        - "--feature-gates=operator.autoinstrumentation.multiinstrumentation,operator.autoinstrumentation.multiinstrumentation.skipcontainervalidation"
         command:
         - /manager
         name: manager

charts/amazon-cloudwatch-observability/templates/target-allocator-clusterrole.yaml

@@ -1,7 +1,22 @@
 {{- if .Values.agent.enabled }}
+{{- /*
+  All Target Allocators (targetAgent and clusterScraperAgent) default to the same
+  ServiceAccount (target-allocator-service-acct; see operator serviceaccount.go), so a single
+  ClusterRole covers every TA. Pre-scan the agents to decide whether any TA (and any
+  prometheusCR discovery) is enabled, then render exactly once to avoid duplicate-name objects.
+*/}}
+{{- $needsTA := false }}
+{{- $needsCR := false }}
 {{- range $i, $customAgent := .Values.agents }}
 {{- $otelCIScrape := eq (include "cloudwatch-agent.otelCIScrapeEnabled" (dict "agentName" $customAgent.name "context" $)) "true" }}
 {{- if or (and (hasKey ($customAgent.prometheus) "targetAllocator") (hasKey ($customAgent.prometheus.targetAllocator) "enabled") $customAgent.prometheus.targetAllocator.enabled) $otelCIScrape }}
+{{- $needsTA = true }}
+{{- end }}
+{{- if or (and (hasKey ($customAgent.prometheus) "targetAllocator") (hasKey ($customAgent.prometheus.targetAllocator) "prometheusCR") $customAgent.prometheus.targetAllocator.prometheusCR.enabled) $otelCIScrape }}
+{{- $needsCR = true }}
+{{- end }}
+{{- end }}
+{{- if $needsTA }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -23,12 +38,14 @@ rules:
     verbs: ["get", "list", "watch"]
   - nonResourceURLs: ["/metrics"]
     verbs: ["get"]
-  {{- if or (and (hasKey ($customAgent.prometheus) "targetAllocator") (hasKey ($customAgent.prometheus.targetAllocator) "prometheusCR") $customAgent.prometheus.targetAllocator.prometheusCR.enabled) $otelCIScrape }}
+  {{- if $needsCR }}
   - apiGroups: [ "monitoring.coreos.com"]
     resources: ["podmonitors", "servicemonitors"]
     verbs: ["get", "list", "watch"]
+  # TA watches the SM/PM CRDs to start/stop informers as they appear or disappear (read-only).
+  - apiGroups: [ "apiextensions.k8s.io" ]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "list", "watch"]
   {{- end }}
 {{- end }}
----
 {{- end }}
-{{- end }}

charts/amazon-cloudwatch-observability/templates/target-allocator-clusterrolebinding.yaml

@@ -1,7 +1,17 @@
 {{- if .Values.agent.enabled }}
+{{- /*
+  Render a single ClusterRoleBinding for the shared Target Allocator ServiceAccount
+  (target-allocator-service-acct), used by every TA (targetAgent and clusterScraperAgent).
+  Pre-scan the agents to decide whether any TA is enabled, then render exactly once.
+*/}}
+{{- $needsTA := false }}
 {{- range $i, $customAgent := .Values.agents }}
 {{- $otelCIScrape := eq (include "cloudwatch-agent.otelCIScrapeEnabled" (dict "agentName" $customAgent.name "context" $)) "true" }}
 {{- if or (and (hasKey ($customAgent.prometheus) "targetAllocator") (hasKey ($customAgent.prometheus.targetAllocator) "enabled") $customAgent.prometheus.targetAllocator.enabled) $otelCIScrape }}
+{{- $needsTA = true }}
+{{- end }}
+{{- end }}
+{{- if $needsTA }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -17,6 +27,4 @@ subjects:
   name: "target-allocator-service-acct"
   namespace: {{ $.Release.Namespace }}
 {{- end }}
----
 {{- end }}
-{{- end }}