migrate chart to use CWA json

Author: Paamicky

charts/amazon-cloudwatch-observability/templates/_helpers.tpl

@@ -130,6 +130,17 @@ Logic:
   {{- $needsLogs = true -}}
   {{- $_ := set $metricsCollected "kubernetes" (dict "enhanced_container_insights" true) -}}
 {{- end -}}
+{{/* OTEL CI: emit JSON config; the agent generates the pipeline (node=targetAgent, cluster=clusterScraperAgent). */}}
+{{- if $ctx.Values.otelContainerInsights.enabled -}}
+  {{- $interval := atoi (trimSuffix "s" $ctx.Values.otelContainerInsights.metricResolution) -}}
+  {{- if eq $ctx.Values.otelContainerInsights.targetAgent $agentName -}}
+    {{- $ci := dict "collection_interval" $interval "role" "node" "logs" (dict "enabled" $ctx.Values.otelContainerInsights.logs.enabled) -}}
+    {{- $_ := set $config "opentelemetry" (dict "cluster_name" $ctx.Values.clusterName "collect" (dict "container_insights" $ci)) -}}
+  {{- else if eq $ctx.Values.otelContainerInsights.clusterScraperAgent $agentName -}}
+    {{- $ci := dict "collection_interval" $interval "role" "cluster" -}}
+    {{- $_ := set $config "opentelemetry" (dict "cluster_name" $ctx.Values.clusterName "collect" (dict "container_insights" $ci)) -}}
+  {{- end -}}
+{{- end -}}
 {{- if $needsLogs -}}
   {{- $_ := set $config "logs" (dict "metrics_collected" $metricsCollected) -}}
 {{- end -}}
@@ -180,15 +191,8 @@ Logic:
 {{- $agentName := .agentName -}}
 {{- $ctx := .context -}}
 {{- include "cloudwatch-agent.validate-flags" $ctx -}}
-{{- if not $ctx.Values.otelContainerInsights.enabled -}}
+{{/* OTEL CI now comes from the JSON config; the agent generates the pipeline, so no otelConfig here. */}}
 {}
-{{- else if eq $ctx.Values.otelContainerInsights.targetAgent $agentName -}}
-{{- include "otel-container-insights.config" $ctx -}}
-{{- else if eq $ctx.Values.otelContainerInsights.clusterScraperAgent $agentName -}}
-{{- include "otel-container-insights-cluster-scraper.config" $ctx -}}
-{{- else -}}
-{}
-{{- end -}}
 {{- end -}}
 
 {{/*

charts/amazon-cloudwatch-observability/tests/flag_matrix.sh

@@ -84,7 +84,7 @@ run_case() {
     if [[ -n "$must_have" ]]; then
         IFS=',' read -ra fragments <<< "$must_have"
         for f in "${fragments[@]}"; do
-            if ! grep -q "$f" <<< "$output"; then
+            if ! grep -qF -- "$f" <<< "$output"; then
                 echo -e "  ${R}FAIL${N}: missing required fragment: $f"
                 local_fail=1
             fi
@@ -94,7 +94,7 @@ run_case() {
     if [[ -n "$must_not" ]]; then
         IFS=',' read -ra fragments <<< "$must_not"
         for f in "${fragments[@]}"; do
-            if grep -q "$f" <<< "$output"; then
+            if grep -qF -- "$f" <<< "$output"; then
                 echo -e "  ${R}FAIL${N}: forbidden fragment present: $f"
                 local_fail=1
             fi
@@ -109,22 +109,23 @@ run_case() {
     fi
 }
 
-# Fragment shortcuts used across states.
-METRICS_EXPORTER="otlphttp/cw_k8s_ci_v0_metrics_dest"
-METRICS_SIGV4="sigv4auth/cw_k8s_ci_v0_metrics_dest"
-LOG_EXPORTER_APP="otlphttp/cw_k8s_ci_v0_app_logs_dest"
-LOG_EXPORTER_NODE="otlphttp/cw_k8s_ci_v0_node_logs_dest"
-LOG_SIGV4="sigv4auth/cw_k8s_ci_v0_logs_dest"
-LOG_PIPELINE_APP="logs/cw_k8s_ci_v0_app"
-FILELOG_APP="filelog/cw_k8s_ci_v0_app"
+# OTEL CI is delivered via the agent's JSON config
+# (opentelemetry.collect.container_insights); the agent generates the OTEL
+# pipeline at runtime, so we assert on the escaped JSON in the CR `config:`
+# field rather than on rendered collector fragments.
+OTEL_CI='collect\":{\"container_insights'
+OTEL_CI_NODE='\"role\":\"node\"'
+OTEL_CI_CLUSTER='\"role\":\"cluster\"'
+OTEL_CI_LOGS_ON='\"logs\":{\"enabled\":true}'
+OTEL_CI_LOGS_OFF='\"logs\":{\"enabled\":false}'
 # aws-for-fluent-bit is the container image string — unique to the FluentBit
 # DaemonSet. Using this instead of bare "fluent-bit" avoids false matches in
 # OTEL config paths like /var/log/containers/fluent-bit* (which exist in the
 # filelog exclude list regardless of the FB DaemonSet flag).
 FLUENT_BIT_IMAGE="aws-for-fluent-bit"
 
-# All OTEL log pipeline fragments (app + host).
-ALL_LOG_FRAGMENTS="$LOG_EXPORTER_APP,$LOG_EXPORTER_NODE,$LOG_SIGV4,$LOG_PIPELINE_APP,$FILELOG_APP"
+# Both node (daemonset) and cluster (scraper) CI config present.
+OTEL_CI_BOTH_ROLES="$OTEL_CI,$OTEL_CI_NODE,$OTEL_CI_CLUSTER"
 
 # ──────────────────────────────────────────────────────────────────────────
 # Run all 8 combinations.
@@ -136,43 +137,43 @@ echo "Chart: $CHART_DIR"
 # State #1: all false — no monitoring.
 run_case 1 false false false "ok" \
     "No monitoring — all flags off" \
-    "" "$METRICS_EXPORTER,$FLUENT_BIT_IMAGE"
+    "" "$OTEL_CI,$FLUENT_BIT_IMAGE"
 
 # State #2: FluentBit only (pure v1 legacy).
 run_case 2 false false true "ok" \
     "FluentBit legacy only" \
-    "$FLUENT_BIT_IMAGE" "$METRICS_EXPORTER,$ALL_LOG_FRAGMENTS"
+    "$FLUENT_BIT_IMAGE" "$OTEL_CI"
 
 # State #3: logs=true without enabled — silently ignored.
 run_case 3 false true false "ok" \
     "logs=true without enabled — no OTEL output" \
-    "" "$METRICS_EXPORTER,$ALL_LOG_FRAGMENTS,$FLUENT_BIT_IMAGE"
+    "" "$OTEL_CI,$FLUENT_BIT_IMAGE"
 
 # State #4: same as #3 with FluentBit.
 run_case 4 false true true "ok" \
     "logs=true without enabled + FluentBit — only FluentBit" \
-    "$FLUENT_BIT_IMAGE" "$METRICS_EXPORTER,$ALL_LOG_FRAGMENTS"
+    "$FLUENT_BIT_IMAGE" "$OTEL_CI"
 
-# State #5: OTEL metrics only.
+# State #5: OTEL metrics only (logs disabled).
 run_case 5 true false false "ok" \
     "OTEL metrics only, no logs" \
-    "$METRICS_EXPORTER,$METRICS_SIGV4" "$ALL_LOG_FRAGMENTS,$FLUENT_BIT_IMAGE"
+    "$OTEL_CI_BOTH_ROLES,$OTEL_CI_LOGS_OFF" "$OTEL_CI_LOGS_ON,$FLUENT_BIT_IMAGE"
 
 # State #6: hybrid — OTEL metrics + FluentBit logs.
 run_case 6 true false true "ok" \
     "Hybrid — OTEL metrics + FluentBit logs" \
-    "$METRICS_EXPORTER,$METRICS_SIGV4,$FLUENT_BIT_IMAGE" "$ALL_LOG_FRAGMENTS"
+    "$OTEL_CI_BOTH_ROLES,$OTEL_CI_LOGS_OFF,$FLUENT_BIT_IMAGE" "$OTEL_CI_LOGS_ON"
 
 # State #7: full OTEL (metrics + logs, no FluentBit).
 run_case 7 true true false "ok" \
     "Full OTEL (metrics + logs)" \
-    "$METRICS_EXPORTER,$METRICS_SIGV4,$LOG_EXPORTER_APP,$LOG_EXPORTER_NODE,$LOG_SIGV4,$FILELOG_APP" \
+    "$OTEL_CI_BOTH_ROLES,$OTEL_CI_LOGS_ON" \
     "$FLUENT_BIT_IMAGE"
 
 # State #8: dual-publish (migration window — OTEL logs + FluentBit both active).
 run_case 8 true true true "ok" \
     "Dual-publish — OTEL logs + FluentBit both active" \
-    "$METRICS_EXPORTER,$LOG_EXPORTER_APP,$LOG_EXPORTER_NODE,$FILELOG_APP,$FLUENT_BIT_IMAGE" \
+    "$OTEL_CI_BOTH_ROLES,$OTEL_CI_LOGS_ON,$FLUENT_BIT_IMAGE" \
     ""
 
 # ──────────────────────────────────────────────────────────────────────────

charts/amazon-cloudwatch-observability/values.yaml

@@ -726,7 +726,7 @@ agent:
   replicas: 1 # The total number non-terminated pods targeted by this AmazonCloudWatchAgent's deployment or statefulSet.
   image:
     repository: cloudwatch-agent
-    tag: 1.300070.0b1586
+    tag: 1.300071.0b1720
     repositoryDomainMap:
       public: public.ecr.aws/cloudwatch-agent
       cn-north-1: 934860584483.dkr.ecr.cn-north-1.amazonaws.com.cn
@@ -1161,8 +1161,10 @@ otelContainerInsights:
   enabled: false
   ## OTEL-native CI log pipelines (application, host).
   ## Only takes effect when otelContainerInsights.enabled is true.
+  ## OTEL CI logs turned off for now due to agent side fix
+  ## logs are handled by FluentBit (containerLogs).
   logs:
-    enabled: true
+    enabled: false
   ## The agent in the agents array that receives node-level OTEL Container Insights config.
   targetAgent: "cloudwatch-agent"
   ## The agent in the agents array that receives cluster-level OTEL Container Insights config (apiserver, kube-state-metrics scraping).

integration-tests/amazon-cloudwatch-observability/validations/minikube/common.go

@@ -112,3 +112,34 @@ func GetOperatorAutoInstrumentationConfig(t *testing.T) map[string]interface{} {
 	t.Logf("auto-instrumentation-config: %s", autoInstrumentationArg)
 	return config
 }
+
+// OTEL CI assertions — CI is delivered via spec.config JSON (operator generates
+// the pipeline), so tests assert on spec.config, not spec.otelConfig.
+
+// otelCIMarker matches the V2 container_insights block.
+const otelCIMarker = `"container_insights":{`
+
+// AssertOtelContainerInsights asserts OTEL CI is enabled with the given role (node|cluster).
+func AssertOtelContainerInsights(t *testing.T, config, role string) {
+	assert.Contains(t, config, otelCIMarker,
+		"config must enable opentelemetry.collect.container_insights")
+	assert.Contains(t, config, `"role":"`+role+`"`,
+		"container_insights role must be %q", role)
+}
+
+// AssertOtelCILogsEnabled asserts the container_insights logs toggle.
+func AssertOtelCILogsEnabled(t *testing.T, config string, enabled bool) {
+	if enabled {
+		assert.Contains(t, config, `"logs":{"enabled":true}`,
+			"container_insights logs must be enabled")
+	} else {
+		assert.Contains(t, config, `"logs":{"enabled":false}`,
+			"container_insights logs must be disabled")
+	}
+}
+
+// AssertNoOtelContainerInsights asserts OTEL CI is not configured.
+func AssertNoOtelContainerInsights(t *testing.T, config string) {
+	assert.NotContains(t, config, otelCIMarker,
+		"config must not enable opentelemetry.collect.container_insights")
+}

integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/feature_targeted_default_test.go

@@ -95,7 +95,7 @@ func validateClusterScraperCRExists(t *testing.T, agentMap map[string]unstructur
 // - cloudwatch-agent gets node-level pipelines (cadvisor, kubeletstats, node-exporter receivers)
 // - cloudwatch-agent-cluster-scraper gets cluster-level pipelines (apiserver, kube-state-metrics receivers)
 func validateOTELConfigRouting(t *testing.T, agentMap map[string]unstructured.Unstructured) {
-	// Validate cloudwatch-agent has node-level OTEL config
+	// Validate cloudwatch-agent has node-level OTEL CI config
 	t.Run("CloudWatchAgentNodeLevel", func(t *testing.T) {
 		agent, exists := agentMap["cloudwatch-agent"]
 		if !assert.True(t, exists, "cloudwatch-agent CR should exist") {
@@ -107,24 +107,18 @@ func validateOTELConfigRouting(t *testing.T, agentMap map[string]unstructured.Un
 			return
 		}
 
-		otelConfig, ok := spec["otelConfig"].(string)
-		if !assert.True(t, ok, "otelConfig should be a string") {
+		config, ok := spec["config"].(string)
+		if !assert.True(t, ok, "config should be a string") {
 			return
 		}
-		assert.NotEmpty(t, otelConfig, "otelConfig should not be empty")
 
-		// Node-level config should contain node-exporter, cadvisor, kubeletstats receivers
-		assert.True(t, strings.Contains(otelConfig, "kubeletstats"),
-			"cloudwatch-agent otelConfig should contain kubeletstats receiver (node-level)")
-
-		// Node-level config should NOT contain cluster-level receivers
-		assert.False(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_apiserver"),
-			"cloudwatch-agent otelConfig should NOT contain apiserver receiver (cluster-level)")
-		assert.False(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_kube_state_metrics"),
-			"cloudwatch-agent otelConfig should NOT contain kube_state_metrics receiver (cluster-level)")
+		// role=node; operator generates the node pipeline.
+		minikube.AssertOtelContainerInsights(t, config, "node")
+		assert.False(t, strings.Contains(config, `"role":"cluster"`),
+			"cloudwatch-agent config should not request cluster role")
 	})
 
-	// Validate cluster-scraper has cluster-level OTEL config
+	// Validate cluster-scraper has cluster-level OTEL CI config
 	t.Run("ClusterScraperClusterLevel", func(t *testing.T) {
 		agent, exists := agentMap["cloudwatch-agent-cluster-scraper"]
 		if !assert.True(t, exists, "cloudwatch-agent-cluster-scraper CR should exist") {
@@ -136,20 +130,14 @@ func validateOTELConfigRouting(t *testing.T, agentMap map[string]unstructured.Un
 			return
 		}
 
-		otelConfig, ok := spec["otelConfig"].(string)
-		if !assert.True(t, ok, "otelConfig should be a string") {
+		config, ok := spec["config"].(string)
+		if !assert.True(t, ok, "config should be a string") {
 			return
 		}
-		assert.NotEmpty(t, otelConfig, "otelConfig should not be empty")
-
-		// Cluster-level config should contain apiserver and kube-state-metrics receivers
-		assert.True(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_apiserver"),
-			"cluster-scraper otelConfig should contain apiserver receiver (cluster-level)")
-		assert.True(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_kube_state_metrics"),
-			"cluster-scraper otelConfig should contain kube_state_metrics receiver (cluster-level)")
 
-		// Cluster-level config should NOT contain node-level receivers
-		assert.False(t, strings.Contains(otelConfig, "kubeletstats"),
-			"cluster-scraper otelConfig should NOT contain kubeletstats receiver (node-level)")
+		// role=cluster; operator generates the cluster pipeline.
+		minikube.AssertOtelContainerInsights(t, config, "cluster")
+		assert.False(t, strings.Contains(config, `"role":"node"`),
+			"cluster-scraper config should not request node role")
 	})
 }

integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/feature_targeted_multi_agent_test.go

@@ -124,16 +124,15 @@ func validateCloudWatchAgentFullConfig(t *testing.T, agentMap map[string]unstruc
 		}
 	}
 
-	// Validate OTEL config has node-level pipelines
-	otelConfig, ok := spec["otelConfig"].(string)
-	if !assert.True(t, ok, "otelConfig should be a string") {
+	// OTEL CI, node role.
+	cfgStr, ok := spec["config"].(string)
+	if !assert.True(t, ok, "config should be a string") {
 		return
 	}
 
-	assert.True(t, strings.Contains(otelConfig, "kubeletstats"),
-		"cloudwatch-agent otelConfig should contain kubeletstats receiver (node-level)")
-	assert.False(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_apiserver"),
-		"cloudwatch-agent otelConfig should NOT contain apiserver receiver (cluster-level)")
+	minikube.AssertOtelContainerInsights(t, cfgStr, "node")
+	assert.False(t, strings.Contains(cfgStr, `"role":"cluster"`),
+		"cloudwatch-agent config should not request cluster role")
 }
 
 // validatePrometheusAgentMinimalConfig verifies prometheus-agent gets minimal config:
@@ -180,17 +179,8 @@ func validatePrometheusAgentMinimalConfig(t *testing.T, agentMap map[string]unst
 	_, hasTraces := config["traces"]
 	assert.False(t, hasTraces, "prometheus-agent config should NOT have traces section")
 
-	// Validate OTEL config is absent or empty (not targeted by any OTEL CI feature)
-	otelConfig, ok := spec["otelConfig"].(string)
-	if ok {
-		assert.False(t, strings.Contains(otelConfig, "kubeletstats"),
-			"prometheus-agent otelConfig should NOT contain kubeletstats receiver")
-		assert.False(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_apiserver"),
-			"prometheus-agent otelConfig should NOT contain apiserver receiver")
-		assert.False(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_kube_state_metrics"),
-			"prometheus-agent otelConfig should NOT contain kube_state_metrics receiver")
-	}
-	// otelConfig may be absent entirely when no OTEL CI features target this agent — that's valid
+	// No OTEL CI for this agent.
+	minikube.AssertNoOtelContainerInsights(t, configStr)
 }
 
 // validateClusterScraperConfig verifies cluster-scraper gets cluster-level OTEL config
@@ -211,16 +201,13 @@ func validateClusterScraperConfig(t *testing.T, agentMap map[string]unstructured
 	assert.True(t, ok, "hostNetwork should be a bool")
 	assert.True(t, hostNetwork, "cluster-scraper should have hostNetwork=true")
 
-	// Validate OTEL config has cluster-level pipelines
-	otelConfig, ok := spec["otelConfig"].(string)
-	if !assert.True(t, ok, "otelConfig should be a string") {
+	// OTEL CI, cluster role.
+	cfgStr, ok := spec["config"].(string)
+	if !assert.True(t, ok, "config should be a string") {
 		return
 	}
 
-	assert.True(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_apiserver"),
-		"cluster-scraper otelConfig should contain apiserver receiver (cluster-level)")
-	assert.True(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_kube_state_metrics"),
-		"cluster-scraper otelConfig should contain kube_state_metrics receiver (cluster-level)")
-	assert.False(t, strings.Contains(otelConfig, "kubeletstats"),
-		"cluster-scraper otelConfig should NOT contain kubeletstats receiver (node-level)")
+	minikube.AssertOtelContainerInsights(t, cfgStr, "cluster")
+	assert.False(t, strings.Contains(cfgStr, `"role":"node"`),
+		"cluster-scraper config should not request node role")
 }

integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/feature_targeted_otlp_disabled_test.go

@@ -6,7 +6,6 @@ package scenarios
 import (
 	"context"
 	"encoding/json"
-	"strings"
 	"testing"
 
 	"github.com/aws-observability/helm-charts/integration-tests/amazon-cloudwatch-observability/util"
@@ -96,19 +95,12 @@ func validateCloudWatchAgentNoOtelConfig(t *testing.T, agentMap map[string]unstr
 		return
 	}
 
-	otelConfig, ok := spec["otelConfig"].(string)
-	if ok {
-		// If otelConfig is present, it should NOT contain any CI receivers
-		assert.False(t, strings.Contains(otelConfig, "kubeletstats"),
-			"cloudwatch-agent otelConfig should NOT contain kubeletstats receiver when OTLP disabled")
-		assert.False(t, strings.Contains(otelConfig, "cadvisor"),
-			"cloudwatch-agent otelConfig should NOT contain cadvisor receiver when OTLP disabled")
-		assert.False(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_apiserver"),
-			"cloudwatch-agent otelConfig should NOT contain apiserver receiver when OTLP disabled")
-		assert.False(t, strings.Contains(otelConfig, "cw_k8s_ci_v0_kube_state_metrics"),
-			"cloudwatch-agent otelConfig should NOT contain kube_state_metrics receiver when OTLP disabled")
+	config, ok := spec["config"].(string)
+	if !assert.True(t, ok, "config should be a string") {
+		return
 	}
-	// otelConfig may be absent entirely when no OTEL CI features target this agent — that's valid
+	// OTEL CI disabled → no container_insights in config.
+	minikube.AssertNoOtelContainerInsights(t, config)
 }
 
 // validateNodeExporterNotDeployed verifies node-exporter resources are NOT present