fix(collector): roll pods when Prometheus config changes

The pod-template restart-trigger sha256 was computed from Spec.Config only,
so a change to Spec.Prometheus (rendered into a separate ConfigMap) left the
pod template byte-identical and the workload controller did not roll the pods.

Fold the serialized Spec.Prometheus (PrometheusConfig.Yaml()) into the hash
input when it is non-empty, so a Prometheus-only change bumps the pod-template
annotation and triggers a rolling restart, matching agent-config behavior.
When no Prometheus config is set the hash input is byte-identical to the agent
config alone, leaving non-Prometheus agents unaffected.

Author: musa-asad

RELEASE_NOTES

@@ -1,9 +1,3 @@
-========================================================================
-Amazon CloudWatch Agent Operator (Unreleased)
-========================================================================
-Bug Fixes:
-* [TargetAllocator] Register the `--enable-prometheus-cr-watcher` CLI flag so the target-allocator no longer exits with "unknown flag" (CrashLoopBackOff) when the operator passes it for PrometheusCR.enabled. The flag is OR'd with the YAML `prometheus_cr.enabled` setting.
-
 ========================================================================
 Amazon CloudWatch Agent Operator v3.5.0 (2026-06-05)
 ========================================================================

cmd/amazon-cloudwatch-agent-target-allocator/config/config.go

@@ -119,9 +119,7 @@ func LoadFromCLI(target *Config, flagSet *pflag.FlagSet) error {
 		return err
 	}
 
-	// Enable the Prometheus CR watcher when requested via the CLI flag. The YAML
-	// `prometheus_cr.enabled` value is loaded before this point, so OR the flag in
-	// rather than overwriting it: the watcher is enabled if either source sets it.
+	// OR the CLI flag into the YAML value so either source can enable the watcher.
 	prometheusCREnabled, err := getPrometheusCREnabled(flagSet)
 	if err != nil {
 		return err

cmd/amazon-cloudwatch-agent-target-allocator/main.go

@@ -88,11 +88,7 @@ func main() {
 	srv := server.NewServer(log, allocator, cfg.ListenAddr, httpOptions...)
 
 	discoveryCtx, discoveryCancel := context.WithCancel(ctx)
-	// Service Discovery metrics MUST be created and passed to the discovery
-	// manager. Each SD provider's NewDiscoverer fails when its DiscovererMetrics
-	// is nil, so passing a nil sdMetrics map makes every provider (including
-	// kubernetes_sd) fail to register, yielding zero discovered targets. Mirror
-	// the upstream opentelemetry-operator target-allocator.
+	// SD metrics must be non-nil; providers fail to register without them.
 	sdRegistry := prometheus.NewRegistry()
 	sdMetrics, sdMetricsErr := discovery.CreateAndRegisterSDMetrics(sdRegistry)
 	if sdMetricsErr != nil {

cmd/amazon-cloudwatch-agent-target-allocator/watcher/promOperator.go

@@ -50,12 +50,7 @@ func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*Pr
 	}
 
 	// TODO: We should make these durations configurable
-	// The synthetic Prometheus object must carry a non-empty namespace: the
-	// prometheus-operator config generator calls store.ForNamespace(prom.Namespace)
-	// when generating the server configuration, which panics on an empty namespace
-	// ("namespace can't be empty"). Mirror the upstream opentelemetry-operator
-	// target-allocator by setting it to the collector/TA namespace, derived from the
-	// operator-injected OTELCOL_NAMESPACE downward-API env (fallback: amazon-cloudwatch).
+	// Namespace must be non-empty; the config generator panics otherwise.
 	collectorNamespace := os.Getenv("OTELCOL_NAMESPACE")
 	if collectorNamespace == "" {
 		collectorNamespace = "amazon-cloudwatch"
@@ -68,11 +63,7 @@ func NewPrometheusCRWatcher(logger logr.Logger, cfg allocatorconfig.Config) (*Pr
 			CommonPrometheusFields: monitoringv1.CommonPrometheusFields{
 				ScrapeInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()),
 			},
-			// EvaluationInterval must be non-empty: the prometheus-operator config
-			// generator renders it verbatim as global.evaluation_interval, and the
-			// downstream prometheus config parser rejects an empty value with
-			// "empty duration string". The CWA TA does not evaluate rules, so mirror
-			// the scrape interval as a sane non-empty default.
+			// Must be non-empty; default to scrape interval.
 			EvaluationInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()),
 		},
 	}

internal/manifests/collector/annotations.go

@@ -28,7 +28,7 @@ func Annotations(instance v1alpha1.AmazonCloudWatchAgent) map[string]string {
 	}
 
 	// make sure sha256 for configMap is always calculated
-	annotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(instance.Spec.Config)
+	annotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(configHashInput(instance))
 
 	return annotations
 }
@@ -51,11 +51,22 @@ func PodAnnotations(instance v1alpha1.AmazonCloudWatchAgent) map[string]string {
 	}
 
 	// make sure sha256 for configMap is always calculated
-	podAnnotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(instance.Spec.Config)
+	podAnnotations["amazon-cloudwatch-agent-operator-config/sha256"] = getConfigMapSHA(configHashInput(instance))
 
 	return podAnnotations
 }
 
+// configHashInput returns the combined config string used for the pod-template restart hash.
+func configHashInput(instance v1alpha1.AmazonCloudWatchAgent) string {
+	config := instance.Spec.Config
+	if !instance.Spec.Prometheus.IsEmpty() {
+		if promYaml, err := instance.Spec.Prometheus.Yaml(); err == nil {
+			config += promYaml
+		}
+	}
+	return config
+}
+
 func getConfigMapSHA(config string) string {
 	h := sha256.Sum256([]byte(config))
 	return fmt.Sprintf("%x", h)

internal/manifests/collector/annotations_test.go

@@ -79,3 +79,68 @@ func TestAnnotationsPropagateDown(t *testing.T) {
 	assert.Equal(t, "mycomponent", podAnnotations["myapp"])
 	assert.Equal(t, "pod_annotation_value", podAnnotations["pod_annotation"])
 }
+
+func promConfig(t *testing.T, replacement string) v1alpha1.PrometheusConfig {
+	t.Helper()
+	cfg := map[string]interface{}{
+		"scrape_configs": []interface{}{
+			map[string]interface{}{
+				"job_name": "kubernetes-pods-annotated",
+				"relabel_configs": []interface{}{
+					map[string]interface{}{
+						"target_label": "bug2probe",
+						"replacement":  replacement,
+					},
+				},
+			},
+		},
+	}
+	return v1alpha1.PrometheusConfig{
+		Config: &v1alpha1.AnyConfig{Object: cfg},
+	}
+}
+
+// TestPrometheusConfigChangeBumpsHash asserts that changing only Spec.Prometheus
+// changes the pod-template config hash (so the pods roll on a Prometheus-only
+// change), while an unchanged spec yields a stable hash.
+func TestPrometheusConfigChangeBumpsHash(t *testing.T) {
+	base := v1alpha1.AmazonCloudWatchAgent{
+		ObjectMeta: metav1.ObjectMeta{Name: "my-instance", Namespace: "my-ns"},
+		Spec: v1alpha1.AmazonCloudWatchAgentSpec{
+			Config:     "agent-config",
+			Prometheus: promConfig(t, "value2"),
+		},
+	}
+
+	// same spec twice -> stable hash
+	h1 := PodAnnotations(base)["amazon-cloudwatch-agent-operator-config/sha256"]
+	h2 := PodAnnotations(base)["amazon-cloudwatch-agent-operator-config/sha256"]
+	assert.Equal(t, h1, h2, "hash must be stable when nothing changes")
+
+	// change ONLY the prometheus config -> hash must change
+	changed := base
+	changed.Spec.Prometheus = promConfig(t, "value3")
+	h3 := PodAnnotations(changed)["amazon-cloudwatch-agent-operator-config/sha256"]
+	assert.NotEqual(t, h1, h3, "pod annotation hash must change when only Spec.Prometheus changes")
+
+	// metadata annotations hash must also reflect the prometheus change
+	a1 := Annotations(base)["amazon-cloudwatch-agent-operator-config/sha256"]
+	a3 := Annotations(changed)["amazon-cloudwatch-agent-operator-config/sha256"]
+	assert.NotEqual(t, a1, a3, "metadata annotation hash must change when only Spec.Prometheus changes")
+}
+
+// TestEmptyPrometheusHashUnchanged asserts that when no Prometheus config is set,
+// the hash is byte-identical to the agent-config-only sha256 (non-Prometheus
+// agents are unaffected by the fix).
+func TestEmptyPrometheusHashUnchanged(t *testing.T) {
+	otelcol := v1alpha1.AmazonCloudWatchAgent{
+		ObjectMeta: metav1.ObjectMeta{Name: "my-instance", Namespace: "my-ns"},
+		Spec:       v1alpha1.AmazonCloudWatchAgentSpec{Config: "test"},
+	}
+
+	// sha256("test") == 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+	assert.Equal(t, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
+		Annotations(otelcol)["amazon-cloudwatch-agent-operator-config/sha256"])
+	assert.Equal(t, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
+		PodAnnotations(otelcol)["amazon-cloudwatch-agent-operator-config/sha256"])
+}