Skip to content

Commit e490516

Browse files
committed
Support default:otel config
1 parent b9b8325 commit e490516

6 files changed

Lines changed: 215 additions & 3 deletions

File tree

.github/workflows/amazon-cloudwatch-observability-integration-test.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ jobs:
5353
- certificate-recreate-enabled
5454
- configmap-permission-scoping
5555
- default
56+
- default-otel
5657
- deployment-rolling-disabled
5758
- deployment-rolling-enabled
5859
- dualstack-endpoint-enabled

charts/amazon-cloudwatch-observability/templates/_helpers.tpl

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ Helper function to modify auto-monitor config based on agent configurations
8181
{{- range .Values.agents -}}
8282
{{- $agent := mergeOverwrite (deepCopy $.Values.agent) . -}}
8383
{{- if and $.Values.applicationSignals.enabled (eq $.Values.applicationSignals.targetAgent $agent.name) -}}
84-
{{- if and $agent.config (ne ($agent.config | toString) "default") -}}
84+
{{- if and $agent.config (not (has ($agent.config | toString) (list "default" "default:otel"))) -}}
8585
{{- $agentConfig := $agent.config -}}
8686
{{- if or (and (hasKey $agentConfig "logs") (hasKey $agentConfig.logs "metrics_collected") (hasKey $agentConfig.logs.metrics_collected "application_signals")) (and (hasKey $agentConfig "traces") (hasKey $agentConfig.traces "traces_collected") (hasKey $agentConfig.traces.traces_collected "application_signals")) -}}
8787
{{- $hasAppSignals = true -}}
@@ -136,6 +136,26 @@ Logic:
136136
{{- $config | toJson -}}
137137
{{- end -}}
138138

139+
{{/*
140+
Build the "default:otel" CW Agent JSON config: the default config plus an OTLP receiver.
141+
Accepts a dict with "agentName" (string) and "context" (root context $).
142+
Returns a dict serialized to JSON.
143+
144+
Adds otlp to whatever collect block build-default-config produced, rather than replacing it, so an
145+
otelContainerInsights container_insights section survives alongside it.
146+
147+
Endpoints are set explicitly rather than left to the agent default so the receiver accepts traffic
148+
from other pods.
149+
*/}}
150+
{{- define "cloudwatch-agent.build-config-default-otel" -}}
151+
{{- $config := include "cloudwatch-agent.build-default-config" . | fromJson -}}
152+
{{- $otlp := dict "opentelemetry" (dict "collect" (dict "otlp" (dict
153+
"span_metrics_enabled" true
154+
"grpc_endpoint" "0.0.0.0:4317"
155+
"http_endpoint" "0.0.0.0:4318"))) -}}
156+
{{- mergeOverwrite $config $otlp | toJson -}}
157+
{{- end -}}
158+
139159
{{/*
140160
Build the default OTEL YAML config for a given agent based on which feature flags target it.
141161
Accepts a dict with "agentName" (string) and "context" (root context $).
@@ -450,7 +470,7 @@ Set DCGM_EXPORTER_INTERVAL environment variable for dcgmExporter if accelerated_
450470
{{- range .Values.agents -}}
451471
{{- $agent := mergeOverwrite (deepCopy $.Values.agent) . -}}
452472
{{- $agentConfig := $agent.config -}}
453-
{{- if or (not $agentConfig) (eq ($agentConfig | toString) "default") -}}
473+
{{- if or (not $agentConfig) (has ($agentConfig | toString) (list "default" "default:otel")) -}}
454474
{{- $agentConfig = dict -}}
455475
{{- end -}}
456476
{{- if and (hasKey $agentConfig "logs") (hasKey $agentConfig.logs "metrics_collected") (hasKey $agentConfig.logs.metrics_collected "kubernetes") (hasKey $agentConfig.logs.metrics_collected.kubernetes "accelerated_compute_gpu_metrics_collection_interval") -}}

charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-custom-resource.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,9 @@ spec:
101101
affinity: {{- toYaml . | nindent 4 }}
102102
{{- end }}
103103
hostNetwork: {{ $agent.hostNetwork }}
104-
{{- if and $agent.config (ne ($agent.config | toString) "default") }}
104+
{{- if eq ($agent.config | toString) "default:otel" }}
105+
config: {{ include "cloudwatch-agent.modify-config" (merge (dict "Config" (include "cloudwatch-agent.build-config-default-otel" (dict "agentName" $agent.name "context" $) | fromJson)) $ ) }}
106+
{{- else if and $agent.config (ne ($agent.config | toString) "default") }}
105107
config: {{ include "cloudwatch-agent.modify-config" (merge (dict "Config" $agent.config) $ ) }}
106108
{{- else }}
107109
config: {{ include "cloudwatch-agent.modify-config" (merge (dict "Config" (include "cloudwatch-agent.build-default-config" (dict "agentName" $agent.name "context" $) | fromJson)) $ ) }}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
module "base" {
5+
source = "../.."
6+
helm_values_file = "${path.module}/values.yaml"
7+
helm_dir = var.helm_dir
8+
}
9+
10+
resource "null_resource" "validator" {
11+
depends_on = [module.base.helm_release]
12+
13+
provisioner "local-exec" {
14+
command = "go test ${var.test_dir} -v -run=TestDefaultOtel"
15+
}
16+
}
17+
18+
variable "test_dir" {
19+
type = string
20+
default = "../../../../validations/minikube/scenarios"
21+
}
22+
23+
variable "helm_dir" {
24+
type = string
25+
default = "../../../../../../charts/amazon-cloudwatch-observability"
26+
}
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
region: us-west-2
2+
clusterName: minikube
3+
# Exercise config: "default:otel": the node agent gets the default CW Agent JSON config plus an
4+
# OTLP receiver (0.0.0.0:4317/4318), while the cluster-scraper keeps "default".
5+
# The agents array is respecified in full because --set/list-merge would otherwise clobber it.
6+
agents:
7+
- name: cloudwatch-agent
8+
config: "default:otel"
9+
- name: cloudwatch-agent-cluster-scraper
10+
mode: deployment
11+
config: "default"
12+
# OTEL CI on so the cluster-scraper CR is rendered (it is gated on otelContainerInsights.enabled),
13+
# letting the test assert the scraper does NOT pick up the OTLP receiver.
14+
otelContainerInsights:
15+
enabled: true
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package scenarios
5+
6+
import (
7+
"context"
8+
"encoding/json"
9+
"testing"
10+
11+
"github.com/aws-observability/helm-charts/integration-tests/amazon-cloudwatch-observability/util"
12+
"github.com/aws-observability/helm-charts/integration-tests/amazon-cloudwatch-observability/validations/minikube"
13+
"github.com/stretchr/testify/assert"
14+
"github.com/stretchr/testify/require"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
17+
"k8s.io/apimachinery/pkg/runtime/schema"
18+
)
19+
20+
// TestDefaultOtel validates the config: "default:otel" value, which merges an OTLP receiver onto
21+
// the default CW Agent config. Asserts on the rendered CR's JSON spec.config, like the other scenarios.
22+
func TestDefaultOtel(t *testing.T) {
23+
k8sClient, err := util.NewK8sClient()
24+
require.NoError(t, err, "failed to create k8s client")
25+
26+
ns, err := k8sClient.GetNamespace(minikube.Namespace)
27+
assert.NoError(t, err)
28+
assert.Equal(t, minikube.Namespace, ns.Name)
29+
30+
exists, err := k8sClient.ValidateDeploymentExists(minikube.Namespace, "amazon-cloudwatch-observability-controller-manager")
31+
assert.NoError(t, err)
32+
assert.True(t, exists, "operator deployment should exist")
33+
34+
dynamicClient, err := k8sClient.GetDynamicClient()
35+
require.NoError(t, err, "failed to get dynamic client")
36+
37+
gvr := schema.GroupVersionResource{
38+
Group: "cloudwatch.aws.amazon.com",
39+
Version: "v1alpha1",
40+
Resource: "amazoncloudwatchagents",
41+
}
42+
agentList, err := dynamicClient.Resource(gvr).Namespace(minikube.Namespace).List(
43+
context.Background(), metav1.ListOptions{},
44+
)
45+
require.NoError(t, err, "failed to list AmazonCloudWatchAgent CRs")
46+
47+
agentMap := make(map[string]unstructured.Unstructured)
48+
for _, agent := range agentList.Items {
49+
agentMap[agent.GetName()] = agent
50+
}
51+
52+
t.Run("NodeAgentHasOtlpReceiver", func(t *testing.T) {
53+
validateNodeAgentOtlpReceiver(t, agentMap)
54+
})
55+
t.Run("NodeAgentDefaultConfigSurvives", func(t *testing.T) {
56+
validateNodeAgentDefaultConfigSurvives(t, agentMap)
57+
})
58+
t.Run("ClusterScraperHasNoOtlpReceiver", func(t *testing.T) {
59+
validateClusterScraperNoOtlpReceiver(t, agentMap)
60+
})
61+
62+
t.Log("default:otel config scenario validation passed")
63+
}
64+
65+
// configJSONOf parses a CR's spec.config (a JSON string) into a map, failing the test on any issue.
66+
func configJSONOf(t *testing.T, agentMap map[string]unstructured.Unstructured, name string) map[string]interface{} {
67+
t.Helper()
68+
agent, exists := agentMap[name]
69+
if !assert.True(t, exists, "%s CR should exist", name) {
70+
return nil
71+
}
72+
spec, ok := agent.Object["spec"].(map[string]interface{})
73+
if !assert.True(t, ok, "%s spec should be a map", name) {
74+
return nil
75+
}
76+
configStr, ok := spec["config"].(string)
77+
if !assert.True(t, ok, "%s config should be a string", name) {
78+
return nil
79+
}
80+
var config map[string]interface{}
81+
if !assert.NoError(t, json.Unmarshal([]byte(configStr), &config), "%s config should be valid JSON", name) {
82+
return nil
83+
}
84+
return config
85+
}
86+
87+
// otlpCollectBlockOf returns opentelemetry.collect.otlp from a parsed config, or nil if absent.
88+
func otlpCollectBlockOf(config map[string]interface{}) map[string]interface{} {
89+
otel, ok := config["opentelemetry"].(map[string]interface{})
90+
if !ok {
91+
return nil
92+
}
93+
collect, ok := otel["collect"].(map[string]interface{})
94+
if !ok {
95+
return nil
96+
}
97+
otlp, _ := collect["otlp"].(map[string]interface{})
98+
return otlp
99+
}
100+
101+
// validateNodeAgentOtlpReceiver checks the node agent's "default:otel" config carries the OTLP receiver.
102+
func validateNodeAgentOtlpReceiver(t *testing.T, agentMap map[string]unstructured.Unstructured) {
103+
config := configJSONOf(t, agentMap, "cloudwatch-agent")
104+
if config == nil {
105+
return
106+
}
107+
otlp := otlpCollectBlockOf(config)
108+
if !assert.NotNil(t, otlp, "node agent config should have opentelemetry.collect.otlp for default:otel") {
109+
return
110+
}
111+
// Endpoints are pinned to 0.0.0.0 so the receiver accepts traffic from other pods (the agent
112+
// default binds loopback only).
113+
assert.Equal(t, "0.0.0.0:4317", otlp["grpc_endpoint"], "otlp grpc_endpoint should be 0.0.0.0:4317")
114+
assert.Equal(t, "0.0.0.0:4318", otlp["http_endpoint"], "otlp http_endpoint should be 0.0.0.0:4318")
115+
assert.Equal(t, true, otlp["span_metrics_enabled"], "otlp span_metrics_enabled should be true")
116+
}
117+
118+
// validateNodeAgentDefaultConfigSurvives checks the OTLP receiver was MERGED onto the default config,
119+
// not substituted for it: the default Container Insights block (logs.metrics_collected.kubernetes)
120+
// must remain alongside it.
121+
func validateNodeAgentDefaultConfigSurvives(t *testing.T, agentMap map[string]unstructured.Unstructured) {
122+
config := configJSONOf(t, agentMap, "cloudwatch-agent")
123+
if config == nil {
124+
return
125+
}
126+
logs, ok := config["logs"].(map[string]interface{})
127+
if !assert.True(t, ok, "node agent config should retain the default logs section") {
128+
return
129+
}
130+
metricsCollected, ok := logs["metrics_collected"].(map[string]interface{})
131+
if !assert.True(t, ok, "node agent config should retain logs.metrics_collected") {
132+
return
133+
}
134+
_, hasKubernetes := metricsCollected["kubernetes"]
135+
assert.True(t, hasKubernetes,
136+
"default Container Insights (logs.metrics_collected.kubernetes) should survive the OTLP merge")
137+
}
138+
139+
// validateClusterScraperNoOtlpReceiver checks the cluster-scraper (config: "default", not
140+
// "default:otel") does NOT get the OTLP receiver; it describes other workloads, not local ingest.
141+
func validateClusterScraperNoOtlpReceiver(t *testing.T, agentMap map[string]unstructured.Unstructured) {
142+
config := configJSONOf(t, agentMap, "cloudwatch-agent-cluster-scraper")
143+
if config == nil {
144+
return
145+
}
146+
assert.Nil(t, otlpCollectBlockOf(config),
147+
"cluster-scraper config should NOT have an OTLP receiver (config is plain default)")
148+
}

0 commit comments

Comments
 (0)