Skip to content

Commit 24c8523

Browse files
PaamickyPaamicky
committed
split logs and metrics rendering
1 parent 9d227fe commit 24c8523

11 files changed

Lines changed: 336 additions & 95 deletions

File tree

‎charts/amazon-cloudwatch-observability/templates/_helpers.tpl‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,8 @@ Logic:
134134
{{- if $ctx.Values.otelContainerInsights.enabled -}}
135135
{{- $interval := atoi (trimSuffix "s" $ctx.Values.otelContainerInsights.metricResolution) -}}
136136
{{- if eq $ctx.Values.otelContainerInsights.targetAgent $agentName -}}
137-
{{- $ci := dict "collection_interval" $interval "role" "node" "logs" (dict "enabled" $ctx.Values.otelContainerInsights.logs.enabled) -}}
137+
{{/* Hybrid: agent renders metrics only; CI logs come from the chart otelConfig. */}}
138+
{{- $ci := dict "collection_interval" $interval "role" "node" "logs" (dict "enabled" false) -}}
138139
{{- $_ := set $config "opentelemetry" (dict "cluster_name" $ctx.Values.clusterName "collect" (dict "container_insights" $ci)) -}}
139140
{{- else if eq $ctx.Values.otelContainerInsights.clusterScraperAgent $agentName -}}
140141
{{- $ci := dict "collection_interval" $interval "role" "cluster" -}}
@@ -191,9 +192,14 @@ Logic:
191192
{{- $agentName := .agentName -}}
192193
{{- $ctx := .context -}}
193194
{{- include "cloudwatch-agent.validate-flags" $ctx -}}
194-
{{/* OTEL CI now comes from the JSON config; the agent generates the pipeline, so no otelConfig here. */}}
195+
{{/* Hybrid: metrics come from the JSON config (agent-generated). The chart supplies
196+
ONLY the CI logs pipelines here, on the node agent, when logs are enabled. */}}
197+
{{- if and $ctx.Values.otelContainerInsights.enabled $ctx.Values.otelContainerInsights.logs.enabled (eq $ctx.Values.otelContainerInsights.targetAgent $agentName) -}}
198+
{{- include "otel-container-insights-logs.config" $ctx -}}
199+
{{- else -}}
195200
{}
196201
{{- end -}}
202+
{{- end -}}
197203

198204
{{/*
199205
Helper function to modify cloudwatch-agent config

‎charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-logs-config.tpl‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
{{/*
2+
OTEL Container Insights LOGS-only otelConfig (hybrid model).
3+
4+
Metrics are generated by the agent from spec.config (JSON); this template supplies
5+
ONLY the CI log pipelines via spec.otelConfig. The agent merges the two at startup.
6+
7+
The shared enrichment processors (resourcedetection, k8sattributes) are named with a
8+
"_logs" suffix so they never collide with the agent's metrics-side copies during that
9+
merge (the agent rejects same-name/different-body components in restricted sections).
10+
Their definitions are otherwise identical.
11+
*/}}
12+
{{- define "otel-container-insights-logs.config" -}}
13+
extensions:
14+
sigv4auth/cw_k8s_ci_v0_logs_dest:
15+
region: {{ .Values.region }}
16+
service: logs
17+
awscloudwatchlogsprovisioner/cw_k8s_ci_v0_logs:
18+
region: {{ .Values.region }}
19+
additional_auth: sigv4auth/cw_k8s_ci_v0_logs_dest
20+
21+
receivers:
22+
filelog/cw_k8s_ci_v0_app:
23+
include:
24+
- /var/log/containers/*.log
25+
exclude:
26+
- /var/log/containers/cloudwatch-agent*
27+
- /var/log/containers/fluent-bit*
28+
- /var/log/containers/aws-node*
29+
- /var/log/containers/kube-proxy*
30+
start_at: end
31+
include_file_path: true
32+
include_file_name: false
33+
max_concurrent_files: 100
34+
operators:
35+
- type: regex_parser
36+
id: extract_metadata_from_filepath
37+
regex: '^.*\/(?P<pod>[^_]+)_(?P<namespace>[^_]+)_(?P<container>.+)-[a-f0-9]{64}\.log$'
38+
parse_from: attributes["log.file.path"]
39+
parse_to: resource
40+
- type: move
41+
from: resource.pod
42+
to: resource["k8s.pod.name"]
43+
- type: move
44+
from: resource.namespace
45+
to: resource["k8s.namespace.name"]
46+
- type: move
47+
from: resource.container
48+
to: resource["k8s.container.name"]
49+
- id: parse_container_log
50+
type: container_log_parser
51+
52+
filelog/cw_k8s_ci_v0_node:
53+
include:
54+
- /var/log/messages
55+
- /var/log/dmesg
56+
- /var/log/secure
57+
start_at: end
58+
include_file_path: true
59+
include_file_name: false
60+
max_concurrent_files: 100
61+
62+
processors:
63+
# Shared enrichment processors — renamed with _logs suffix to avoid collision
64+
# with the agent's metrics-side copies during the agent-side merge.
65+
resourcedetection/cw_k8s_ci_v0_logs:
66+
detectors: [eks, ec2]
67+
ec2:
68+
resource_attributes:
69+
host.id: { enabled: true }
70+
host.type: { enabled: true }
71+
host.name: { enabled: true }
72+
host.image.id: { enabled: true }
73+
cloud.provider: { enabled: true }
74+
cloud.platform: { enabled: true }
75+
cloud.region: { enabled: true }
76+
cloud.availability_zone: { enabled: true }
77+
cloud.account.id: { enabled: true }
78+
79+
k8sattributes/cw_k8s_ci_v0_logs_node:
80+
auth_type: serviceAccount
81+
passthrough: false
82+
filter:
83+
node_from_env_var: K8S_NODE_NAME
84+
extract:
85+
metadata:
86+
- k8s.node.name
87+
labels:
88+
- tag_name: "k8s.node.label.$$$1"
89+
key_regex: "(.*)"
90+
from: node
91+
pod_association:
92+
- sources:
93+
- from: resource_attribute
94+
name: k8s.node.name
95+
96+
k8sattributes/cw_k8s_ci_v0_logs_pod:
97+
auth_type: serviceAccount
98+
passthrough: false
99+
filter:
100+
node_from_env_var: K8S_NODE_NAME
101+
extract:
102+
metadata:
103+
- k8s.pod.uid
104+
- k8s.node.name
105+
- k8s.deployment.name
106+
- k8s.statefulset.name
107+
- k8s.daemonset.name
108+
- k8s.replicaset.name
109+
- k8s.job.name
110+
- k8s.cronjob.name
111+
labels:
112+
- tag_name: "k8s.pod.label.$$$1"
113+
key_regex: "(.*)"
114+
from: pod
115+
pod_association:
116+
- sources:
117+
- from: resource_attribute
118+
name: k8s.pod.name
119+
- from: resource_attribute
120+
name: k8s.namespace.name
121+
122+
transform/cw_k8s_ci_v0_logs_set_workload:
123+
error_mode: ignore
124+
log_statements:
125+
- context: resource
126+
statements:
127+
- set(attributes["k8s.workload.name"], attributes["k8s.deployment.name"]) where attributes["k8s.deployment.name"] != nil
128+
- set(attributes["k8s.workload.type"], "Deployment") where attributes["k8s.deployment.name"] != nil
129+
- set(attributes["k8s.workload.name"], attributes["k8s.statefulset.name"]) where attributes["k8s.workload.name"] == nil and attributes["k8s.statefulset.name"] != nil
130+
- set(attributes["k8s.workload.type"], "StatefulSet") where attributes["k8s.statefulset.name"] != nil and attributes["k8s.workload.type"] == nil
131+
- set(attributes["k8s.workload.name"], attributes["k8s.daemonset.name"]) where attributes["k8s.workload.name"] == nil and attributes["k8s.daemonset.name"] != nil
132+
- set(attributes["k8s.workload.type"], "DaemonSet") where attributes["k8s.daemonset.name"] != nil and attributes["k8s.workload.type"] == nil
133+
- set(attributes["k8s.workload.name"], attributes["k8s.job.name"]) where attributes["k8s.workload.name"] == nil and attributes["k8s.job.name"] != nil
134+
- set(attributes["k8s.workload.type"], "Job") where attributes["k8s.job.name"] != nil and attributes["k8s.workload.type"] == nil
135+
- set(attributes["k8s.workload.name"], attributes["k8s.cronjob.name"]) where attributes["k8s.workload.name"] == nil and attributes["k8s.cronjob.name"] != nil
136+
- set(attributes["k8s.workload.type"], "CronJob") where attributes["k8s.cronjob.name"] != nil and attributes["k8s.workload.type"] == nil
137+
- set(attributes["k8s.workload.name"], attributes["k8s.replicaset.name"]) where attributes["k8s.workload.name"] == nil and attributes["k8s.replicaset.name"] != nil
138+
- set(attributes["k8s.workload.type"], "ReplicaSet") where attributes["k8s.replicaset.name"] != nil and attributes["k8s.workload.type"] == nil
139+
- set(attributes["service.name"], attributes["k8s.workload.name"]) where attributes["service.name"] == nil and attributes["k8s.workload.name"] != nil
140+
141+
transform/cw_k8s_ci_v0_logs_set_cluster_and_node:
142+
error_mode: ignore
143+
log_statements:
144+
- context: resource
145+
statements:
146+
- set(attributes["k8s.cluster.name"], "{{ .Values.clusterName }}")
147+
- set(attributes["k8s.node.name"], "${env:K8S_NODE_NAME}")
148+
149+
transform/cw_k8s_ci_v0_logs_set_cloud_resource_id:
150+
error_mode: ignore
151+
log_statements:
152+
- context: resource
153+
statements:
154+
- set(attributes["cloud.resource_id"], Concat(["arn:aws:eks:", attributes["cloud.region"], ":", attributes["cloud.account.id"], ":cluster/", attributes["k8s.cluster.name"]], ""))
155+
where attributes["cloud.region"] != nil and attributes["cloud.account.id"] != nil and attributes["k8s.cluster.name"] != nil
156+
157+
transform/cw_k8s_ci_v0_logs_clear_schema_url:
158+
error_mode: ignore
159+
log_statements:
160+
- context: resource
161+
statements:
162+
- set(resource.schema_url, "")
163+
164+
transform/cw_k8s_ci_v0_logs_set_scope_app:
165+
error_mode: ignore
166+
log_statements:
167+
- context: scope
168+
statements:
169+
- set(scope.schema_url, "")
170+
- set(attributes["cloudwatch.source"], "cloudwatch-agent")
171+
- set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
172+
- set(attributes["cloudwatch.pipeline"], "application-logs")
173+
174+
transform/cw_k8s_ci_v0_logs_set_scope_host:
175+
error_mode: ignore
176+
log_statements:
177+
- context: scope
178+
statements:
179+
- set(scope.schema_url, "")
180+
- set(attributes["cloudwatch.source"], "cloudwatch-agent")
181+
- set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
182+
- set(attributes["cloudwatch.pipeline"], "host-logs")
183+
184+
batch/cw_k8s_ci_v0_logs_dest:
185+
send_batch_size: 500
186+
send_batch_max_size: 500
187+
timeout: 5s
188+
189+
exporters:
190+
otlphttp/cw_k8s_ci_v0_app_logs_dest:
191+
endpoint: {{ if .Values.otelContainerInsights.cloudwatchLogsEndpoint }}{{ .Values.otelContainerInsights.cloudwatchLogsEndpoint | quote }}{{ else if hasKey .Values.adcEndpointOverrides .Values.region }}"https://logs.{{ .Values.region }}.{{ index .Values.adcEndpointOverrides .Values.region }}:443"{{ else }}"https://logs.{{ .Values.region }}.amazonaws.com:443"{{ end }}
192+
compression: none
193+
headers:
194+
x-aws-log-group: "/aws/otel/containerinsights/{{ .Values.clusterName }}/application"
195+
x-aws-log-stream: "${env:K8S_NODE_NAME}-application"
196+
sending_queue:
197+
queue_size: 500
198+
num_consumers: 10
199+
tls:
200+
insecure: false
201+
auth:
202+
authenticator: awscloudwatchlogsprovisioner/cw_k8s_ci_v0_logs
203+
204+
otlphttp/cw_k8s_ci_v0_node_logs_dest:
205+
endpoint: {{ if .Values.otelContainerInsights.cloudwatchLogsEndpoint }}{{ .Values.otelContainerInsights.cloudwatchLogsEndpoint | quote }}{{ else if hasKey .Values.adcEndpointOverrides .Values.region }}"https://logs.{{ .Values.region }}.{{ index .Values.adcEndpointOverrides .Values.region }}:443"{{ else }}"https://logs.{{ .Values.region }}.amazonaws.com:443"{{ end }}
206+
compression: none
207+
headers:
208+
x-aws-log-group: "/aws/otel/containerinsights/{{ .Values.clusterName }}/host"
209+
x-aws-log-stream: "${env:K8S_NODE_NAME}-host"
210+
sending_queue:
211+
queue_size: 500
212+
num_consumers: 10
213+
tls:
214+
insecure: false
215+
auth:
216+
authenticator: awscloudwatchlogsprovisioner/cw_k8s_ci_v0_logs
217+
218+
service:
219+
extensions:
220+
- sigv4auth/cw_k8s_ci_v0_logs_dest
221+
- awscloudwatchlogsprovisioner/cw_k8s_ci_v0_logs
222+
pipelines:
223+
logs/cw_k8s_ci_v0_app:
224+
receivers: [filelog/cw_k8s_ci_v0_app]
225+
processors:
226+
- transform/cw_k8s_ci_v0_logs_set_cluster_and_node
227+
- resourcedetection/cw_k8s_ci_v0_logs
228+
- transform/cw_k8s_ci_v0_logs_set_cloud_resource_id
229+
- k8sattributes/cw_k8s_ci_v0_logs_node
230+
- k8sattributes/cw_k8s_ci_v0_logs_pod
231+
- transform/cw_k8s_ci_v0_logs_set_scope_app
232+
- transform/cw_k8s_ci_v0_logs_clear_schema_url
233+
- transform/cw_k8s_ci_v0_logs_set_workload
234+
- batch/cw_k8s_ci_v0_logs_dest
235+
exporters:
236+
- otlphttp/cw_k8s_ci_v0_app_logs_dest
237+
238+
logs/cw_k8s_ci_v0_node:
239+
receivers: [filelog/cw_k8s_ci_v0_node]
240+
processors:
241+
- transform/cw_k8s_ci_v0_logs_set_cluster_and_node
242+
- resourcedetection/cw_k8s_ci_v0_logs
243+
- transform/cw_k8s_ci_v0_logs_set_cloud_resource_id
244+
- k8sattributes/cw_k8s_ci_v0_logs_node
245+
- transform/cw_k8s_ci_v0_logs_set_scope_host
246+
- transform/cw_k8s_ci_v0_logs_clear_schema_url
247+
- batch/cw_k8s_ci_v0_logs_dest
248+
exporters:
249+
- otlphttp/cw_k8s_ci_v0_node_logs_dest
250+
{{- end -}}

‎charts/amazon-cloudwatch-observability/tests/flag_matrix.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,10 @@ run_case() {
116116
OTEL_CI='collect\":{\"container_insights'
117117
OTEL_CI_NODE='\"role\":\"node\"'
118118
OTEL_CI_CLUSTER='\"role\":\"cluster\"'
119-
OTEL_CI_LOGS_ON='\"logs\":{\"enabled\":true}'
119+
# Hybrid model: metrics come from the JSON config (spec.config), logs from the chart
120+
# otelConfig (spec.otelConfig). The node JSON always sets logs.enabled:false; the
121+
# presence of CI logs is asserted via the logs exporter in the rendered otelConfig.
122+
OTEL_CI_LOGS_ON='cw_k8s_ci_v0_app_logs_dest'
120123
OTEL_CI_LOGS_OFF='\"logs\":{\"enabled\":false}'
121124
# aws-for-fluent-bit is the container image string — unique to the FluentBit
122125
# DaemonSet. Using this instead of bare "fluent-bit" avoids false matches in

‎charts/amazon-cloudwatch-observability/values.yaml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1161,10 +1161,8 @@ otelContainerInsights:
11611161
enabled: false
11621162
## OTEL-native CI log pipelines (application, host).
11631163
## Only takes effect when otelContainerInsights.enabled is true.
1164-
## OTEL CI logs turned off for now due to agent side fix
1165-
## logs are handled by FluentBit (containerLogs).
11661164
logs:
1167-
enabled: false
1165+
enabled: true
11681166
## The agent in the agents array that receives node-level OTEL Container Insights config.
11691167
targetAgent: "cloudwatch-agent"
11701168
## The agent in the agents array that receives cluster-level OTEL Container Insights config (apiserver, kube-state-metrics scraping).

‎integration-tests/amazon-cloudwatch-observability/validations/minikube/common.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,19 @@ func AssertOtelCILogsEnabled(t *testing.T, config string, enabled bool) {
138138
}
139139
}
140140

141+
// AssertOtelCILogs asserts presence/absence of the CI logs pipeline in the chart-provided
142+
// otelConfig. In the hybrid model, metrics come from spec.config (JSON) and logs from
143+
// spec.otelConfig, so log collection is verified here rather than via spec.config.
144+
func AssertOtelCILogs(t *testing.T, otelConfig string, present bool) {
145+
if present {
146+
assert.Contains(t, otelConfig, "cw_k8s_ci_v0_app_logs_dest",
147+
"CI logs pipeline must be present in otelConfig")
148+
} else {
149+
assert.NotContains(t, otelConfig, "cw_k8s_ci_v0_app_logs_dest",
150+
"CI logs pipeline must not be present in otelConfig")
151+
}
152+
}
153+
141154
// AssertNoOtelContainerInsights asserts OTEL CI is not configured.
142155
func AssertNoOtelContainerInsights(t *testing.T, config string) {
143156
assert.NotContains(t, config, otelCIMarker,

0 commit comments

Comments
 (0)