Skip to content

Add Karpenter metrics integration to OTEL Container Insights #325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sky333999 merged 7 commits into from
Aug 7, 2026
Merged

Add Karpenter metrics integration to OTEL Container Insights #325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7811e1f
feat: add Karpenter metrics integration to OTEL Container Insights
vaishnavi-30-beep Jun 23, 2026
783f33e
added K8s semconv,added dedicated resourcedetection for karpenter
vaishnavi-30-beep Jun 24, 2026
a297dde
fix: remove deprecated net.host.name, net.host.port, url.scheme from …
vaishnavi-30-beep Jun 26, 2026
b8e0af5
fix: drop standby pod metrics in HA Karpenter deployments
vaishnavi-30-beep Jul 7, 2026
17004cf
revert: remove ineffective metric_relabel_configs for HA standby filt…
vaishnavi-30-beep Jul 7, 2026
6d0919f
Renamed integrations to solutions
vaishnavi-30-beep Jul 21, 2026
74d8d85
fix: Move shared processors outside KSM conditional block
vaishnavi-30-beep Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 120 additions & 16 deletions ...udwatch-observability/templates/linux/_otel-container-insights-cluster-scraper-config.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,34 @@ receivers:
- {{ include "kube-state-metrics.name" . }}.{{ .Release.Namespace }}.svc:{{ .Values.kubeStateMetrics.service.port }}
{{- end }}

{{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.karpenter.enabled }}
prometheus/cw_k8s_ci_v0_karpenter:
config:
scrape_configs:
- job_name: karpenter
scrape_interval: {{ .Values.otelContainerInsights.metricResolution }}
scrape_timeout: {{ include "otel-container-insights.scrapeTimeout" . }}
metrics_path: /metrics
kubernetes_sd_configs:
- role: pod
namespaces:
names:
- {{ .Values.otelContainerInsights.solutions.karpenter.namespace }}
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
regex: karpenter
action: keep
- source_labels: [__meta_kubernetes_pod_container_port_name]
regex: http-metrics
action: keep
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
{{- end }}

processors:
filter/cw_k8s_ci_v0_scrape_metadata:
error_mode: ignore
Expand Down Expand Up @@ -134,6 +162,61 @@ processors:
- set(attributes["cloudwatch.pipeline"], "kube-state-metrics")
{{- end }}

{{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.karpenter.enabled }}
transform/cw_k8s_ci_v0_set_scope_karpenter:
error_mode: ignore
metric_statements:
- context: scope
statements:
- set(scope.name, "github.com/aws/karpenter")
- set(scope.schema_url, "")
- set(attributes["cloudwatch.source"], "cloudwatch-agent")
- set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
- set(attributes["cloudwatch.pipeline"], "karpenter")

# Promote scraped pod/namespace/node labels to OTel K8s resource attributes.
# This allows the shared k8sattributes processor to enrich with deployment/workload info.
groupbyattrs/cw_k8s_ci_v0_karpenter:
keys:
- pod
- namespace
- node

transform/cw_k8s_ci_v0_karpenter_promote:
error_mode: ignore
metric_statements:
- context: resource
statements:
- set(attributes["k8s.pod.name"], attributes["pod"]) where attributes["pod"] != nil
- set(attributes["k8s.namespace.name"], attributes["namespace"]) where attributes["namespace"] != nil
- set(attributes["k8s.node.name"], attributes["node"]) where attributes["node"] != nil
# Remove deprecated/unwanted attributes auto-injected by the Prometheus receiver.
- delete_key(attributes, "net.host.name") where attributes["net.host.name"] != nil
- delete_key(attributes, "net.host.port") where attributes["net.host.port"] != nil
- delete_key(attributes, "url.scheme") where attributes["url.scheme"] != nil
- context: datapoint
statements:
- set(attributes["pod"], resource.attributes["pod"]) where resource.attributes["pod"] != nil
- set(attributes["namespace"], resource.attributes["namespace"]) where resource.attributes["namespace"] != nil
- set(attributes["node"], resource.attributes["node"]) where resource.attributes["node"] != nil

# Karpenter-specific resource detection: only cloud-level attributes (region, account).
# No host/AZ attributes — those would incorrectly reflect the scraper's node, not Karpenter's.
resourcedetection/cw_k8s_ci_v0_karpenter:
detectors: [eks, ec2]
ec2:
resource_attributes:
host.id: { enabled: false }
host.type: { enabled: false }
host.name: { enabled: false }
host.image.id: { enabled: false }
cloud.provider: { enabled: true }
cloud.platform: { enabled: true }
cloud.region: { enabled: true }
cloud.availability_zone: { enabled: false }
cloud.account.id: { enabled: true }
{{- end }}

transform/cw_k8s_ci_v0_set_cluster_name:
error_mode: ignore
metric_statements:
Expand Down Expand Up @@ -217,6 +300,22 @@ processors:
- set(attributes["owner_name"], resource.attributes["owner_name"]) where resource.attributes["owner_name"] != nil
- set(attributes["owner_kind"], resource.attributes["owner_kind"]) where resource.attributes["owner_kind"] != nil

k8sattributes/cw_k8s_ci_v0_node:
auth_type: serviceAccount
passthrough: false
extract:
metadata:
- k8s.node.name
labels:
- tag_name: "k8s.node.label.$$$1"
key_regex: "(.*)"
from: node
pod_association:
- sources:
- from: resource_attribute
name: k8s.node.name
{{- end }}

k8sattributes/cw_k8s_ci_v0_pod:
auth_type: serviceAccount
passthrough: false
Expand Down Expand Up @@ -260,22 +359,6 @@ processors:
- set(resource.attributes["k8s.workload.name"], resource.attributes["k8s.replicaset.name"]) where resource.attributes["k8s.workload.name"] == nil and resource.attributes["k8s.replicaset.name"] != nil
- set(resource.attributes["k8s.workload.type"], "ReplicaSet") where resource.attributes["k8s.workload.type"] == nil and resource.attributes["k8s.replicaset.name"] != nil

k8sattributes/cw_k8s_ci_v0_node:
auth_type: serviceAccount
passthrough: false
extract:
metadata:
- k8s.node.name
labels:
- tag_name: "k8s.node.label.$$$1"
key_regex: "(.*)"
from: node
pod_association:
- sources:
- from: resource_attribute
name: k8s.node.name
{{- end }}

transform/cw_k8s_ci_v0_set_component:
error_mode: ignore
metric_statements:
Expand Down Expand Up @@ -405,5 +488,26 @@ service:
exporters:
- otlphttp/cw_k8s_ci_v0_cwotel
{{- end }}
{{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.karpenter.enabled }}
metrics/cw_k8s_ci_v0_karpenter:
receivers: [prometheus/cw_k8s_ci_v0_karpenter]
processors:
- filter/cw_k8s_ci_v0_scrape_metadata
- transform/cw_k8s_ci_v0_set_unit
- metricstarttime/cw_k8s_ci_v0
- transform/cw_k8s_ci_v0_set_scope_karpenter
- transform/cw_k8s_ci_v0_set_cluster_name
- groupbyattrs/cw_k8s_ci_v0_karpenter
- transform/cw_k8s_ci_v0_karpenter_promote
- k8sattributes/cw_k8s_ci_v0_pod
Comment thread
vaishnavi-30-beep marked this conversation as resolved.
- transform/cw_k8s_ci_v0_set_workload
- resourcedetection/cw_k8s_ci_v0_karpenter
- transform/cw_k8s_ci_v0_clear_schema_url
- transform/cw_k8s_ci_v0_set_cloud_resource_id
- awsattributelimit/cw_k8s_ci_v0
- batch/cw_k8s_ci_v0_cwotel
exporters:
- otlphttp/cw_k8s_ci_v0_cwotel
{{- end }}
{{- end -}}

15 changes: 15 additions & 0 deletions charts/amazon-cloudwatch-observability/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1173,3 +1173,18 @@ otelContainerInsights:
cloudwatchMetricsEndpoint: ""
# Override the CloudWatch Logs OTLP endpoint. If empty, defaults to logs.<region>.amazonaws.com.
cloudwatchLogsEndpoint: ""
## ── Solutions ──────────────────────────────────────────────────────────
## Optional Prometheus metric scraping for well-known cluster components.
## Each solution is gated by its own enabled flag so customers only scrape
## what they actually run. Solutions only take effect when
## otelContainerInsights.enabled is true.
solutions:
## Global toggle: set to false to disable ALL solutions at once.
## Individual solutions won't activate even if their own enabled flag is true.
enabled: true
## Karpenter — cluster autoscaler for Kubernetes.
## Scrapes the Prometheus metrics endpoint exposed by the Karpenter controller.
karpenter:
enabled: true
## Namespace where Karpenter is deployed.
namespace: "kube-system"
Loading