add extra pipeline for kubeletstats monitoring in fargate pods

Author: obs-gh-virjramakrishnan

charts/agent/Chart.lock

@@ -20,5 +20,8 @@ dependencies:
 - name: opentelemetry-collector
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
   version: 0.130.2
-digest: sha256:1463a6ca81d2cffd7c7cdf60a8bbc1f490ca721a50328f17a9b1f8d06a1dc6b1
-generated: "2025-08-26T11:41:10.527947-04:00"
+- name: opentelemetry-operator
+  repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+  version: 0.93.1
+digest: sha256:b5548207946689a925841cca60cf59984043d00886dcb93b407c144630af909f
+generated: "2025-09-29T16:49:24.216039-07:00"

charts/agent/Chart.yaml

@@ -40,6 +40,11 @@ dependencies:
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
     alias: gateway
     condition: gatewayDeployment.enabled
+  - name: opentelemetry-operator
+    version: 0.93.1
+    repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+    alias: fargate-sidecar-injector
+    condition: node.fargateMode
 maintainers:
   - name: Observe
     email: [email protected]

charts/agent/templates/_config-processors.tpl

@@ -112,6 +112,14 @@ attributes/debug_source_cadvisor_metrics:
 {{- end -}}
 {{- end -}}
 
+{{- define "config.processors.attributes.sidecar_kubeletstats_metrics" -}}
+attributes/debug_source_sidecar_kubeletstats_metrics:
+  actions:
+    - key: debug_source
+      action: insert
+      value: sidecar_kubeletstats_metrics
+{{- end -}}
+
 {{- define "config.processors.attributes.drop_container_info" -}}
 resource/drop_container_info:
   attributes:
@@ -126,6 +134,21 @@ resource/drop_service_name:
       key: service.name
 {{- end -}}
 
+{{- define "config.processors.metricstransform.duplicate_k8s_cpu_metrics" -}}
+# convert new k8s metric names to the names our Kubernetes Explorer relies on
+metricstransform/duplicate_k8s_cpu_metrics:
+  transforms:
+    - include: container.cpu.usage
+      action: insert
+      new_name: container.cpu.utilization
+    - include: k8s.pod.cpu.usage
+      action: insert
+      new_name: k8s.pod.cpu.utilization
+    - include: k8s.node.cpu.usage
+      action: insert
+      new_name: k8s.node.cpu.utilization
+{{- end -}}
+
 {{- define "config.processors.filter.drop_long_spans" -}}
 {{- if eq .Values.node.forwarder.traces.maxSpanDuration "none" }}
 {{- else if (regexMatch "^[0-9]+(ns|us|ms|s|m|h)$" .Values.node.forwarder.traces.maxSpanDuration) }}

charts/agent/templates/_config-receivers.tpl

@@ -137,3 +137,26 @@ prometheus/cadvisor:
             replacement: /api/v1/nodes/$$1/proxy/metrics/cadvisor
 {{ end }}
 {{ end }}
+
+{{- define "config.receivers.prometheus.kubeletstats" -}}
+prometheus/kubeletstats:
+  config:
+    scrape_configs:
+      - job_name: 'kubernetes-nodes-kubeletstats'
+        scheme: https
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          insecure_skip_verify: true
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+        kubernetes_sd_configs:
+          - role: node
+
+        relabel_configs:
+          - target_label: __address__
+            replacement: kubernetes.default.svc:443
+          - source_labels: [__meta_kubernetes_node_name]
+            regex: (.+)
+            target_label: __metrics_path__
+            replacement: /api/v1/nodes/$$1/proxy/stats/summary
+{{- end -}}

charts/agent/templates/_config.tpl

@@ -12,6 +12,14 @@
 {{- toYaml $config | indent 2 }}
 {{- end }}
 
+{{- define "observe.sidecar.applyFargateSidecarMetricsConfig" -}}
+{{- $values := deepCopy .Values }}
+{{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
+{{- $config := mustMergeOverwrite ( include "observe.sidecar.fargateSidecarMetrics.config" $data |  fromYaml ) ($values.agent.config.fargateSidecarMetrics) ($values.agent.config.global.overrides) -}}
+{{- toYaml $config | indent 2 }}
+{{- end }}
+
+
 {{- define "observe.deployment.applyPrometheusScraperConfig" -}}
 {{- $values := deepCopy .Values }}
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}

charts/agent/templates/_kubeletstats-sidecar.tpl

@@ -0,0 +1,73 @@
+{{- define "observe.sidecar.fargateSidecarMetrics.config" -}}
+
+{{- $kubeletstatsExporters := (list "otlphttp" "debug") -}}
+
+receivers:
+  kubeletstats:
+    collection_interval: {{.Values.node.containers.metrics.interval}}
+    auth_type: 'serviceAccount'
+    endpoint: https://kubernetes.default.svc/api/v1/nodes/${env:K8S_NODE_NAME}/proxy
+    node: '${env:K8S_NODE_NAME}'
+    insecure_skip_verify: true
+    k8s_api_config:
+        auth_type: serviceAccount
+    metric_groups:
+      - node 
+      - pod
+      - container
+    metrics:
+      # The following metrics are optional and must be enabled manually as per:
+      # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/kubeletstatsreceiver/documentation.md#optional-metrics
+      container.cpu.usage:
+        enabled: true
+      container.uptime:
+        enabled: true
+      k8s.container.cpu.node.utilization:
+        enabled: true
+      k8s.container.cpu_limit_utilization:
+        enabled: true
+      k8s.container.cpu_request_utilization:
+        enabled: true
+      k8s.container.memory.node.utilization:
+        enabled: true
+      k8s.container.memory_limit_utilization:
+        enabled: true
+      k8s.container.memory_request_utilization:
+        enabled: true
+      k8s.node.cpu.usage:
+        enabled: true
+      k8s.node.uptime:
+        enabled: true
+      k8s.pod.cpu.node.utilization:
+        enabled: true
+      k8s.pod.cpu.usage:
+        enabled: true
+      k8s.pod.cpu_limit_utilization:
+        enabled: true
+      k8s.pod.cpu_request_utilization:
+        enabled: true
+      k8s.pod.memory.node.utilization:
+        enabled: true
+      k8s.pod.memory_limit_utilization:
+        enabled: true
+      k8s.pod.memory_request_utilization:
+        enabled: true
+      k8s.pod.uptime:
+        enabled: true
+    extra_metadata_labels:
+      - container.id
+
+exporters:
+  otlphttp:
+    endpoint: http://observe-agent-forwarder.observe.svc:4318
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    {{- if .Values.node.containers.metrics.enabled }}
+      metrics/kubeletstats:
+        receivers: [kubeletstats] # should add processors back eventually
+        exporters: [{{ join ", " $kubeletstatsExporters }}]
+    {{- end -}}
+{{- end }}

charts/agent/templates/_node-logs-metrics-config.tpl

@@ -157,6 +157,7 @@ processors:
 {{- include "config.processors.batch" . | nindent 2 }}
 {{- include "config.processors.attributes.k8sattributes" . | nindent 2 }}
 {{- include "config.processors.resource.observe_common" . | nindent 2 }}
+{{- include "config.processors.metricstransform.duplicate_k8s_cpu_metrics" . | nindent 2 }}
 
   # attributes to append to objects
   attributes/debug_source_pod_logs:
@@ -175,18 +176,6 @@ processors:
         action: insert
         value: kubeletstats_metrics
 
-  # convert new k8s metric names to the names our Kubernetes Explorer relies on
-  metricstransform/duplicate_k8s_cpu_metrics:
-    transforms:
-      - include: container.cpu.usage
-        action: insert
-        new_name: container.cpu.utilization
-      - include: k8s.pod.cpu.usage
-        action: insert
-        new_name: k8s.pod.cpu.utilization
-      - include: k8s.node.cpu.usage
-        action: insert
-        new_name: k8s.node.cpu.utilization
 
 # Create intermediate lists for pipeline arrays to then modify based on values.yaml
 {{- $logsExporters := (list "otlphttp/observe/base") -}}

charts/agent/templates/kubeletstats-sidecar.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.node.fargateSidecar.enabled }}
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: fargate-sidecar-metrics
+spec:
+  mode: sidecar
+  env:
+    - name: K8S_NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+  config:
+    {{- include "observe.sidecar.applyFargateSidecarMetricsConfig" . | nindent 4 }}
+  initContainers:
+    - name: kube-cluster-info
+      image: observeinc/kube-cluster-info:v0.11.5
+      imagePullPolicy: Always
+      env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+{{- end }}

charts/agent/values.yaml

@@ -47,7 +47,8 @@ node:
   # -- Enables the node-logs-metrics agent daemonset for collection of node logs and metrics.
   # The nodes on which metrics and logs are collected can be configured via `affinity` in the `node-logs-metrics` section below.
   # This should be set to false to disable the node-log-metrics daemonset when running in a serverless environment (ex: EKS Fargate).
-  enabled: true
+  enabled: false
+  fargateMode: false
   # collects host level metrics from node
   metrics:
     enabled: true
@@ -100,6 +101,7 @@ node:
     # this resolves issues similar to https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/26481#issuecomment-1720797914 for `no such host` or `connection refused`.
     useNodeIp: false
   forwarder:
+    mode: deployment
     enabled: true
     traces:
       enabled: true
@@ -229,6 +231,10 @@ agent:
     #       exporters:
     #         - otlphttp/extra
     #         - otlphttp/observe/forward/trace
+    
+    # -- Additional OTel collector config for fargate-sidecar-metrics custom resource
+    fargateSidecarMetrics:
+    # Put any OTel config overrides here.
 
     # -- Additional OTel collector config for gateway deployment
     gateway:
@@ -951,7 +957,7 @@ monitor:
 forwarder:
   # -- The forwarder is run as a daemonset by default, but can be run as a deployment by setting mode to "deployment". Deployment mode
   # must be used when running in a serverless environment (ex: EKS Fargate) where daemonsets are not supported.
-  mode: daemonset
+  mode: deployment
 
   # -- The `replicaCount` is only used when `mode` is set to "deployment". It is ignored when `mode` is set to "daemonset".
   # In deployment mode, this sets the number of replicas (ie the number of forwarder pods to run).
@@ -1245,3 +1251,27 @@ gateway:
     - name: observe-agent-deployment-config
       mountPath: /observe-agent-conf
   # ----------------------------------------- #
+image:
+  repository: "otel/opentelemetry-collector-k8s"
+
+
+fargate-sidecar-injector:
+
+  # -- This is an otel operator that will inject a sidecar container into all pods in the cluster. This is only needed when running 
+  # in a serverless environment (ex: EKS Fargate) where daemonsets are not supported.
+
+  replicaCount: 1
+
+  # ----------------------------------------- #
+  # Different for each deployment/daemonset #
+  nameOverride: "fargate-sidecar-injector"
+  # !!! IMPORTANT !!! This needs to have same value as namespaceOverride in cluster above
+  namespaceOverride: "observe"
+  # for now, use defaults for the rest of the values
+  # ----------------------------------------- #
+  manager:
+    collectorImage:
+      repository: observeinc/observe-agent
+      tag: 2.8.1
+
+

cluster_role.yaml

@@ -0,0 +1,31 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: otel-sidecar-role
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - namespaces
+      - pods
+    verbs: ["get", "list", "watch"]
+
+  - apiGroups: ["apps"]
+    resources:
+      - replicasets
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: otel-sidecar-role-binding
+subjects:
+- kind: ServiceAccount
+  name: my-otel-demo
+  namespace: demo-fargate
+roleRef:
+  kind: ClusterRole
+  name: otel-sidecar-role
+  apiGroup: rbac.authorization.k8s.io