send sidecar metrics to observe directly

Author: obs-gh-virjramakrishnan

charts/agent/Chart.lock

@@ -20,8 +20,5 @@ dependencies:
 - name: opentelemetry-collector
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
   version: 0.130.2
-- name: opentelemetry-operator
-  repository: https://open-telemetry.github.io/opentelemetry-helm-charts
-  version: 0.93.1
-digest: sha256:b5548207946689a925841cca60cf59984043d00886dcb93b407c144630af909f
-generated: "2025-09-29T16:49:24.216039-07:00"
+digest: sha256:1463a6ca81d2cffd7c7cdf60a8bbc1f490ca721a50328f17a9b1f8d06a1dc6b1
+generated: "2025-10-01T13:00:21.618875-07:00"

charts/agent/Chart.yaml

@@ -40,11 +40,6 @@ dependencies:
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
     alias: gateway
     condition: gatewayDeployment.enabled
-  - name: opentelemetry-operator
-    version: 0.93.1
-    repository: https://open-telemetry.github.io/opentelemetry-helm-charts
-    alias: fargate-sidecar-injector
-    condition: node.fargateMode
 maintainers:
   - name: Observe
     email: [email protected]

charts/agent/templates/_config-receivers.tpl

@@ -137,26 +137,3 @@ prometheus/cadvisor:
             replacement: /api/v1/nodes/$$1/proxy/metrics/cadvisor
 {{ end }}
 {{ end }}
-
-{{- define "config.receivers.prometheus.kubeletstats" -}}
-prometheus/kubeletstats:
-  config:
-    scrape_configs:
-      - job_name: 'kubernetes-nodes-kubeletstats'
-        scheme: https
-        tls_config:
-          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-          insecure_skip_verify: true
-        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
-        kubernetes_sd_configs:
-          - role: node
-
-        relabel_configs:
-          - target_label: __address__
-            replacement: kubernetes.default.svc:443
-          - source_labels: [__meta_kubernetes_node_name]
-            regex: (.+)
-            target_label: __metrics_path__
-            replacement: /api/v1/nodes/$$1/proxy/stats/summary
-{{- end -}}

charts/agent/templates/_kubeletstats-sidecar.tpl

@@ -1,7 +1,5 @@
 {{- define "observe.sidecar.fargateSidecarMetrics.config" -}}
 
-{{- $kubeletstatsExporters := (list "otlphttp" "debug") -}}
-
 receivers:
   kubeletstats:
     collection_interval: {{.Values.node.containers.metrics.interval}}
@@ -57,17 +55,29 @@ receivers:
     extra_metadata_labels:
       - container.id
 
+processors:
+
+{{- include "config.processors.memory_limiter" . | nindent 2 }}
+{{- include "config.processors.batch" . | nindent 2 }}
+{{- include "config.processors.resource_detection.cloud" . | nindent 2 }}
+{{- include "config.processors.attributes.k8sattributes" . | nindent 2 }}
+{{- include "config.processors.resource.observe_common" . | nindent 2 }}
+{{- include "config.processors.deltatocumulative" . | nindent 2 }}
+{{- include "config.processors.attributes.add_empty_service_attributes" . | nindent 2 }}
+{{- include "config.processors.metricstransform.duplicate_k8s_cpu_metrics" . | nindent 2 }}
+{{- include "config.processors.attributes.sidecar_kubeletstats_metrics" . | nindent 2 }}
+
 exporters:
-  otlphttp:
-    endpoint: http://observe-agent-forwarder.observe.svc:4318
   debug:
     verbosity: detailed
+{{- include "config.exporters.prometheusremotewrite" . | nindent 2 }}
+
+{{ $kubeletstatsExporters := (list "debug" "prometheusremotewrite/observe") -}}
 
 service:
   pipelines:
-    {{- if .Values.node.containers.metrics.enabled }}
-      metrics/kubeletstats:
-        receivers: [kubeletstats] # should add processors back eventually
-        exporters: [{{ join ", " $kubeletstatsExporters }}]
-    {{- end -}}
-{{- end }}
+    metrics/kubeletstats:
+      receivers: [kubeletstats]
+      processors: [memory_limiter, metricstransform/duplicate_k8s_cpu_metrics, k8sattributes, deltatocumulative/observe, batch, resourcedetection/cloud, resource/observe_common, attributes/debug_source_sidecar_kubeletstats_metrics]
+      exporters: [{{ join ", " $kubeletstatsExporters }}]
+{{- end }}

charts/agent/templates/kubeletstats-sidecar.yaml

@@ -1,15 +1,27 @@
-{{- if .Values.node.fargateSidecar.enabled }}
+{{- if .Values.node.fargateMode }}
 apiVersion: opentelemetry.io/v1beta1
 kind: OpenTelemetryCollector
 metadata:
   name: fargate-sidecar-metrics
 spec:
   mode: sidecar
+  image: "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest"
   env:
     - name: K8S_NODE_NAME
       valueFrom:
         fieldRef:
           fieldPath: spec.nodeName
+    - name: OBSERVE_CLUSTER_NAME
+      value: "{{ .Values.cluster.name }}"
+    - name: OBSERVE_CLUSTER_UID
+      valueFrom:
+        configMapKeyRef:
+          name: cluster-info
+          key: id
+    - name: OBSERVE_PROMETHEUS_ENDPOINT
+      value: "{{ .Values.observe.collectionEndpoint.value }}v1/prometheus"
+    - name: OBSERVE_AUTHORIZATION_HEADER
+      value: "Bearer {{ .Values.observe.token.value }}"
   config:
     {{- include "observe.sidecar.applyFargateSidecarMetricsConfig" . | nindent 4 }}
   initContainers:

charts/agent/values.yaml

@@ -47,7 +47,7 @@ node:
   # -- Enables the node-logs-metrics agent daemonset for collection of node logs and metrics.
   # The nodes on which metrics and logs are collected can be configured via `affinity` in the `node-logs-metrics` section below.
   # This should be set to false to disable the node-log-metrics daemonset when running in a serverless environment (ex: EKS Fargate).
-  enabled: false
+  enabled: true
   fargateMode: false
   # collects host level metrics from node
   metrics:
@@ -101,7 +101,6 @@ node:
     # this resolves issues similar to https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/26481#issuecomment-1720797914 for `no such host` or `connection refused`.
     useNodeIp: false
   forwarder:
-    mode: deployment
     enabled: true
     traces:
       enabled: true
@@ -231,10 +230,6 @@ agent:
     #       exporters:
     #         - otlphttp/extra
     #         - otlphttp/observe/forward/trace
-    
-    # -- Additional OTel collector config for fargate-sidecar-metrics custom resource
-    fargateSidecarMetrics:
-    # Put any OTel config overrides here.
 
     # -- Additional OTel collector config for gateway deployment
     gateway:
@@ -957,7 +952,7 @@ monitor:
 forwarder:
   # -- The forwarder is run as a daemonset by default, but can be run as a deployment by setting mode to "deployment". Deployment mode
   # must be used when running in a serverless environment (ex: EKS Fargate) where daemonsets are not supported.
-  mode: deployment
+  mode: daemonset
 
   # -- The `replicaCount` is only used when `mode` is set to "deployment". It is ignored when `mode` is set to "daemonset".
   # In deployment mode, this sets the number of replicas (ie the number of forwarder pods to run).
@@ -1253,25 +1248,3 @@ gateway:
   # ----------------------------------------- #
 image:
   repository: "otel/opentelemetry-collector-k8s"
-
-
-fargate-sidecar-injector:
-
-  # -- This is an otel operator that will inject a sidecar container into all pods in the cluster. This is only needed when running 
-  # in a serverless environment (ex: EKS Fargate) where daemonsets are not supported.
-
-  replicaCount: 1
-
-  # ----------------------------------------- #
-  # Different for each deployment/daemonset #
-  nameOverride: "fargate-sidecar-injector"
-  # !!! IMPORTANT !!! This needs to have same value as namespaceOverride in cluster above
-  namespaceOverride: "observe"
-  # for now, use defaults for the rest of the values
-  # ----------------------------------------- #
-  manager:
-    collectorImage:
-      repository: observeinc/observe-agent
-      tag: 2.8.1
-
-