send sidecar metrics to observe directly

Author: obs-gh-virjramakrishnan

charts/agent/templates/_config-receivers.tpl

@@ -137,26 +137,3 @@ prometheus/cadvisor:
             replacement: /api/v1/nodes/$$1/proxy/metrics/cadvisor
 {{ end }}
 {{ end }}
-
-{{- define "config.receivers.prometheus.kubeletstats" -}}
-prometheus/kubeletstats:
-  config:
-    scrape_configs:
-      - job_name: 'kubernetes-nodes-kubeletstats'
-        scheme: https
-        tls_config:
-          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-          insecure_skip_verify: true
-        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
-        kubernetes_sd_configs:
-          - role: node
-
-        relabel_configs:
-          - target_label: __address__
-            replacement: kubernetes.default.svc:443
-          - source_labels: [__meta_kubernetes_node_name]
-            regex: (.+)
-            target_label: __metrics_path__
-            replacement: /api/v1/nodes/$$1/proxy/stats/summary
-{{- end -}}

charts/agent/templates/_kubeletstats-sidecar.tpl

@@ -1,7 +1,5 @@
 {{- define "observe.sidecar.fargateSidecarMetrics.config" -}}
 
-{{- $kubeletstatsExporters := (list "otlphttp" "debug") -}}
-
 receivers:
   kubeletstats:
     collection_interval: {{.Values.node.containers.metrics.interval}}
@@ -57,17 +55,41 @@ receivers:
     extra_metadata_labels:
       - container.id
 
+processors:
+
+{{- include "config.processors.memory_limiter" . | nindent 2 }}
+{{- include "config.processors.batch" . | nindent 2 }}
+{{- include "config.processors.resource_detection.cloud" . | nindent 2 }}
+{{- include "config.processors.attributes.k8sattributes" . | nindent 2 }}
+{{- include "config.processors.resource.observe_common" . | nindent 2 }}
+{{- include "config.processors.deltatocumulative" . | nindent 2 }}
+{{- include "config.processors.attributes.add_empty_service_attributes" . | nindent 2 }}
+{{- include "config.processors.metricstransform.duplicate_k8s_cpu_metrics" . | nindent 2 }}
+{{- include "config.processors.attributes.sidecar_kubeletstats_metrics" . | nindent 2 }}
+
 exporters:
-  otlphttp:
-    endpoint: http://observe-agent-forwarder.observe.svc:4318
   debug:
     verbosity: detailed
+  prometheusremotewrite/observe-sidecar:
+    endpoint: "{{ .Values.observe.collectionEndpoint.value }}/v1/prometheus"
+    headers:
+      authorization: "Bearer {{ .Values.observe.token.value }}"
+    remote_write_queue:
+      num_consumers: 5
+    max_batch_request_parallelism: 5
+    resource_to_telemetry_conversion:
+      enabled: true
+    send_metadata: true
+    timeout: 10s
+
+{{ $kubeletstatsExporters := (list "debug" "prometheusremotewrite/observe-sidecar") -}}
 
 service:
   pipelines:
     {{- if .Values.node.containers.metrics.enabled }}
       metrics/kubeletstats:
-        receivers: [kubeletstats] # should add processors back eventually
+        receivers: [kubeletstats]
+        processors: [memory_limiter, metricstransform/duplicate_k8s_cpu_metrics, k8sattributes, deltatocumulative/observe, batch, resourcedetection/cloud, resource/observe_common, attributes/debug_source_sidecar_kubeletstats_metrics]
         exporters: [{{ join ", " $kubeletstatsExporters }}]
     {{- end -}}
 {{- end }}

charts/agent/templates/kubeletstats-sidecar.yaml

@@ -5,11 +5,19 @@ metadata:
   name: fargate-sidecar-metrics
 spec:
   mode: sidecar
+  image: "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest"
   env:
     - name: K8S_NODE_NAME
       valueFrom:
         fieldRef:
           fieldPath: spec.nodeName
+    - name: OBSERVE_CLUSTER_NAME
+      value: "{{ .Values.cluster.name }}"
+    - name: OBSERVE_CLUSTER_UID
+      valueFrom:
+        configMapKeyRef:
+          name: cluster-info
+          key: id
   config:
     {{- include "observe.sidecar.applyFargateSidecarMetricsConfig" . | nindent 4 }}
   initContainers:

charts/agent/values.yaml

@@ -47,7 +47,7 @@ node:
   # -- Enables the node-logs-metrics agent daemonset for collection of node logs and metrics.
   # The nodes on which metrics and logs are collected can be configured via `affinity` in the `node-logs-metrics` section below.
   # This should be set to false to disable the node-log-metrics daemonset when running in a serverless environment (ex: EKS Fargate).
-  enabled: false
+  enabled: true
   fargateMode: false
   # collects host level metrics from node
   metrics:
@@ -101,7 +101,6 @@ node:
     # this resolves issues similar to https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/26481#issuecomment-1720797914 for `no such host` or `connection refused`.
     useNodeIp: false
   forwarder:
-    mode: deployment
     enabled: true
     traces:
       enabled: true
@@ -231,10 +230,6 @@ agent:
     #       exporters:
     #         - otlphttp/extra
     #         - otlphttp/observe/forward/trace
-    
-    # -- Additional OTel collector config for fargate-sidecar-metrics custom resource
-    fargateSidecarMetrics:
-    # Put any OTel config overrides here.
 
     # -- Additional OTel collector config for gateway deployment
     gateway:
@@ -957,7 +952,7 @@ monitor:
 forwarder:
   # -- The forwarder is run as a daemonset by default, but can be run as a deployment by setting mode to "deployment". Deployment mode
   # must be used when running in a serverless environment (ex: EKS Fargate) where daemonsets are not supported.
-  mode: deployment
+  mode: daemonset
 
   # -- The `replicaCount` is only used when `mode` is set to "deployment". It is ignored when `mode` is set to "daemonset".
   # In deployment mode, this sets the number of replicas (ie the number of forwarder pods to run).