feat: add heartbeat configs (#430)

* feat: add agent heartbeat configs

* update helm chart to include heartbeat configs

* add podUID env var to pass through for matching

* rm unnecessary processors

* use new agent version

* move configs around a little

* CR

* add pod name too

* remove unnecessary cloud resource detection processor

* fix e2e's?

* for some reason this is a patch change

* update exporter

* add helper to check for pod.uid env var

* bump to 2.10.1

Author: obs-gh-alexlew

charts/agent/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: agent
 description: Chart to install K8s collection stack based on Observe Agent
 type: application
-version: 0.74.2
-appVersion: "2.9.1"
+version: 0.74.3
+appVersion: "2.10.1"
 dependencies:
   - name: opentelemetry-collector
     version: 0.136.1

charts/agent/README.md

@@ -1,9 +1,9 @@
 observe:
   collectionEndpoint:
-    collectionEndpoint: "http://test-stack-collector.testing.svc.cluster.local:8080"
+    value: "http://test-stack-collector.testing.svc.cluster.local:8080"
   token:
     create: true
-    value: "fake-token"
+    value: "fake-ds:fake-token"
 
 application:
   prometheusScrape:
@@ -24,6 +24,14 @@ gatewayDeployment:
   traceSampling:
     enabled: true
 
+agent:
+  config:
+    global:
+      fleet:
+        enabled: true
+        interval: 10s
+        configInterval: 10m
+
 cluster:
   namespaceOverride:
     value: "testing"

charts/agent/ci/test-values.yaml

@@ -4,6 +4,10 @@ exporters:
 {{- include "config.exporters.debug" . | nindent 2 }}
 {{- include "config.exporters.otlphttp.observe.entity" . | nindent 2 }}
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.exporters.otlphttp.observe.metrics.agentheartbeat" . | nindent 2 }}
+{{- end }}
+
 receivers:
   # this is used to create a cluster resource by pulling namespaces and then dropping all but kube-system with filter processor
   k8sobjects/cluster:
@@ -64,6 +68,10 @@ receivers:
       - {name: serviceaccounts, mode: pull, interval: 15m}
       - {name: serviceaccounts, mode: watch}
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.receivers.observe.heartbeat" . | nindent 2 }}
+{{- end }}
+
 processors:
 {{- include "config.processors.memory_limiter" . | nindent 2 }}
 
@@ -78,6 +86,13 @@ processors:
 
 {{- include "config.processors.attributes.observek8sattributes" . | nindent 2 }}
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.processors.attributes.k8sattributes" . | nindent 2 }}
+{{- include "config.processors.resource_detection" . | nindent 2 }}
+{{- include "config.processors.resource.agent_instance" . | nindent 2 }}
+{{- include "config.processors.transform.k8sheartbeat" . | nindent 2 }}
+{{- end }}
+
   transform/unify:
     error_mode: ignore
     log_statements:
@@ -451,4 +466,8 @@ service:
         processors: [memory_limiter, batch, resource/observe_common, filter/cluster, transform/cluster]
         exporters: [{{ join ", " $logsClusterExporters }}]
 
+      {{- if .Values.agent.config.global.fleet.enabled }}
+      {{- include "config.pipelines.heartbeat" . | nindent 6 }}
+      {{- end }}
+
 {{- end }}

charts/agent/templates/_cluster-events-config.tpl

@@ -6,6 +6,10 @@ exporters:
 {{- include "config.exporters.debug" . | nindent 2 }}
 {{- include "config.exporters.prometheusremotewrite" . | nindent 2 }}
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.exporters.otlphttp.observe.metrics.agentheartbeat" . | nindent 2 }}
+{{- end }}
+
 receivers:
   # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/k8sclusterreceiver/documentation.md
   k8s_cluster:
@@ -31,6 +35,10 @@ receivers:
 {{- include "config.receivers.prometheus.cadvisor" . | nindent 2 }}
 {{- end }}
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.receivers.observe.heartbeat" . | nindent 2 }}
+{{- end }}
+
 processors:
 {{- include "config.processors.memory_limiter" . | nindent 2 }}
 {{- include "config.processors.batch" . | nindent 2 }}
@@ -42,6 +50,12 @@ processors:
 {{- if $podMetrics }}
 {{- include "config.processors.attributes.pod_metrics" . | nindent 2 }}
 {{- include "config.processors.attributes.cadvisor_metrics" . | nindent 2 }}
+{{- end }}
+
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.processors.resource_detection" . | nindent 2 }}
+{{- include "config.processors.resource.agent_instance" . | nindent 2 }}
+{{- include "config.processors.transform.k8sheartbeat" . | nindent 2 }}
 {{- end }}
 
   # attributes to append to objects
@@ -67,4 +81,8 @@ service:
     {{- include "config.pipelines.prometheus_scrapers" . | nindent 4 }}
     {{- end }}
 
+    {{- if .Values.agent.config.global.fleet.enabled }}
+    {{- include "config.pipelines.heartbeat" . | nindent 4 }}
+    {{- end }}
+
 {{- end }}

charts/agent/templates/_cluster-metrics-config.tpl

@@ -55,6 +55,26 @@ otlphttp/observe/forward/trace:
   compression: zstd
 {{- end -}}
 
+{{- define "config.exporters.otlphttp.observe.metrics.agentheartbeat" -}}
+otlphttp/observe/agentheartbeat:
+    # These environment variables are provided by the observe-agent:
+    # https://github.com/observeinc/observe-agent/blob/v2.0.1/internal/connections/confighandler.go#L91-L102
+    logs_endpoint: "${env:OBSERVE_COLLECTOR_URL}/v1/kubernetes/v1/entity"
+    headers:
+        authorization: "${env:OBSERVE_AUTHORIZATION_HEADER}"
+        x-observe-target-package: "Observe Agent"
+        x-observe-context: {{ mustToJson (dict "version" "${env:OBSERVE_AGENT_VERSION}" "environment" "kubernetes" ) | toYaml }}
+        x-observe-enable-auth-error-reporting: "true"
+    sending_queue:
+      enabled: {{ .Values.agent.config.global.exporters.sendingQueue.enabled }}
+    retry_on_failure:
+      enabled: {{ .Values.agent.config.global.exporters.retryOnFailure.enabled }}
+      initial_interval: {{ .Values.agent.config.global.exporters.retryOnFailure.initialInterval }}
+      max_interval: {{ .Values.agent.config.global.exporters.retryOnFailure.maxInterval }}
+      max_elapsed_time: {{ .Values.agent.config.global.exporters.retryOnFailure.maxElapsedTime }}
+    compression: zstd
+{{- end -}}
+
 {{- define "config.exporters.otlphttp.observe.metrics.otel" -}}
 otlphttp/observe/otel_metrics:
   # These environment variables are provided by the observe-agent:

charts/agent/templates/_config-exporters.tpl

@@ -63,3 +63,19 @@ metrics/cadvisor:
 {{- end }}
 
 {{- end }}
+
+{{- define "config.pipelines.heartbeat" -}}
+logs/heartbeat:
+    exporters:
+      - otlphttp/observe/agentheartbeat
+      {{- if .Values.agent.config.global.debug.enabled }}
+      - debug/override
+      {{- end }}
+    processors:
+        - resourcedetection
+        - resource/agent_instance
+        - k8sattributes
+        - transform/k8sheartbeat
+    receivers:
+        - heartbeat
+{{- end }}

charts/agent/templates/_config-pipelines.tpl

@@ -5,6 +5,13 @@ resourcedetection/cloud:
   override: false
 {{- end -}}
 
+{{- define "config.processors.resource_detection" -}}
+resourcedetection:
+  detectors: ["env", "system"]
+  timeout: 2s
+  override: false
+{{- end -}}
+
 {{- define "config.processors.batch" -}}
 batch:
   send_batch_size: {{ .Values.agent.config.global.processors.batch.sendBatchSize }}
@@ -84,6 +91,30 @@ resource/observe_common:
     {{ end }}
 {{- end -}}
 
+{{- define "config.processors.resource.agent_instance" -}}
+resource/agent_instance:
+    attributes:
+        - key: k8s.pod.uid
+          value: ${env:OTEL_K8S_POD_UID}
+          action: upsert
+        - key: k8s.pod.name
+          value: ${env:OTEL_K8S_POD_NAME}
+          action: upsert
+{{- end -}}
+
+{{- define "config.processors.transform.k8sheartbeat" -}}
+transform/k8sheartbeat:
+  error_mode: ignore
+  log_statements:
+    - context: log
+      statements:
+        - set(attributes["observe_transform"]["identifiers"]["host.name"], resource.attributes["k8s.node.name"])
+        - set(attributes["observe_transform"]["identifiers"]["k8s.pod.uid"], resource.attributes["k8s.pod.uid"])
+        - set(attributes["observe_transform"]["identifiers"]["k8s.pod.name"], resource.attributes["k8s.pod.name"])
+        - set(attributes["observe_transform"]["identifiers"]["k8s.deployment.name"], resource.attributes["k8s.deployment.name"])
+        - set(attributes["observe_transform"]["identifiers"]["k8s.daemonset.name"], resource.attributes["k8s.daemonset.name"])
+{{- end -}}
+
 {{- define "config.processors.memory_limiter" -}}
 # https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiterprocessor/README.md
 memory_limiter:

charts/agent/templates/_config-processors.tpl

@@ -137,3 +137,14 @@ prometheus/cadvisor:
             replacement: /api/v1/nodes/$$1/proxy/metrics/cadvisor
 {{ end }}
 {{ end }}
+
+{{- define "config.receivers.observe.heartbeat" -}}
+heartbeat:
+    auth_check:
+        headers:
+            authorization: "${env:OBSERVE_AUTHORIZATION_HEADER}"
+        url: "${env:OBSERVE_COLLECTOR_URL}"
+    environment: kubernetes
+    interval: {{ .Values.agent.config.global.fleet.interval }}
+    config_interval: {{ .Values.agent.config.global.fleet.configInterval }}
+{{- end }}

charts/agent/templates/_config-receivers.tpl

@@ -35,6 +35,10 @@ exporters:
   {{- include "config.exporters.prometheusremotewrite" . | nindent 2 }}
 {{- end }}
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.exporters.otlphttp.observe.metrics.agentheartbeat" . | nindent 2 }}
+{{- end }}
+
 receivers:
   otlp/app-telemetry:
     protocols:
@@ -43,6 +47,10 @@ receivers:
       http:
         endpoint: ${env:MY_POD_IP}:4318
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.receivers.observe.heartbeat" . | nindent 2 }}
+{{- end }}
+
 processors:
 
 {{- include "config.processors.memory_limiter" . | nindent 2 }}
@@ -54,6 +62,10 @@ processors:
   # Use passthrough mode to reduce forwarder compute and push the lookup to the gateway whenever it is enabled.
   k8sattributes/passthrough:
     passthrough: true
+  {{- if .Values.agent.config.global.fleet.enabled }}
+  # k8sattributes is needed for the heartbeat pipeline even when gateway is enabled
+  {{- include "config.processors.attributes.k8sattributes" . | nindent 2 }}
+  {{- end }}
 {{- else }}
   {{- include "config.processors.attributes.k8sattributes" . | nindent 2 }}
   {{- include "config.processors.resource.observe_common" . | nindent 2 }}
@@ -90,6 +102,12 @@ processors:
 {{- include "config.processors.RED_metrics" . | nindent 2 }}
 {{- end }}
 
+{{- if .Values.agent.config.global.fleet.enabled }}
+{{- include "config.processors.resource_detection" . | nindent 2 }}
+{{- include "config.processors.resource.agent_instance" . | nindent 2 }}
+{{- include "config.processors.transform.k8sheartbeat" . | nindent 2 }}
+{{- end }}
+
 {{- $traceExporters := (list) -}}
 {{- $logsExporters := (list) -}}
 {{- $metricsExporters := (list) -}}
@@ -185,4 +203,8 @@ service:
     {{- include "config.pipelines.RED_metrics" . | nindent 4 }}
     {{- end }}
 
+    {{- if .Values.agent.config.global.fleet.enabled }}
+    {{- include "config.pipelines.heartbeat" . | nindent 4 }}
+    {{- end }}
+
 {{- end }}