Support pushing span/trace through to splunk HEC (signalfx#629)

mr-miles · web-flow · commit daab50e2b337 · 2023-01-26T13:35:10.000-08:00
I wanted to export spans to our splunk setup from the gateway but the way the chart is currently set up made it difficult to get all the pieces to match up (particularly port declarations across deployment/service/pods) without manually patching each one of them.

So this commit adds tracesIndex/tracesEnabled alongside metricsIndex/metricsEnabled etc so the traces are exported as expected.
diff --git a/helm-charts/splunk-otel-collector/templates/_helpers.tpl b/helm-charts/splunk-otel-collector/templates/_helpers.tpl
@@ -107,6 +107,13 @@ Whether metrics enabled for Splunk Platform.
 {{- and (eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true") .Values.splunkPlatform.metricsEnabled }}
 {{- end -}}
 
+{{/*
+Whether traces enabled for Splunk Platform.
+*/}}
+{{- define "splunk-otel-collector.platformTracesEnabled" -}}
+{{- and (eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true") .Values.splunkPlatform.tracesEnabled }}
+{{- end -}}
+
 {{/*
 Whether metrics enabled for any destination.
 */}}
@@ -115,10 +122,10 @@ Whether metrics enabled for any destination.
 {{- end -}}
 
 {{/*
-Whether traces enabled for any destination. (currently applicable to Splunk Observability only).
+Whether traces enabled for any destination.
 */}}
 {{- define "splunk-otel-collector.tracesEnabled" -}}
-{{- include "splunk-otel-collector.o11yTracesEnabled" . }}
+{{- or (eq (include "splunk-otel-collector.o11yTracesEnabled" .) "true") (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
 {{- end -}}
 
 {{/*
diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl
@@ -211,7 +211,7 @@ splunk_hec/platform_logs:
 {{- end }}
 
 {{/*
-Splunk Platform Logs exporter
+Splunk Platform Metrics exporter
 */}}
 {{- define "splunk-otel-collector.splunkPlatformMetricsExporter" -}}
 splunk_hec/platform_metrics:
@@ -246,6 +246,42 @@ splunk_hec/platform_metrics:
     queue_size: {{ .Values.splunkPlatform.sendingQueue.queueSize }}
 {{- end }}
 
+{{/*
+Splunk Platform Traces exporter
+*/}}
+{{- define "splunk-otel-collector.splunkPlatformTracesExporter" -}}
+splunk_hec/platform_traces:
+  endpoint: {{ .Values.splunkPlatform.endpoint | quote }}
+  token: "${SPLUNK_PLATFORM_HEC_TOKEN}"
+  index: {{ .Values.splunkPlatform.tracesIndex | quote }}
+  source: {{ .Values.splunkPlatform.source | quote }}
+  max_connections: {{ .Values.splunkPlatform.maxConnections }}
+  disable_compression: {{ .Values.splunkPlatform.disableCompression }}
+  timeout: {{ .Values.splunkPlatform.timeout }}
+  splunk_app_name: {{ .Chart.Name }}
+  splunk_app_version: {{ .Chart.Version }}
+  tls:
+    insecure_skip_verify: {{ .Values.splunkPlatform.insecureSkipVerify }}
+    {{- if .Values.splunkPlatform.clientCert }}
+    cert_file: /otel/etc/splunk_platform_hec_client_cert
+    {{- end }}
+    {{- if .Values.splunkPlatform.clientKey  }}
+    key_file: /otel/etc/splunk_platform_hec_client_key
+    {{- end }}
+    {{- if .Values.splunkPlatform.caFile }}
+    ca_file: /otel/etc/splunk_platform_hec_ca_file
+    {{- end }}
+  retry_on_failure:
+    enabled: {{ .Values.splunkPlatform.retryOnFailure.enabled }}
+    initial_interval: {{ .Values.splunkPlatform.retryOnFailure.initialInterval }}
+    max_interval: {{ .Values.splunkPlatform.retryOnFailure.maxInterval }}
+    max_elapsed_time: {{ .Values.splunkPlatform.retryOnFailure.maxElapsedTime }}
+  sending_queue:
+    enabled:  {{ .Values.splunkPlatform.sendingQueue.enabled }}
+    num_consumers: {{ .Values.splunkPlatform.sendingQueue.numConsumers }}
+    queue_size: {{ .Values.splunkPlatform.sendingQueue.queueSize }}
+{{- end }}
+
 {{/*
 Add Extra Labels
 */}}
diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
@@ -587,6 +587,9 @@ exporters:
   {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" .) "true") }}
   {{- include "splunk-otel-collector.splunkPlatformMetricsExporter" . | nindent 2 }}
   {{- end }}
+  {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.splunkPlatformTracesExporter" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }}
@@ -698,7 +701,13 @@ service:
     {{- if (eq (include "splunk-otel-collector.tracesEnabled" .) "true") }}
     # Default traces pipeline.
     traces:
-      receivers: [otlp, jaeger, smartagent/signalfx-forwarder, zipkin]
+      receivers:
+        - otlp
+        - jaeger
+        {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" $) "true") }}
+        - smartagent/signalfx-forwarder
+        {{- end }}
+        - zipkin
       processors:
         - memory_limiter
         - k8sattributes
@@ -712,8 +721,13 @@ service:
         {{- if $gatewayEnabled }}
         - otlp
         {{- else }}
+        {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" .) "true") }}
         - sapm
         {{- end }}
+        {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+        - splunk_hec/platform_traces
+        {{- end }}
+        {{- end }}
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" $) "true") }}
         # For trace/metric correlation.
         - signalfx
diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl
@@ -164,6 +164,9 @@ exporters:
   {{- include "splunk-otel-collector.splunkPlatformMetricsExporter" . | nindent 2 }}
   {{- end }}
 
+  {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.splunkPlatformTracesExporter" . | nindent 2 }}
+  {{- end }}
 service:
   telemetry:
     metrics:
@@ -179,7 +182,7 @@ service:
   # The default pipelines should not need to be changed. You can add any custom pipeline instead.
   # In order to disable a default pipeline just set it to `null` in gateway.config overrides.
   pipelines:
-    {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" $) "true") }}
+    {{- if (eq (include "splunk-otel-collector.tracesEnabled" $) "true") }}
     # default traces pipeline
     traces:
       receivers: [otlp, jaeger, zipkin]
@@ -194,7 +197,13 @@ service:
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
-      exporters: [sapm]
+      exporters:
+        {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" .) "true") }}
+        - sapm
+        {{- end }}
+        {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+        - splunk_hec/platform_traces
+        {{- end }}
     {{- end }}
 
     {{- if (eq (include "splunk-otel-collector.metricsEnabled" .) "true") }}
diff --git a/helm-charts/splunk-otel-collector/templates/daemonset.yaml b/helm-charts/splunk-otel-collector/templates/daemonset.yaml
@@ -232,7 +232,7 @@ spec:
         {{- end }}
         ports:
         {{- range $key, $port := $agent.ports }}
-        {{- if eq true (and (eq (include "splunk-otel-collector.metricsEnabled" $) "true") (has "metrics" $port.enabled_for)) (and (eq (include "splunk-otel-collector.o11yTracesEnabled" $) "true") (has "traces" $port.enabled_for)) (and (eq (include "splunk-otel-collector.logsEnabled" $) "true") (has "logs" $port.enabled_for)) (and (eq (include "splunk-otel-collector.profilingEnabled" $) "true") (has "profiling" $port.enabled_for)) }}
+        {{- if eq true (and (eq (include "splunk-otel-collector.metricsEnabled" $) "true") (has "metrics" $port.enabled_for)) (and (eq (include "splunk-otel-collector.tracesEnabled" $) "true") (has "traces" $port.enabled_for)) (and (eq (include "splunk-otel-collector.logsEnabled" $) "true") (has "logs" $port.enabled_for)) (and (eq (include "splunk-otel-collector.profilingEnabled" $) "true") (has "profiling" $port.enabled_for)) }}
         - name: {{ $key }}
           {{- omit $port "enabled_for" | toYaml | trim | nindent 10 }}
         {{- end }}
diff --git a/helm-charts/splunk-otel-collector/values.schema.json b/helm-charts/splunk-otel-collector/values.schema.json
@@ -46,6 +46,9 @@
         "metricsIndex": {
           "type": "string"
         },
+        "tracesIndex": {
+          "type": "string"
+        },
         "source": {
           "type": "string"
         },
@@ -81,6 +84,10 @@
           "description": "Send Metrics to Splunk Platform",
           "type": "boolean"
         },
+        "tracesEnabled": {
+          "description": "Send Traces to Splunk Platform",
+          "type": "boolean"
+        },
         "fieldNameConvention": {
           "type": "object",
           "additionalProperties": false,
diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml
@@ -35,6 +35,8 @@ splunkPlatform:
   index: "main"
   # Name of the Splunk metric type index targeted. Required when ingesting metrics to Splunk Platform.
   metricsIndex: ""
+  # Name of the Splunk event type index targeted. Required when ingesting traces to Splunk Platform.
+  tracesIndex: ""
   # Optional. Default value for `source` field.
   source: "kubernetes"
   # Optional. Default value for `sourcetype` field. For container logs, it will
@@ -67,7 +69,8 @@ splunkPlatform:
   logsEnabled: true
   # If you enable metrics collection, make sure that `metricsIndex` is provided as well.
   metricsEnabled: false
-
+  # If you enable traces collection, make sure that `tracesIndex` is provided as well.
+  tracesEnabled: false
   # Field name conventions to use. (Only for those who are migrating from Splunk Connect for Kubernetes helm chart)
   fieldNameConvention:
     # Boolean for renaming pod metadata fields to match to Splunk Connect for Kubernetes helm chart.
