Merge branch 'main' into otl-3816-otlp-log-ingest

Author: akhileshsingh85

.chloggen/otl-3816-otlp-log-ingest.yaml

@@ -2,7 +2,7 @@
 change_type: enhancement
 
 # The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other)
-component: agent, gateway
+component: agent, gateway, clusterReceiver
 
 # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
 note: Add OTLP log ingest option to send logs to Splunk Connect for OTLP instead of HEC
@@ -25,10 +25,13 @@ subtext: |
       enabled: true
       endpoint: "<splunk-connect-for-otlp-endpoint>"
       protocol: grpc   # or "http"
-      insecure: true
+      insecure: false
   ```
 
-  When `otlpIngest.enabled` is true and `logsEnabled` is true, the agent and gateway
-  will use `otlp/platform_logs` (gRPC) or `otlp_http/platform_logs` (HTTP) exporters
-  instead of `splunk_hec/platform_logs`. The `splunkPlatform.endpoint` field is not
-  required when using OTLP ingest.
+  When `otlpIngest.enabled` is true and `logsEnabled` is true, the agent, gateway,
+  and cluster receiver log pipelines will use `otlp/platform_logs` (gRPC) or
+  `otlp_http/platform_logs` (HTTP) exporters instead of `splunk_hec/platform_logs`.
+  Use a base endpoint for HTTP, for example `http://splunk-host:4318`.
+  The `splunkPlatform.endpoint` field is not required for logs when using OTLP
+  ingest, but it is still required when `splunkPlatform.metricsEnabled` or
+  `splunkPlatform.tracesEnabled` sends metrics or traces to Splunk Platform via HEC.

helm-charts/splunk-otel-collector/templates/_helpers.tpl

@@ -36,14 +36,33 @@ Create chart name and version as used by the chart label.
 Whether to send data to Splunk Platform endpoint
 */}}
 {{- define "splunk-otel-collector.splunkPlatformEnabled" -}}
-{{- or (not (eq .Values.splunkPlatform.endpoint "")) .Values.splunkPlatform.otlpIngest.enabled }}
+{{- not (eq .Values.splunkPlatform.endpoint "") }}
 {{- end -}}
 
 {{/*
 Whether logs should be sent via OTLP to Splunk Connect for OTLP instead of HEC.
 */}}
 {{- define "splunk-otel-collector.platformLogsViaOtlpEnabled" -}}
-{{- and .Values.splunkPlatform.otlpIngest.enabled .Values.splunkPlatform.logsEnabled }}
+{{- and ((.Values.splunkPlatform.otlpIngest).enabled) .Values.splunkPlatform.logsEnabled }}
+{{- end -}}
+
+{{/*
+The exporter name for platform logs sent via OTLP (otlp/platform_logs or otlp_http/platform_logs).
+*/}}
+{{- define "splunk-otel-collector.otlpPlatformLogsExporterName" -}}
+{{- if eq .Values.splunkPlatform.otlpIngest.protocol "http" }}otlp_http{{- else }}otlp{{- end }}/platform_logs
+{{- end -}}
+
+{{/*
+Whether Splunk Platform HEC token is required.
+*/}}
+{{- define "splunk-otel-collector.platformHecTokenRequired" -}}
+{{- or
+      (and
+        (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true")
+        (not (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true")))
+      (eq (include "splunk-otel-collector.platformMetricsEnabled" .) "true")
+      (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
 {{- end -}}
 
 {{/*
@@ -113,7 +132,7 @@ Whether traces enabled for any destination.
 Whether logs enabled for any destination.
 */}}
 {{- define "splunk-otel-collector.logsEnabled" -}}
-{{- include "splunk-otel-collector.platformLogsEnabled" . }}
+{{- or (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
 {{- end -}}
 
 {{/*

helm-charts/splunk-otel-collector/templates/config/_common.tpl

@@ -333,6 +333,17 @@ resource/logs:
     {{- end }}
 {{- end }}
 
+{{/*
+Resource processor for Splunk Platform log defaults when logs are sent via OTLP.
+*/}}
+{{- define "splunk-otel-collector.otlpPlatformLogsResourceProcessor" -}}
+resource/otlp_platform_logs:
+  attributes:
+    - key: com.splunk.index
+      value: {{ .Values.splunkPlatform.index | quote }}
+      action: insert
+{{- end }}
+
 {{/*
 Resource processor for metrics manipulations
 */}}
@@ -389,7 +400,7 @@ Splunk Platform Logs exporter via OTLP (Splunk Connect for OTLP)
 {{- define "splunk-otel-collector.otlpPlatformLogsExporter" -}}
 {{- if eq .Values.splunkPlatform.otlpIngest.protocol "http" }}
 otlp_http/platform_logs:
-  logs_endpoint: {{ .Values.splunkPlatform.otlpIngest.endpoint | quote }}
+  endpoint: {{ .Values.splunkPlatform.otlpIngest.endpoint | quote }}
   tls:
     insecure: {{ .Values.splunkPlatform.otlpIngest.insecure }}
 {{- else }}
@@ -398,6 +409,28 @@ otlp/platform_logs:
   tls:
     insecure: {{ .Values.splunkPlatform.otlpIngest.insecure }}
 {{- end }}
+  retry_on_failure:
+    enabled: {{ .Values.splunkPlatform.retryOnFailure.enabled }}
+    initial_interval: {{ .Values.splunkPlatform.retryOnFailure.initialInterval }}
+    max_interval: {{ .Values.splunkPlatform.retryOnFailure.maxInterval }}
+    {{- if .Values.featureGates.noDropLogsPipeline }}
+    max_elapsed_time: 0s
+    {{- else }}
+    max_elapsed_time: {{ .Values.splunkPlatform.retryOnFailure.maxElapsedTime }}
+    {{- end }}
+  sending_queue:
+    enabled:  {{ .Values.splunkPlatform.sendingQueue.enabled }}
+    queue_size: {{ .Values.splunkPlatform.sendingQueue.queueSize }}
+    {{- if .addPersistentStorage }}
+    storage: file_storage/persistent_queue
+    {{- end }}
+    num_consumers: {{ .Values.splunkPlatform.sendingQueue.numConsumers }}
+    {{- if .Values.featureGates.noDropLogsPipeline }}
+    batch:
+      flush_timeout: 200ms
+      min_size: 2048
+      sizer: items
+    {{- end }}
 {{- end -}}
 
 {{/*

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -934,6 +934,10 @@ processors:
         key: k8s.namespace.name
         value: "${K8S_NAMESPACE}"
 
+  {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.otlpPlatformLogsResourceProcessor" . | nindent 2 }}
+  {{- end }}
+
   {{- if .Values.environment }}
   resource/add_environment:
     attributes:
@@ -1196,6 +1200,9 @@ service:
         - transform/istio_service_name
         {{- end }}
         - resource/logs
+        {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+        - resource/otlp_platform_logs
+        {{- end }}
         {{- end }}
         {{- if .Values.environment }}
         - resource/add_environment
@@ -1208,7 +1215,7 @@ service:
         - splunk_hec/o11y
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
-        - {{ if eq .Values.splunkPlatform.otlpIngest.protocol "http" }}otlp_http{{- else }}otlp{{- end }}/platform_logs
+        - {{ include "splunk-otel-collector.otlpPlatformLogsExporterName" . }}
         {{- else if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
         - splunk_hec/platform_logs
         {{- end }}
@@ -1237,6 +1244,9 @@ service:
         - resourcedetection/k8s_cluster_name
         {{- end }}
         - resource
+        {{- if and (not .Values.gateway.enabled) (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+        - resource/otlp_platform_logs
+        {{- end }}
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
@@ -1245,7 +1255,7 @@ service:
         - otlp_grpc
         {{- else }}
         {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
-        - {{ if eq .Values.splunkPlatform.otlpIngest.protocol "http" }}otlp_http{{- else }}otlp{{- end }}/platform_logs
+        - {{ include "splunk-otel-collector.otlpPlatformLogsExporterName" . }}
         {{- else if eq (include "splunk-otel-collector.platformLogsEnabled" .) "true" }}
         - splunk_hec/platform_logs
         {{- end }}

helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl

@@ -52,6 +52,9 @@ processors:
   {{- end }}
 
   {{- include "splunk-otel-collector.resourceLogsProcessor" . | nindent 2 }}
+  {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.otlpPlatformLogsResourceProcessor" . | nindent 2 }}
+  {{- end }}
   {{- if .Values.autodetect.istio }}
   {{- include "splunk-otel-collector.transformLogsProcessor" . | nindent 2 }}
   {{- end }}
@@ -327,12 +330,15 @@ service:
         - transform/istio_service_name
         {{- end }}
         - resource/logs
+        {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+        - resource/otlp_platform_logs
+        {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yProfilingEnabled" .) "true") }}
         - splunk_hec/o11y
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
-        - {{ if eq .Values.splunkPlatform.otlpIngest.protocol "http" }}otlp_http{{- else }}otlp{{- end }}/platform_logs
+        - {{ include "splunk-otel-collector.otlpPlatformLogsExporterName" . }}
         {{- else if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
         - splunk_hec/platform_logs
         {{- end }}

helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl

@@ -156,6 +156,10 @@ processors:
         value: {{ .Values.clusterName }}
   {{- end }}
 
+  {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.otlpPlatformLogsResourceProcessor" . | nindent 2 }}
+  {{- end }}
+
   {{- if eq (include "splunk-otel-collector.clusterReceiverEventsPipelineEnabled" $) "true" }}
 
   # Add k8s event attributes - k8s.<kind>.name and k8s.<kind>.uid
@@ -178,6 +182,14 @@ processors:
           - merge_maps(resource.cache, ExtractPatterns(resource.attributes["k8s.object.fieldpath"], "spec.containers\\{(?P<k8s_container_name>[^\\}]+)\\}"), "insert")
           - set(resource.attributes["k8s.container.name"], resource.cache["k8s_container_name"])
 
+  {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+  transform/add_events_sourcetype:
+    log_statements:
+      - context: log
+        statements:
+          - set(resource.attributes["com.splunk.sourcetype"], "kube:events")
+  {{- end }}
+
   # Drop high cardinality k8s event attributes
   attributes/drop_event_attrs:
     actions:
@@ -305,7 +317,9 @@ exporters:
   {{- include "splunk-otel-collector.splunkPlatformMetricsExporter" . | nindent 2 }}
   {{- end }}
 
-  {{- if and (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") (eq (include "splunk-otel-collector.objectsOrEventsEnabled" .) "true") }}
+  {{- if and (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") (eq (include "splunk-otel-collector.objectsOrEventsEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.otlpPlatformLogsExporter" . | nindent 2 }}
+  {{- else if and (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") (eq (include "splunk-otel-collector.objectsOrEventsEnabled" .) "true") }}
   {{- include "splunk-otel-collector.splunkPlatformLogsExporter" . | nindent 2 }}
   {{- if .Values.clusterReceiver.eventsEnabled }}
     sourcetype: kube:events
@@ -437,9 +451,15 @@ service:
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
+        {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+        - resource/otlp_platform_logs
+        - transform/add_events_sourcetype
+        {{- end }}
         - transform/k8sevents
       exporters:
-        {{- if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
+        {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+        - {{ include "splunk-otel-collector.otlpPlatformLogsExporterName" . }}
+        {{- else if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
         - splunk_hec/platform_logs
         {{- end }}
         {{- if eq (include "splunk-otel-collector.splunkO11yEventsEndpointEnabled" .) "true" }}
@@ -460,11 +480,16 @@ service:
         - resource
         - resource/add_cluster_host
         - transform/add_sourcetype
+        {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+        - resource/otlp_platform_logs
+        {{- end }}
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
       exporters:
-      {{- if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
+      {{- if (eq (include "splunk-otel-collector.platformLogsViaOtlpEnabled" .) "true") }}
+        - {{ include "splunk-otel-collector.otlpPlatformLogsExporterName" . }}
+      {{- else if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
         - splunk_hec/platform_logs
       {{- end }}
       {{- if (eq (include "splunk-otel-collector.splunkO11yEventsEndpointEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/daemonset.yaml

@@ -207,7 +207,7 @@ spec:
                 name: {{ include "splunk-otel-collector.secret" . }}
                 key: splunk_observability_access_token
           {{- end }}
-          {{- if (eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true") }}
+          {{- if (eq (include "splunk-otel-collector.platformHecTokenRequired" .) "true") }}
           - name: SPLUNK_PLATFORM_HEC_TOKEN
             valueFrom:
               secretKeyRef:

helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml

@@ -171,7 +171,7 @@ spec:
                 name: {{ include "splunk-otel-collector.secret" . }}
                 key: splunk_observability_access_token
           {{- end }}
-          {{- if (eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true") }}
+          {{- if (eq (include "splunk-otel-collector.platformHecTokenRequired" .) "true") }}
           - name: SPLUNK_PLATFORM_HEC_TOKEN
             valueFrom:
               secretKeyRef:

helm-charts/splunk-otel-collector/templates/deployment-gateway.yaml

@@ -124,7 +124,7 @@ spec:
                 name: {{ include "splunk-otel-collector.secret" . }}
                 key: splunk_observability_access_token
           {{- end }}
-          {{- if (eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true") }}
+          {{- if (eq (include "splunk-otel-collector.platformHecTokenRequired" .) "true") }}
           - name: SPLUNK_PLATFORM_HEC_TOKEN
             valueFrom:
               secretKeyRef:

helm-charts/splunk-otel-collector/templates/secret-splunk-validation-hook.yaml

@@ -25,7 +25,7 @@ spec:
           doesn\'t have required field \"splunk_observability_access_token\".;
           export TOKEN_INVALID=true;
         fi;
-        if [ "{{ include "splunk-otel-collector.splunkPlatformEnabled" . }}" = "true" ] && [ ! -f /otel/secret/splunk_platform_hec_token ]; then
+        if [ "{{ include "splunk-otel-collector.platformHecTokenRequired" . }}" = "true" ] && [ ! -f /otel/secret/splunk_platform_hec_token ]; then
           echo Splunk Platform destination is enabled, but custom Kubernetes\
           secret \"{{ template "splunk-otel-collector.secret" . }}\" doesn\'t\
           have required field \"splunk_platform_hec_token\".;