feat: add add_kubernetes_metadata processor to OTel Beat processor (#47942) (#47981)

* feat: add `add_kubernetes_metadata` processor to OTel Beat processor

* test: use test Logger in tests

* refactor: fix typo in function name

* test: revert to no-op logger

(cherry picked from commit 3a9d1f4)

Co-authored-by: Andrzej Stencel <[email protected]>

Author: mergify[bot]

x-pack/otel/processor/beatprocessor/README.md

@@ -6,9 +6,6 @@
 
 [development]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#development
 
-> [!NOTE]
-> This component is currently in development and functionality is limited.
-
 The Beat processor (`beat`) is an OpenTelemetry Collector processor that wraps the [Beat processors].
 This allows you to use Beat processors like e.g. [add_host_metadata] anywhere in the OpenTelemetry Collector's pipeline, independently of Beat receivers.
 
@@ -17,11 +14,10 @@ This allows you to use Beat processors like e.g. [add_host_metadata] anywhere in
 > This is because it relies on the specific structure of telemetry emitted by those components.
 > Using it with data coming from other components is not recommended and may result in unexpected behavior.
 
-The processor enriches the telemetry with host metadata by using the [add_host_metadata] processor under the hood.
-Note that configuration is limited at this stage.
-Host metadata is added unconditionally and cannot be disabled.
-You can configure the host metadata enrichment using the options that the [add_host_metadata] processor allows.
-The only exception is that the option `replace_fields` is always set to `true` and setting it to `false` has no effect.
+Here are the currently supported processors:
+
+- [add_host_metadata]
+- [add_kubernetes_metadata]
 
 ## Default processors in Beat receivers
 
@@ -61,11 +57,17 @@ receivers:
 The above Filebeat receiver configuration specifies an empty list of processors.
 In this case, none of the default processors are ran as part of the Filebeat receiver.
 
-## Example
+## Examples
 
-The following [Filebeat receiver] configuration
+The following OpenTelemetry Collector configuration using only the [Filebeat receiver]:
 
 ```yaml
+service:
+  pipelines:
+    logs:
+      receivers: [filebeatreceiver]
+      exporters: [debug]
+
 receivers:
   filebeatreceiver:
     filebeat:
@@ -78,11 +80,21 @@ receivers:
       - add_host_metadata:
           netinfo:
             enabled: false
+
+exporters:
+  debug:
 ```
 
 is functionally equivalent to this one, using the Beat processor:
 
 ```yaml
+service:
+  pipelines:
+    logs:
+      receivers: [filebeatreceiver]
+      processors: [beat]
+      exporters: [debug]
+
 receivers:
   filebeatreceiver:
     filebeat:
@@ -99,12 +111,50 @@ processors:
       - add_host_metadata:
           netinfo:
             enabled: false
+
+exporters:
+  debug:
 ```
 
+## Using the `add_host_metadata` processor
+
+To use the [add_host_metadata] processor, configure the processor as follows:
+
+```yaml
+processors:
+  beat:
+    processors:
+      - add_host_metadata:
+```
+
+You can configure the host metadata enrichment using the options supported by the [add_host_metadata] processor.
+
+## Using the `add_kubernetes_metadata` processor
+
+To use the [add_kubernetes_metadata] processor, configure the processor as follows:
+
+```yaml
+processors:
+  beat:
+    processors:
+      - add_kubernetes_metadata:
+          indexers:
+            - container:
+          matchers:
+            - logs_path:
+```
+
+You can configure the Kubernetes metadata enrichment using the options supported by the [add_kubernetes_metadata] processor.
+
+Note that you need to explicitly configure at least one [indexer][indexers] and at least one [matcher][matchers] for the enrichment to work.
+In the example above, the `container` indexer and the `logs_path` matcher are configured.
+
 [Beat processors]: https://www.elastic.co/docs/reference/beats/filebeat/filtering-enhancing-data#using-processors
 [Filebeat receiver]: https://github.com/elastic/beats/tree/main/x-pack/filebeat/fbreceiver
 [Metricbeat receiver]: https://github.com/elastic/beats/tree/main/x-pack/metricbeat/mbreceiver
 [add_cloud_metadata]: https://www.elastic.co/docs/reference/beats/filebeat/add-cloud-metadata
 [add_docker_metadata]: https://www.elastic.co/docs/reference/beats/filebeat/add-docker-metadata
 [add_host_metadata]: https://www.elastic.co/docs/reference/beats/filebeat/add-host-metadata
 [add_kubernetes_metadata]: https://www.elastic.co/docs/reference/beats/filebeat/add-kubernetes-metadata
+[indexers]: https://www.elastic.co/docs/reference/beats/filebeat/add-kubernetes-metadata#_indexers
+[matchers]: https://www.elastic.co/docs/reference/beats/filebeat/add-kubernetes-metadata#_matchers

x-pack/otel/processor/beatprocessor/processor.go

@@ -10,12 +10,13 @@ import (
 	"fmt"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/otelbeat/otelmap"
 	"github.com/elastic/beats/v7/libbeat/processors/add_host_metadata"
+	"github.com/elastic/beats/v7/libbeat/processors/add_kubernetes_metadata"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 
-	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/plog"
 	"go.opentelemetry.io/collector/processor"
 	"go.uber.org/zap"
@@ -32,20 +33,30 @@ func newBeatProcessor(set processor.Settings, cfg *Config) (*beatProcessor, erro
 		processors: []beat.Processor{},
 	}
 
+	logpLogger, err := logp.ConfigureWithCoreLocal(logp.Config{}, set.Logger.Core())
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure logp logger: %w", err)
+	}
+
 	for _, processorConfig := range cfg.Processors {
-		processor, err := createProcessor(processorConfig)
+		processor, err := createProcessor(processorConfig, logpLogger)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create processor: %w", err)
 		}
 		if processor != nil {
 			bp.processors = append(bp.processors, processor)
+			bp.logger.Info("Configured Beat processor", zap.String("processor_name", processor.String()))
 		}
 	}
 
 	return bp, nil
 }
 
-func createProcessor(cfg map[string]any) (beat.Processor, error) {
+// createProcessor creates a Beat processor using the provided configuration.
+// The configuration is expected to be a map with a single key containing the processor name
+// and the processor's configuration as the value for that key.
+// For example: {"add_host_metadata":{"netinfo":{"enabled":false}}}
+func createProcessor(cfg map[string]any, logpLogger *logp.Logger) (beat.Processor, error) {
 	if len(cfg) == 0 {
 		return nil, nil
 	}
@@ -59,58 +70,70 @@ func createProcessor(cfg map[string]any) (beat.Processor, error) {
 		}
 		return nil, fmt.Errorf("expected single processor name but got %v", len(cfg))
 	}
+
 	for processorName, processorConfig := range cfg {
 		switch processorName {
 		case "add_host_metadata":
-			return createAddHostMetadataProcessor(processorConfig)
+			return createAddHostMetadataProcessor(processorConfig, logpLogger)
+		case "add_kubernetes_metadata":
+			return createAddKubernetesMetadataProcessor(processorConfig, logpLogger)
 		default:
 			return nil, fmt.Errorf("invalid processor name '%s'", processorName)
 		}
 	}
 	return nil, errors.New("malformed processor config")
 }
 
-func createAddHostMetadataProcessor(cfg any) (beat.Processor, error) {
+func createAddHostMetadataProcessor(cfg any, logpLogger *logp.Logger) (beat.Processor, error) {
 	addHostMetadataConfig, err := config.NewConfigFrom(cfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create add_host_metadata processor config: %w", err)
 	}
-	addHostMetadataProcessor, err := add_host_metadata.New(addHostMetadataConfig, logp.NewLogger("beatprocessor"))
+	addHostMetadataProcessor, err := add_host_metadata.New(addHostMetadataConfig, logpLogger)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create add_host_metadata processor: %w", err)
 	}
 	return addHostMetadataProcessor, nil
 }
 
+func createAddKubernetesMetadataProcessor(cfg any, logpLogger *logp.Logger) (beat.Processor, error) {
+	addKubernetesMetadataConfig, err := config.NewConfigFrom(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create add_kubernetes_metadata processor config: %w", err)
+	}
+	addKubernetesMetadataProcessor, err := add_kubernetes_metadata.New(addKubernetesMetadataConfig, logpLogger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create add_kubernetes_metadata processor: %w", err)
+	}
+	return addKubernetesMetadataProcessor, nil
+}
+
 func (p *beatProcessor) ConsumeLogs(_ context.Context, logs plog.Logs) (plog.Logs, error) {
 	if len(p.processors) == 0 {
 		return logs, nil
 	}
 
-	for _, hostProcessor := range p.processors {
-		dummyEvent := &beat.Event{}
-		dummyEvent.Fields = mapstr.M{}
-		dummyEvent.Meta = mapstr.M{}
-		dummyEventWithHostMetadata, err := hostProcessor.Run(dummyEvent)
-		if err != nil {
-			p.logger.Error("error processing host metadata", zap.Error(err))
-			continue
-		}
-		hostMap, ok := dummyEventWithHostMetadata.Fields["host"].(mapstr.M)
-		if !ok {
-			p.logger.Error("error casting host metadata to mapstr.M", zap.Error(err))
-			continue
-		}
-		otelMap, err := toOtelMap(&hostMap)
-		if err != nil {
-			p.logger.Error("error converting host metadata", zap.Error(err))
-			continue
-		}
-		for _, resourceLogs := range logs.ResourceLogs().All() {
-			for _, scopeLogs := range resourceLogs.ScopeLogs().All() {
-				for _, logRecord := range scopeLogs.LogRecords().All() {
-					bodyMap := logRecord.Body().Map().PutEmptyMap("host")
-					otelMap.CopyTo(bodyMap)
+	for _, resourceLogs := range logs.ResourceLogs().All() {
+		for _, scopeLogs := range resourceLogs.ScopeLogs().All() {
+			for _, logRecord := range scopeLogs.LogRecords().All() {
+				beatEvent, err := unpackBeatEventFromOTelLogRecord(logRecord)
+				if err != nil {
+					p.logger.Error("error converting OTel log to Beat event", zap.Error(err))
+					continue
+				}
+
+				for _, processor := range p.processors {
+					processedEvent, err := processor.Run(beatEvent)
+					if err != nil {
+						p.logger.Error("error processing Beat event", zap.Error(err))
+						continue
+					}
+					beatEvent = processedEvent
+				}
+
+				packingError := packBeatEventIntoOTelLogRecord(beatEvent, logRecord)
+				if packingError != nil {
+					p.logger.Error("error converting processed Beat event to OTel log record", zap.Error(packingError))
 				}
 			}
 		}
@@ -119,29 +142,20 @@ func (p *beatProcessor) ConsumeLogs(_ context.Context, logs plog.Logs) (plog.Log
 	return logs, nil
 }
 
-func toOtelMap(m *mapstr.M) (pcommon.Map, error) {
-	otelMap := pcommon.NewMap()
-	for key, value := range *m {
-		switch typedValue := value.(type) {
-		case mapstr.M:
-			subMap, err := toOtelMap(&typedValue)
-			if err != nil {
-				return pcommon.Map{}, fmt.Errorf("failed to convert map for key '%s': %w", key, err)
-			}
-			otelSubMap := otelMap.PutEmptyMap(key)
-			subMap.MoveTo(otelSubMap)
-		case []string:
-			otelValue := otelMap.PutEmptySlice(key)
-			for _, item := range typedValue {
-				otelValue.AppendEmpty().SetStr(item)
-			}
-		default:
-			otelValue := otelMap.PutEmpty(key)
-			err := otelValue.FromRaw(typedValue)
-			if err != nil {
-				return pcommon.Map{}, fmt.Errorf("failed to convert value for key '%s': %w", key, err)
-			}
-		}
-	}
-	return otelMap, nil
+func unpackBeatEventFromOTelLogRecord(logRecord plog.LogRecord) (*beat.Event, error) {
+	beatEvent := &beat.Event{}
+	beatEvent.Timestamp = logRecord.Timestamp().AsTime()
+
+	beatEvent.Meta = mapstr.M{}
+
+	beatEvent.Fields = logRecord.Body().Map().AsRaw()
+
+	return beatEvent, nil
+}
+
+func packBeatEventIntoOTelLogRecord(beatEvent *beat.Event, logRecord plog.LogRecord) error {
+	beatEvent.Fields = beatEvent.Fields.Clone()
+	otelmap.ConvertNonPrimitive((map[string]any)(beatEvent.Fields))
+	err := logRecord.Body().Map().FromRaw(beatEvent.Fields)
+	return err
 }

x-pack/otel/processor/beatprocessor/processor_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 
 	"github.com/stretchr/testify/assert"
@@ -65,15 +66,19 @@ func TestConsumeLogs(t *testing.T) {
 	}
 }
 
+func testLogger() *logp.Logger {
+	return logp.NewNopLogger()
+}
+
 func TestCreateProcessor(t *testing.T) {
 	t.Run("nil config returns nil processor", func(t *testing.T) {
-		processor, err := createProcessor(nil)
+		processor, err := createProcessor(nil, testLogger())
 		require.NoError(t, err)
 		assert.Nil(t, processor)
 	})
 
 	t.Run("empty config returns nil processor", func(t *testing.T) {
-		processor, err := createProcessor(map[string]any{})
+		processor, err := createProcessor(map[string]any{}, testLogger())
 		require.NoError(t, err)
 		assert.Nil(t, processor)
 	})
@@ -82,27 +87,36 @@ func TestCreateProcessor(t *testing.T) {
 		_, err := createProcessor(map[string]any{
 			"add_host_metadata": map[string]any{},
 			"another_key":       map[string]any{},
-		})
+		}, testLogger())
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "expected single processor name")
 	})
 
 	t.Run("unknown processor returns error", func(t *testing.T) {
 		_, err := createProcessor(map[string]any{
 			"unknown_processor": map[string]any{},
-		})
+		}, testLogger())
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "invalid processor name 'unknown_processor'")
 	})
 
 	t.Run("valid add_host_metadata processor config returns processor", func(t *testing.T) {
 		processor, err := createProcessor(map[string]any{
 			"add_host_metadata": map[string]any{},
-		})
+		}, testLogger())
 		require.NoError(t, err)
 		require.NotNil(t, processor)
 		assert.Equal(t, "add_host_metadata", processor.String()[:len("add_host_metadata")])
 	})
+
+	t.Run("valid add_kubernetes_metadata processor config returns processor", func(t *testing.T) {
+		processor, err := createProcessor(map[string]any{
+			"add_kubernetes_metadata": map[string]any{},
+		}, testLogger())
+		require.NoError(t, err)
+		require.NotNil(t, processor)
+		assert.Equal(t, "add_kubernetes_metadata", processor.String()[:len("add_kubernetes_metadata")])
+	})
 }
 
 type mockProcessor struct {