perf: optimize FileMetaReader hot path and EncoderReader map sizing (#50020)

FileMetaReader.Next() is called for every line harvested by filebeat.

Replace DeepUpdate (recursive map merge) and dotted-key Put
("log.file.device_id") with direct map assignment. Lazy-cache the
per-file metadata (path, device_id, inode, fingerprint) on first call
since these values are constant for the file's lifetime.

The cached interface{} values are copied via maps.Copy without
re-boxing, which is where the alloc savings come from. The cache is
built into a local variable and only assigned on success, so a failed
setFileSystemMetadata retries on the next call.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* perf: pre-size Fields map in EncoderReader to avoid grow on first write

EncoderReader.Next() creates a Fields map for every line. Changing
from mapstr.M{} (capacity 0) to make(mapstr.M, 1) avoids a runtime
map grow when the first downstream consumer writes into it. Benefits
all inputs using EncoderReader: filestream, legacy log, and S3.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix: add changelog fragment and resolve lint issues

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* chore: remove changelog fragment, using skip-changelog label

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* test: use require.IsType and consolidate benchmark into metafields_test.go

Replace bare type assertion + require.True(ok) patterns with require.IsType
in metafields_other_test.go and metafields_windows_test.go. Move benchmark
functions from metafields_bench_test.go into metafields_test.go and delete
the separate bench file, following the one-test-file-per-processor principle.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix: use comma-ok type assertions to satisfy errcheck linter

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* perf: exact cachedMeta pre-sizing via platformFileFields constant

Add a platformFileFields constant to each platform file (2 on non-Windows,
3 on Windows) so metafields.go can compute the exact map capacity without
over-allocating. Add TestCachedMetaSizing on both platforms to verify the
pre-allocation matches the actual entry count. Also document that the direct
log key assignment is intentional, and fix TestMetaFieldsOwnerAndGroup to
use the platform-correct group name (wheel on macOS, root on Linux).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix testifylint: use require.Len instead of len() comparison

The linter requires require.Len(t, x, n) instead of
require.Equal(t, n, len(x)).

Assisted-By: Cursor
Made-with: Cursor

* fix testifylint: use require.Len in windows test file

Same fix as previous commit, but for the Windows-specific test.

Assisted-By: Cursor
Made-with: Cursor

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: strawgate

libbeat/reader/readfile/encode.go

@@ -69,7 +69,7 @@ func (r EncoderReader) Next() (reader.Message, error) {
 		Ts:      time.Now(),
 		Content: bytes.TrimPrefix(c, []byte("\uFEFF")),
 		Bytes:   sz,
-		Fields:  mapstr.M{},
+		Fields:  make(mapstr.M, 1),
 	}, err
 }
 

libbeat/reader/readfile/fs_metafields_other.go

@@ -28,44 +28,38 @@ import (
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
+// platformFileFields is the number of fields setFileSystemMetadata always writes
+// into the log.file submap on this platform (device_id and inode).
+// Optional fields (owner, group) are counted separately in metafields.go.
+const platformFileFields = 2
+
+// Keys written into the log.file submap by setFileSystemMetadata.
 const (
-	deviceIDKey = "log.file.device_id"
-	inodeKey    = "log.file.inode"
-	ownerKey    = "log.file.owner"
-	groupKey    = "log.file.group"
+	deviceIDKey = "device_id"
+	inodeKey    = "inode"
+	ownerKey    = "owner"
+	groupKey    = "group"
 )
 
-func setFileSystemMetadata(fi file.ExtendedFileInfo, fields mapstr.M, includeOwner bool, includeGroup bool) error {
+func setFileSystemMetadata(fi file.ExtendedFileInfo, fileMap mapstr.M, includeOwner bool, includeGroup bool) error {
 	osstate := fi.GetOSState()
-	_, err := fields.Put(deviceIDKey, strconv.FormatUint(osstate.Device, 10))
-	if err != nil {
-		return fmt.Errorf("failed to set %q: %w", deviceIDKey, err)
-	}
-	_, err = fields.Put(inodeKey, osstate.InodeString())
-	if err != nil {
-		return fmt.Errorf("failed to set %q: %w", inodeKey, err)
-	}
+	fileMap[deviceIDKey] = strconv.FormatUint(osstate.Device, 10)
+	fileMap[inodeKey] = osstate.InodeString()
 
 	if includeOwner {
 		o, err := user.LookupId(strconv.FormatUint(osstate.UID, 10))
 		if err != nil {
 			return fmt.Errorf("failed to lookup uid %q: %w", osstate.UID, err)
 		}
-		_, err = fields.Put(ownerKey, o.Username)
-		if err != nil {
-			return fmt.Errorf("failed to set %q: %w", ownerKey, err)
-		}
+		fileMap[ownerKey] = o.Username
 	}
 
 	if includeGroup {
 		g, err := user.LookupGroupId(strconv.FormatUint(osstate.GID, 10))
 		if err != nil {
 			return fmt.Errorf("failed to lookup gid %q: %w", osstate.GID, err)
 		}
-		_, err = fields.Put(groupKey, g.Name)
-		if err != nil {
-			return fmt.Errorf("failed to set %q: %w", groupKey, err)
-		}
+		fileMap[groupKey] = g.Name
 	}
 	return nil
 }

libbeat/reader/readfile/fs_metafields_windows.go

@@ -18,33 +18,27 @@
 package readfile
 
 import (
-	"fmt"
 	"strconv"
 
 	"github.com/elastic/beats/v7/libbeat/common/file"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
+// platformFileFields is the number of fields setFileSystemMetadata always writes
+// into the log.file submap on this platform (idxhi, idxlo, and vol).
+const platformFileFields = 3
+
+// Keys written into the log.file submap by setFileSystemMetadata.
 const (
-	idxhiKey = "log.file.idxhi"
-	idxloKey = "log.file.idxlo"
-	volKey   = "log.file.vol"
+	idxhiKey = "idxhi"
+	idxloKey = "idxlo"
+	volKey   = "vol"
 )
 
-func setFileSystemMetadata(fi file.ExtendedFileInfo, fields mapstr.M, includeOwner bool, includeGroup bool) error {
+func setFileSystemMetadata(fi file.ExtendedFileInfo, fileMap mapstr.M, includeOwner bool, includeGroup bool) error {
 	osstate := fi.GetOSState()
-	_, err := fields.Put(idxhiKey, strconv.FormatUint(osstate.IdxHi, 10))
-	if err != nil {
-		return fmt.Errorf("failed to set %q: %w", idxhiKey, err)
-	}
-	_, err = fields.Put(idxloKey, strconv.FormatUint(osstate.IdxLo, 10))
-	if err != nil {
-		return fmt.Errorf("failed to set %q: %w", idxloKey, err)
-	}
-	_, err = fields.Put(volKey, strconv.FormatUint(osstate.Vol, 10))
-	if err != nil {
-		return fmt.Errorf("failed to set %q: %w", volKey, err)
-	}
-
+	fileMap[idxhiKey] = strconv.FormatUint(osstate.IdxHi, 10)
+	fileMap[idxloKey] = strconv.FormatUint(osstate.IdxLo, 10)
+	fileMap[volKey] = strconv.FormatUint(osstate.Vol, 10)
 	return nil
 }

libbeat/reader/readfile/metafields.go

@@ -19,14 +19,16 @@ package readfile
 
 import (
 	"fmt"
+	"maps"
 
 	"github.com/elastic/beats/v7/libbeat/common/file"
 	"github.com/elastic/beats/v7/libbeat/reader"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
-// Reader produces lines by reading lines from an io.Reader
-// through a decoder converting the reader it's encoding to utf-8.
+// FileMetaReader enriches every message with per-file metadata.
+// OS metadata strings (device_id, inode, etc.) are cached after the
+// first call to Next because they are constant for the file's lifetime.
 type FileMetaReader struct {
 	reader       reader.Reader
 	path         string
@@ -35,12 +37,21 @@ type FileMetaReader struct {
 	includeGroup bool
 	fingerprint  string
 	offset       int64
+	cachedMeta   mapstr.M // lazily populated on first Next()
 }
 
 // New creates a new Encode reader from input reader by applying
 // the given codec.
 func NewFilemeta(r reader.Reader, path string, fi file.ExtendedFileInfo, includeOwner bool, includeGroup bool, fingerprint string, offset int64) reader.Reader {
-	return &FileMetaReader{r, path, fi, includeOwner, includeGroup, fingerprint, offset}
+	return &FileMetaReader{
+		reader:       r,
+		path:         path,
+		fi:           fi,
+		includeOwner: includeOwner,
+		includeGroup: includeGroup,
+		fingerprint:  fingerprint,
+		offset:       offset,
+	}
 }
 
 // Next reads the next line from it's initial io.Reader
@@ -54,25 +65,47 @@ func (r *FileMetaReader) Next() (reader.Message, error) {
 		return message, err
 	}
 
-	message.Fields.DeepUpdate(mapstr.M{
-		"log": mapstr.M{
-			"offset": r.offset,
-			"file": mapstr.M{
-				"path": r.path,
-			},
-		},
-	})
-
-	err = setFileSystemMetadata(r.fi, message.Fields, r.includeOwner, r.includeGroup)
-	if err != nil {
-		return message, fmt.Errorf("failed to set file system metadata: %w", err)
+	// On first call, compute and cache the per-file OS metadata (device_id,
+	// inode, etc.) since they are constant for this file's lifetime.
+	// Build into a local variable so a failed setFileSystemMetadata does not
+	// leave r.cachedMeta in a partial state.
+	if r.cachedMeta == nil {
+		// Pre-size exactly: path + platform-invariant fields + optional fields.
+		// platformFileFields is defined per-platform in fs_metafields_*.go.
+		// On Windows, includeOwner/includeGroup are not supported so they add
+		// no fields; the slight over-allocation is harmless.
+		size := 1 + platformFileFields // path + platform fields
+		if r.includeOwner {
+			size++
+		}
+		if r.includeGroup {
+			size++
+		}
+		if r.fingerprint != "" {
+			size++
+		}
+		m := make(mapstr.M, size)
+		m["path"] = r.path
+		if err := setFileSystemMetadata(r.fi, m, r.includeOwner, r.includeGroup); err != nil {
+			return message, fmt.Errorf("failed to set file system metadata: %w", err)
+		}
+		if r.fingerprint != "" {
+			m["fingerprint"] = r.fingerprint
+		}
+		r.cachedMeta = m
 	}
 
-	if r.fingerprint != "" {
-		_, err = message.Fields.Put("log.file.fingerprint", r.fingerprint)
-		if err != nil {
-			return message, fmt.Errorf("failed to set fingerprint: %w", err)
-		}
+	// Copy cached fields into a fresh map for this event.
+	fileMap := make(mapstr.M, len(r.cachedMeta))
+	maps.Copy(fileMap, r.cachedMeta)
+	// Direct assignment replaces any existing "log" key rather than merging.
+	// This is intentional and safe: FileMetaReader is always the first component
+	// in the pipeline to write log.* fields. Downstream readers (parsers,
+	// LimitReader) run after this and merge into the map we write here via
+	// AddFields() / Update(), so nothing is lost.
+	message.Fields["log"] = mapstr.M{
+		"offset": r.offset,
+		"file":   fileMap,
 	}
 	r.offset += int64(message.Bytes)
 

libbeat/reader/readfile/metafields_other_test.go

@@ -20,13 +20,15 @@
 package readfile
 
 import (
+	"runtime"
 	"syscall"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/common/file"
+	"github.com/elastic/beats/v7/libbeat/reader"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
@@ -42,53 +44,87 @@ func createTestFileInfo() file.ExtendedFileInfo {
 func checkFields(t *testing.T, expected, actual mapstr.M) {
 	t.Helper()
 
-	dev, err := actual.GetValue(deviceIDKey)
-	require.NoError(t, err)
-	require.Equal(t, "17", dev)
-	err = actual.Delete(deviceIDKey)
-	require.NoError(t, err)
+	require.IsType(t, mapstr.M{}, actual["log"], "expected log to be mapstr.M")
+	logMap, _ := actual["log"].(mapstr.M)
+	require.IsType(t, mapstr.M{}, logMap["file"], "expected log.file to be mapstr.M")
+	fileMap, _ := logMap["file"].(mapstr.M)
 
-	inode, err := actual.GetValue(inodeKey)
-	require.NoError(t, err)
-	require.Equal(t, "999", inode)
-	err = actual.Delete(inodeKey)
-	require.NoError(t, err)
+	require.Equal(t, "17", fileMap[deviceIDKey])
+	delete(fileMap, deviceIDKey)
+	require.Equal(t, "999", fileMap[inodeKey])
+	delete(fileMap, inodeKey)
 
-	_, err = actual.GetValue(ownerKey)
-	require.Error(t, err)
-
-	_, err = actual.GetValue(groupKey)
-	require.Error(t, err)
+	_, hasOwner := fileMap[ownerKey]
+	require.False(t, hasOwner)
+	_, hasGroup := fileMap[groupKey]
+	require.False(t, hasGroup)
 
 	require.Equal(t, expected, actual)
 }
 
+// TestCachedMetaSizing verifies that cachedMeta ends up with exactly the right
+// number of entries for each combination of optional fields on this platform.
+// Because make(mapstr.M, size) pre-allocates exactly `size` slots, matching
+// len(cachedMeta) to the formula proves no map growth occurred.
+func TestCachedMetaSizing(t *testing.T) {
+	fi := createTestFileInfo()
+	msg := reader.Message{Content: []byte("line"), Bytes: 4, Fields: mapstr.M{}}
+
+	tests := []struct {
+		name         string
+		includeOwner bool
+		includeGroup bool
+		fingerprint  string
+		wantLen      int
+	}{
+		{"base only", false, false, "", 1 + platformFileFields},
+		{"with owner", true, false, "", 1 + platformFileFields + 1},
+		{"with group", false, true, "", 1 + platformFileFields + 1},
+		{"with owner and group", true, true, "", 1 + platformFileFields + 2},
+		{"with fingerprint", false, false, "hash", 1 + platformFileFields + 1},
+		{"all fields", true, true, "hash", 1 + platformFileFields + 3},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			r := &FileMetaReader{
+				reader:       msgReader([]reader.Message{msg}),
+				path:         "test/path",
+				fi:           fi,
+				includeOwner: tc.includeOwner,
+				includeGroup: tc.includeGroup,
+				fingerprint:  tc.fingerprint,
+			}
+			_, err := r.Next()
+			require.NoError(t, err)
+			require.Len(t, r.cachedMeta, tc.wantLen,
+				"cachedMeta entry count should match the pre-allocated size")
+		})
+	}
+}
+
 func checkFieldsWithOwnerGroup(t *testing.T, expected, actual mapstr.M) {
 	t.Helper()
 
-	dev, err := actual.GetValue(deviceIDKey)
-	require.NoError(t, err)
-	require.Equal(t, "17", dev)
-	err = actual.Delete(deviceIDKey)
-	require.NoError(t, err)
-
-	inode, err := actual.GetValue(inodeKey)
-	require.NoError(t, err)
-	require.Equal(t, "999", inode)
-	err = actual.Delete(inodeKey)
-	require.NoError(t, err)
-
-	o, err := actual.GetValue(ownerKey)
-	require.NoError(t, err)
-	require.Equal(t, "root", o)
-	err = actual.Delete(ownerKey)
-	require.NoError(t, err)
-
-	g, err := actual.GetValue(groupKey)
-	require.NoError(t, err)
-	require.Equal(t, "root", g)
-	err = actual.Delete(groupKey)
-	require.NoError(t, err)
+	require.IsType(t, mapstr.M{}, actual["log"], "expected log to be mapstr.M")
+	logMap, _ := actual["log"].(mapstr.M)
+	require.IsType(t, mapstr.M{}, logMap["file"], "expected log.file to be mapstr.M")
+	fileMap, _ := logMap["file"].(mapstr.M)
+
+	require.Equal(t, "17", fileMap[deviceIDKey])
+	delete(fileMap, deviceIDKey)
+	require.Equal(t, "999", fileMap[inodeKey])
+	delete(fileMap, inodeKey)
+
+	// macOS uses "wheel" for GID 0; Linux uses "root".
+	expectedGroup := "root"
+	if runtime.GOOS == "darwin" {
+		expectedGroup = "wheel"
+	}
+	require.Equal(t, "root", fileMap[ownerKey])
+	delete(fileMap, ownerKey)
+	require.Equal(t, expectedGroup, fileMap[groupKey])
+	delete(fileMap, groupKey)
 
 	require.Equal(t, expected, actual)
 }