filestream: reduce per-scan allocations (#51863) (#51877)

Two per-file, per-scan allocations are removed:

  - fileWatcher.watch resolved the file identity for every file up
    front. It is now resolved lazily on first use.

  - fileScanner.GetFiles rebuilt its per-scan maps from empty, forcing
    repeated grow/rehash allocations on a stable file set. They are now
    pre-sized from the previous scan's unique-file count.

Combined effect vs main (go test -bench
'BenchmarkGetFiles|BenchmarkWatchIdle' -benchmem -benchtime=200ms
-count=10):

                                             │     main     │             this-commit              │
                                             │    sec/op    │    sec/op     vs base                │
GetFiles-14                                    1.660m ± 16%   1.302m ±  5%  -21.55% (p=0.000 n=10)
GetFilesWithFingerprint-14                     5.657m ±  9%   4.898m ± 15%  -13.42% (p=0.019 n=10)
GetFilesWithFingerprintGrowing/static-14       5.026m ± 14%   4.449m ±  6%  -11.49% (p=0.001 n=10)
GetFilesWithFingerprintGrowing/growing-14      6.823m ± 11%   7.775m ± 19%  +13.96% (p=0.023 n=10)
WatchIdle/active/path/100_files-14             160.6µ ± 12%   155.9µ ±  6%        ~ (p=0.280 n=10)
WatchIdle/active/path/1000_files-14            1.938m ± 12%   1.805m ± 21%        ~ (p=0.218 n=10)
WatchIdle/active/path/10000_files-14           27.05m ± 12%   24.65m ± 14%   -8.87% (p=0.009 n=10)
WatchIdle/active/fingerprint/100_files-14      439.9µ ±  2%   413.3µ ±  3%   -6.04% (p=0.004 n=10)
WatchIdle/active/fingerprint/1000_files-14     5.352m ± 11%   5.300m ± 20%        ~ (p=0.853 n=10)
WatchIdle/active/fingerprint/10000_files-14    58.20m ±  6%   59.59m ± 12%        ~ (p=0.190 n=10)
WatchIdle/ignored/path/100_files-14            170.0µ ±  9%   140.3µ ± 10%  -17.47% (p=0.000 n=10)
WatchIdle/ignored/path/1000_files-14           2.270m ±  9%   1.652m ±  1%  -27.23% (p=0.000 n=10)
WatchIdle/ignored/path/10000_files-14          25.86m ± 12%   25.09m ± 10%        ~ (p=0.218 n=10)
WatchIdle/ignored/fingerprint/100_files-14     428.7µ ± 14%   398.9µ ±  7%        ~ (p=0.218 n=10)
WatchIdle/ignored/fingerprint/1000_files-14    5.107m ± 10%   5.325m ± 15%        ~ (p=0.971 n=10)
WatchIdle/ignored/fingerprint/10000_files-14   58.54m ± 10%   56.75m ±  5%        ~ (p=0.353 n=10)
geomean                                        3.486m         3.230m         -7.35%

                                             │     main      │             this-commit              │
                                             │     B/op      │     B/op      vs base                │
GetFiles-14                                     1.394Mi ± 3%   1.032Mi ± 1%  -25.93% (p=0.000 n=10)
GetFilesWithFingerprint-14                      2.199Mi ± 3%   1.757Mi ± 4%  -20.10% (p=0.000 n=10)
GetFilesWithFingerprintGrowing/static-14        1.611Mi ± 0%   1.275Mi ± 0%  -20.87% (p=0.000 n=10)
GetFilesWithFingerprintGrowing/growing-14       5.521Mi ± 0%   5.190Mi ± 0%   -5.99% (p=0.000 n=10)
WatchIdle/active/path/100_files-14              121.7Ki ± 0%   101.7Ki ± 0%  -16.42% (p=0.000 n=10)
WatchIdle/active/path/1000_files-14             1.452Mi ± 0%   1.108Mi ± 0%  -23.64% (p=0.000 n=10)
WatchIdle/active/path/10000_files-14            13.73Mi ± 0%   10.94Mi ± 0%  -20.29% (p=0.000 n=10)
WatchIdle/active/fingerprint/100_files-14       172.1Ki ± 0%   152.2Ki ± 0%  -11.59% (p=0.000 n=10)
WatchIdle/active/fingerprint/1000_files-14      1.963Mi ± 0%   1.620Mi ± 0%  -17.49% (p=0.000 n=10)
WatchIdle/active/fingerprint/10000_files-14     18.72Mi ± 0%   15.94Mi ± 0%  -14.88% (p=0.000 n=10)
WatchIdle/ignored/path/100_files-14            121.73Ki ± 0%   98.59Ki ± 0%  -19.01% (p=0.000 n=10)
WatchIdle/ignored/path/1000_files-14            1.452Mi ± 0%   1.078Mi ± 0%  -25.76% (p=0.000 n=10)
WatchIdle/ignored/path/10000_files-14           13.73Mi ± 0%   10.64Mi ± 0%  -22.51% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/100_files-14      172.2Ki ± 0%   133.4Ki ± 0%  -22.54% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/1000_files-14     1.964Mi ± 0%   1.436Mi ± 0%  -26.86% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/10000_files-14    19.00Mi ± 0%   14.38Mi ± 0%  -24.30% (p=0.000 n=10)
geomean                                         1.721Mi        1.376Mi       -20.06%

                                             │    main     │             this-commit             │
                                             │  allocs/op  │  allocs/op   vs base                │
GetFiles-14                                    9.155k ± 0%   9.098k ± 0%   -0.62% (p=0.000 n=10)
GetFilesWithFingerprint-14                     16.34k ± 0%   16.25k ± 0%   -0.54% (p=0.000 n=10)
GetFilesWithFingerprintGrowing/static-14       16.09k ± 0%   16.05k ± 0%   -0.28% (p=0.000 n=10)
GetFilesWithFingerprintGrowing/growing-14      18.09k ± 0%   18.05k ± 0%   -0.25% (p=0.000 n=10)
WatchIdle/active/path/100_files-14             1.452k ± 0%   1.433k ± 0%   -1.31% (p=0.000 n=10)
WatchIdle/active/path/1000_files-14            14.09k ± 0%   14.05k ± 0%   -0.33% (p=0.000 n=10)
WatchIdle/active/path/10000_files-14           140.3k ± 0%   140.1k ± 0%   -0.10% (p=0.000 n=10)
WatchIdle/active/fingerprint/100_files-14      1.956k ± 0%   1.937k ± 0%   -0.97% (p=0.000 n=10)
WatchIdle/active/fingerprint/1000_files-14     19.10k ± 0%   19.05k ± 0%   -0.25% (p=0.000 n=10)
WatchIdle/active/fingerprint/10000_files-14    190.3k ± 0%   190.2k ± 0%   -0.07% (p=0.000 n=10)
WatchIdle/ignored/path/100_files-14            1.452k ± 0%   1.233k ± 0%  -15.08% (p=0.000 n=10)
WatchIdle/ignored/path/1000_files-14           14.09k ± 0%   12.05k ± 0%  -14.52% (p=0.000 n=10)
WatchIdle/ignored/path/10000_files-14          140.3k ± 0%   120.1k ± 0%  -14.36% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/100_files-14     1.956k ± 0%   1.737k ± 0%  -11.20% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/1000_files-14    19.10k ± 0%   17.05k ± 0%  -10.72% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/10000_files-14   190.3k ± 0%   170.2k ± 0%  -10.58% (p=0.000 n=10)
geomean                                        15.98k        15.14k        -5.27%

The map pre-sizing (fileScanner.lastCount) in isolation, benchmarked as
this commit with and without the field:

                                             │ without-lastCount │            with-lastCount            │
                                             │      sec/op       │    sec/op     vs base                │
GetFiles-14                                         1.743m ± 16%   1.293m ± 29%  -25.81% (p=0.002 n=10)
GetFilesWithFingerprint-14                          5.056m ± 16%   4.905m ± 20%        ~ (p=0.529 n=10)
GetFilesWithFingerprintGrowing/static-14            4.642m ± 16%   4.282m ± 19%        ~ (p=0.089 n=10)
GetFilesWithFingerprintGrowing/growing-14           8.157m ± 20%   6.981m ± 13%  -14.41% (p=0.009 n=10)
WatchIdle/active/path/100_files-14                  160.9µ ±  3%   142.1µ ±  4%  -11.69% (p=0.000 n=10)
WatchIdle/active/path/1000_files-14                 1.997m ± 15%   1.753m ±  8%  -12.21% (p=0.001 n=10)
WatchIdle/active/path/10000_files-14                28.54m ± 13%   28.13m ± 17%        ~ (p=0.739 n=10)
WatchIdle/active/fingerprint/100_files-14           495.4µ ± 10%   398.5µ ±  7%  -19.55% (p=0.000 n=10)
WatchIdle/active/fingerprint/1000_files-14          6.617m ± 14%   5.502m ± 19%  -16.86% (p=0.009 n=10)
WatchIdle/active/fingerprint/10000_files-14         58.58m ± 10%   58.93m ±  6%        ~ (p=0.796 n=10)
WatchIdle/ignored/path/100_files-14                 161.9µ ± 10%   151.9µ ± 10%        ~ (p=0.089 n=10)
WatchIdle/ignored/path/1000_files-14                1.915m ±  7%   1.774m ±  8%   -7.38% (p=0.015 n=10)
WatchIdle/ignored/path/10000_files-14               25.68m ± 11%   24.15m ± 13%        ~ (p=0.218 n=10)
WatchIdle/ignored/fingerprint/100_files-14          431.7µ ±  4%   407.7µ ±  5%   -5.54% (p=0.019 n=10)
WatchIdle/ignored/fingerprint/1000_files-14         5.428m ± 21%   4.832m ± 31%  -10.98% (p=0.035 n=10)
WatchIdle/ignored/fingerprint/10000_files-14        58.22m ±  7%   55.88m ±  7%        ~ (p=0.063 n=10)
geomean                                             3.551m         3.203m         -9.78%

                                             │ without-lastCount │            with-lastCount            │
                                             │       B/op        │     B/op      vs base                │
GetFiles-14                                         1.401Mi ± 3%   1.037Mi ± 5%  -25.96% (p=0.000 n=10)
GetFilesWithFingerprint-14                          2.117Mi ± 2%   1.755Mi ± 7%  -17.11% (p=0.000 n=10)
GetFilesWithFingerprintGrowing/static-14            1.611Mi ± 0%   1.275Mi ± 0%  -20.86% (p=0.000 n=10)
GetFilesWithFingerprintGrowing/growing-14           5.521Mi ± 0%   5.190Mi ± 0%   -6.00% (p=0.000 n=10)
WatchIdle/active/path/100_files-14                  121.7Ki ± 0%   101.7Ki ± 0%  -16.45% (p=0.000 n=10)
WatchIdle/active/path/1000_files-14                 1.452Mi ± 0%   1.108Mi ± 0%  -23.65% (p=0.000 n=10)
WatchIdle/active/path/10000_files-14                13.73Mi ± 0%   10.94Mi ± 0%  -20.29% (p=0.000 n=10)
WatchIdle/active/fingerprint/100_files-14           172.1Ki ± 0%   152.2Ki ± 0%  -11.59% (p=0.000 n=10)
WatchIdle/active/fingerprint/1000_files-14          1.964Mi ± 0%   1.621Mi ± 0%  -17.48% (p=0.000 n=10)
WatchIdle/active/fingerprint/10000_files-14         18.72Mi ± 0%   15.94Mi ± 0%  -14.88% (p=0.000 n=10)
WatchIdle/ignored/path/100_files-14                118.60Ki ± 0%   98.60Ki ± 0%  -16.86% (p=0.000 n=10)
WatchIdle/ignored/path/1000_files-14                1.421Mi ± 0%   1.078Mi ± 0%  -24.14% (p=0.000 n=10)
WatchIdle/ignored/path/10000_files-14               13.42Mi ± 0%   10.64Mi ± 0%  -20.75% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/100_files-14          153.3Ki ± 0%   133.4Ki ± 0%  -13.00% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/1000_files-14         1.779Mi ± 0%   1.436Mi ± 0%  -19.29% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/10000_files-14        17.17Mi ± 2%   14.38Mi ± 2%  -16.23% (p=0.000 n=10)
geomean                                             1.677Mi        1.376Mi       -17.93%

                                             │ without-lastCount │           with-lastCount           │
                                             │     allocs/op     │  allocs/op   vs base               │
GetFiles-14                                          9.155k ± 0%   9.100k ± 0%  -0.61% (p=0.000 n=10)
GetFilesWithFingerprint-14                           16.31k ± 0%   16.25k ± 0%  -0.33% (p=0.009 n=10)
GetFilesWithFingerprintGrowing/static-14             16.09k ± 0%   16.05k ± 0%  -0.29% (p=0.000 n=10)
GetFilesWithFingerprintGrowing/growing-14            18.09k ± 0%   18.05k ± 0%  -0.25% (p=0.000 n=10)
WatchIdle/active/path/100_files-14                   1.452k ± 0%   1.433k ± 0%  -1.31% (p=0.000 n=10)
WatchIdle/active/path/1000_files-14                  14.09k ± 0%   14.05k ± 0%  -0.33% (p=0.000 n=10)
WatchIdle/active/path/10000_files-14                 140.3k ± 0%   140.1k ± 0%  -0.10% (p=0.000 n=10)
WatchIdle/active/fingerprint/100_files-14            1.956k ± 0%   1.937k ± 0%  -0.97% (p=0.000 n=10)
WatchIdle/active/fingerprint/1000_files-14           19.10k ± 0%   19.05k ± 0%  -0.25% (p=0.000 n=10)
WatchIdle/active/fingerprint/10000_files-14          190.3k ± 0%   190.2k ± 0%  -0.07% (p=0.000 n=10)
WatchIdle/ignored/path/100_files-14                  1.252k ± 0%   1.233k ± 0%  -1.52% (p=0.000 n=10)
WatchIdle/ignored/path/1000_files-14                 12.09k ± 0%   12.05k ± 0%  -0.38% (p=0.000 n=10)
WatchIdle/ignored/path/10000_files-14                120.3k ± 0%   120.1k ± 0%  -0.12% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/100_files-14           1.756k ± 0%   1.737k ± 0%  -1.08% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/1000_files-14          17.10k ± 0%   17.05k ± 0%  -0.27% (p=0.000 n=10)
WatchIdle/ignored/fingerprint/10000_files-14         170.3k ± 0%   170.2k ± 0%  -0.08% (p=0.000 n=10)
geomean                                              15.22k        15.14k       -0.50%

(cherry picked from commit c05890c)

Co-authored-by: Orestis Floros <orestis.floros@elastic.co>

Author: mergify[bot]

changelog/fragments/1783612408-filestream-scanner-reduce-scan-allocations.yaml

@@ -0,0 +1,3 @@
+kind: enhancement
+summary: Reduce filestream scanner per-scan memory allocations for large idle file sets
+component: filebeat

filebeat/input/filestream/fswatch.go

@@ -206,11 +206,12 @@ func (w *fileWatcher) watch(
 
 	// file identity is updated in GetFiles
 	now := time.Now()
-	paths, scanMetrics := w.scanner.GetFiles(loginp.FileScanOptions{
+	scanOpts := loginp.FileScanOptions{
 		CurrentTime:         now,
 		IgnoreOlder:         ignoreOlder,
 		IgnoreInactiveSince: ignoreInactiveSince,
-	})
+	}
+	paths, scanMetrics := w.scanner.GetFiles(scanOpts)
 	metrics.UpdateFileScanMetrics(scanMetrics)
 
 	// for debugging purposes
@@ -225,10 +226,6 @@ func (w *fileWatcher) watch(
 	harvesterFiles := make([]loginp.HarvesterFile, 0, len(paths))
 
 	for path, fd := range paths {
-		// srcID is the file identity, it is the same value used to identify
-		// the harvester and as registry key for the file's state
-		srcID := w.getFileIdentity(fd)
-
 		// if the scanner found a new path or an existing path
 		// with a different file, it is a new file
 		prevDesc, ok := w.prev[path]
@@ -239,61 +236,47 @@ func (w *fileWatcher) watch(
 			continue
 		}
 
-		// If we got notifications about harvesters being closed, update
-		// the state accordingly.
-		//
-		// This is used to prevent a sort of race condition:
-		// When the reader/harvester reaches EOF, it blocks on a backoff,
-		// if during this time [logFile.shouldBeClosed] is called, marks the
-		// file as inactive and closes the reader context, once the backoff
-		// time expires the reader and harvester are closed without ingesting
-		// any more data.
-		//
-		// If the [fileWatcher] sends a write event while the harvester was blocked
-		// no new harvester is started because one is already running, however the
-		// [fileWatcher] updates its internal state and won't send write events until
-		// more data is added to the file.
-		//
-		// This can cause some lines to be missed because the harvester closed
-		// and the write event was lost.
-		//
-		// To prevent this from happening we get notified the offset of the file
-		// (data ingested) when the harvester closes. If we have this data we
-		// update our state to the same as the harvester, therefore starting
-		// a new harvester if needed.
-		w.closedHarvestersMutex.Lock()
-		if size, harvesterClosed := w.closedHarvesters[srcID]; harvesterClosed {
-			w.log.Debugf("Updating previous state because harvester was closed. '%s': %d", srcID, size)
-			prevDesc.SetBytesIngested(size)
+		// srcID is the file identity (harvester ID/registry key), resolved lazily via ensureSrcID:
+		// an unchanged, untracked file (gzip, empty, ignore_older) never needs one, saving allocs.
+		var srcID string
+		ensureSrcID := func() string {
+			if srcID == "" { // getFileIdentity never returns ""
+				srcID = w.getFileIdentity(fd)
+			}
+			return srcID
+		}
+
+		// closedHarvesters is empty in the steady state; this reconciliation is usually skipped.
+		if w.hasClosedHarvesters() {
+			w.reconcileClosedHarvester(&prevDesc, ensureSrcID())
 		}
-		w.closedHarvestersMutex.Unlock()
 
 		var e loginp.FSEvent
 		switch {
 		// the new size is smaller, the file was truncated
 		case prevDesc.Info.Size() > fd.Info.Size():
-			e = truncateEvent(path, fd, srcID)
+			e = truncateEvent(path, fd, ensureSrcID())
 			truncatedCount++
 
 		// the size is the same, timestamps are different, the file was touched
 		case prevDesc.Info.Size() == fd.Info.Size() && prevDesc.Info.ModTime() != fd.Info.ModTime():
 			if w.cfg.ResendOnModTime {
-				e = truncateEvent(path, fd, srcID)
+				e = truncateEvent(path, fd, ensureSrcID())
 				truncatedCount++
 			}
 
 		// the new size is larger, something was written.
 		// If a harvester for this file was closed recently,
 		// we use its state instead of the one we have cached.
 		case prevDesc.SizeOrBytesIngested() < fd.Info.Size():
-			e = writeEvent(path, fd, srcID)
+			e = writeEvent(path, fd, ensureSrcID())
 			writtenCount++
 
 		default:
 			// For the delete feature we need to run the harvester for
 			// files that have not changed until they're deleted.
 			if w.cfg.SendNotChanged {
-				e = notChangedEvent(path, fd, srcID)
+				e = notChangedEvent(path, fd, ensureSrcID())
 			}
 		}
 
@@ -306,17 +289,13 @@ func (w *fileWatcher) watch(
 			}
 		}
 
-		// We want progress metrics for all files except truncated ones
-		if e.Op != loginp.OpTruncate {
-			harvesterFiles = appendHarvesterFile(harvesterFiles, fd, srcID, now, ignoreOlder, ignoreInactiveSince)
+		// Record progress metrics for trackable, non-truncated files (tracksHarvesterProgress).
+		if e.Op != loginp.OpTruncate && tracksHarvesterProgress(&fd, scanOpts) {
+			harvesterFiles = append(harvesterFiles, loginp.HarvesterFile{ID: ensureSrcID(), Size: fd.Info.Size()})
 		}
 
 		// delete from previous state to mark that we've seen the existing file again
 		delete(w.prev, path)
-		// Delete used state from closedHarvesters
-		w.closedHarvestersMutex.Lock()
-		delete(w.closedHarvesters, srcID)
-		w.closedHarvestersMutex.Unlock()
 	}
 
 	// Remaining files in the prev map are missing from this scan — either
@@ -446,9 +425,10 @@ func (w *fileWatcher) watch(
 			createdCount++
 		}
 
-		// The previous for loop has an early continue for new files,
-		// so we need to collect their metrics here.
-		harvesterFiles = appendHarvesterFile(harvesterFiles, *fd, srcID, now, ignoreOlder, ignoreInactiveSince)
+		// New files skip the main loop via early continue, so collect their metrics here.
+		if tracksHarvesterProgress(fd, scanOpts) {
+			harvesterFiles = append(harvesterFiles, loginp.HarvesterFile{ID: srcID, Size: fd.Info.Size()})
+		}
 	}
 
 	w.log.Debugw("File scan complete",
@@ -491,27 +471,31 @@ func (w *fileWatcher) watch(
 	w.prev = paths
 }
 
-func appendHarvesterFile(
-	files []loginp.HarvesterFile,
-	fd loginp.FileDescriptor,
-	srcID string,
-	now time.Time,
-	ignoreOlder time.Duration,
-	ignoreInactiveSince time.Time,
-) []loginp.HarvesterFile {
-	opts := loginp.FileScanOptions{
-		CurrentTime:         now,
-		IgnoreOlder:         ignoreOlder,
-		IgnoreInactiveSince: ignoreInactiveSince,
-	}
-	if fd.GZIP || fd.Info.Size() <= 0 || isFileIgnored(fd, opts) {
-		return files
+// hasClosedHarvesters reports whether any harvester-close notification awaits reconciliation.
+func (w *fileWatcher) hasClosedHarvesters() bool {
+	w.closedHarvestersMutex.Lock()
+	defer w.closedHarvestersMutex.Unlock()
+	return len(w.closedHarvesters) > 0
+}
+
+// reconcileClosedHarvester folds a recently-closed harvester's ingested offset (from
+// closedHarvesters) into prevDesc so a restarted harvester resumes from the right position.
+// It guards a close-during-backoff race that would otherwise withhold writes and lose lines.
+func (w *fileWatcher) reconcileClosedHarvester(prevDesc *loginp.FileDescriptor, id string) {
+	w.closedHarvestersMutex.Lock()
+	defer w.closedHarvestersMutex.Unlock()
+	size, ok := w.closedHarvesters[id]
+	if !ok {
+		return
 	}
+	w.log.Debugf("Updating previous state because harvester was closed. '%s': %d", id, size)
+	prevDesc.SetBytesIngested(size)
+	delete(w.closedHarvesters, id)
+}
 
-	return append(files, loginp.HarvesterFile{
-		ID:   srcID,
-		Size: fd.Info.Size(),
-	})
+// tracksHarvesterProgress reports whether a file contributes to the harvester progress metrics.
+func tracksHarvesterProgress(fd *loginp.FileDescriptor, opts loginp.FileScanOptions) bool {
+	return !fd.GZIP && fd.Info.Size() > 0 && !isFileIgnored(*fd, opts)
 }
 
 // isFileIgnored returns true when a file is ignored, no matter the reason.
@@ -569,10 +553,10 @@ func (w *fileWatcher) Event() loginp.FSEvent {
 }
 
 // GetFiles runs a one-off enumeration scan for the prospector's Init and
-// TakeOver phases. It is side-effect free: unlike the watch loop it does not
-// advance the scanner's completedFingerprints set, so these pre-watch scans
-// cannot suppress the bridging raw header a still-growing entry needs to
-// migrate its registry key after a restart.
+// TakeOver phases. Unlike the watch loop it does not advance the scanner's
+// completedFingerprints set, so these pre-watch scans cannot suppress the
+// bridging raw header a still-growing entry needs to migrate its registry key
+// after a restart.
 func (w *fileWatcher) GetFiles(opts loginp.FileScanOptions) (map[string]loginp.FileDescriptor, loginp.FileScanMetrics) {
 	return w.scanner.GetFiles(opts)
 }
@@ -635,6 +619,9 @@ type fileScanner struct {
 	// prospector runs in Init/TakeOver cannot suppress the header a
 	// still-growing entry needs to migrate across a restart.
 	completedFingerprints map[string]struct{}
+
+	// lastCount is the number of unique files the previous scan produced.
+	lastCount int
 }
 
 func newFileScanner(logger *logp.Logger, paths []string, config fileScannerConfig, compression string) (*fileScanner, error) {
@@ -717,11 +704,12 @@ func (s *fileScanner) GetFiles(opts loginp.FileScanOptions) (map[string]loginp.F
 		opts.CurrentTime = time.Now()
 	}
 
-	fdByName := map[string]loginp.FileDescriptor{}
+	// Pre-size the per-scan maps from the previous scan's count.
+	fdByName := make(map[string]loginp.FileDescriptor, s.lastCount)
 	// used to determine if a symlink resolves in a already known target
-	uniqueIDs := map[string]string{}
+	uniqueIDs := make(map[string]string, s.lastCount)
 	// used to filter out duplicate matches
-	uniqueFiles := map[string]struct{}{}
+	uniqueFiles := make(map[string]struct{}, s.lastCount)
 	scanMetrics := loginp.FileScanMetrics{}
 
 	for _, path := range s.paths {
@@ -792,6 +780,7 @@ func (s *fileScanner) GetFiles(opts loginp.FileScanOptions) (map[string]loginp.F
 	}
 
 	scanMetrics.FilesUnique = int64(len(fdByName))
+	s.lastCount = len(fdByName)
 	return fdByName, scanMetrics
 }
 

filebeat/input/filestream/fswatch_test.go

@@ -1560,17 +1560,20 @@ func (s *testFileScanner) GetFiles(loginp.FileScanOptions) (map[string]loginp.Fi
 
 const benchmarkFileCount = 1000
 
-func BenchmarkGetFiles(b *testing.B) {
-	dir := b.TempDir()
-	basenameFormat := "file-%d.log"
-
-	for i := 0; i < benchmarkFileCount; i++ {
-		filename := filepath.Join(dir, fmt.Sprintf(basenameFormat, i))
+// writeBenchmarkFiles creates n log files under dir and returns the glob that matches them.
+func writeBenchmarkFiles(tb testing.TB, dir string, n int) []string {
+	tb.Helper()
+	for i := range n {
+		filename := filepath.Join(dir, fmt.Sprintf("file-%d.log", i))
 		content := fmt.Sprintf("content-%d\n", i)
-		err := os.WriteFile(filename, []byte(strings.Repeat(content, 1024)), 0777)
-		require.NoError(b, err)
+		require.NoError(tb, os.WriteFile(
+			filename, []byte(strings.Repeat(content, 1024)), 0o644))
 	}
-	paths := []string{filepath.Join(dir, "*.log")}
+	return []string{filepath.Join(dir, "*.log")}
+}
+
+func BenchmarkGetFiles(b *testing.B) {
+	paths := writeBenchmarkFiles(b, b.TempDir(), benchmarkFileCount)
 	cfg := fileScannerConfig{
 		Fingerprint: fingerprintConfig{
 			Enabled: false,
@@ -1586,16 +1589,7 @@ func BenchmarkGetFiles(b *testing.B) {
 }
 
 func BenchmarkGetFilesWithFingerprint(b *testing.B) {
-	dir := b.TempDir()
-	basenameFormat := "file-%d.log"
-
-	for i := 0; i < benchmarkFileCount; i++ {
-		filename := filepath.Join(dir, fmt.Sprintf(basenameFormat, i))
-		content := fmt.Sprintf("content-%d\n", i)
-		err := os.WriteFile(filename, []byte(strings.Repeat(content, 1024)), 0777)
-		require.NoError(b, err)
-	}
-	paths := []string{filepath.Join(dir, "*.log")}
+	paths := writeBenchmarkFiles(b, b.TempDir(), benchmarkFileCount)
 	cfg := fileScannerConfig{
 		Fingerprint: fingerprintConfig{
 			Enabled: true,
@@ -1637,16 +1631,7 @@ func BenchmarkGetFilesWithFingerprintGrowing(b *testing.B) {
 	}
 	for _, tc := range cases {
 		b.Run(tc.name, func(b *testing.B) {
-			dir := b.TempDir()
-			for i := range benchmarkFileCount {
-				filename := filepath.Join(dir, fmt.Sprintf("file-%d.log", i))
-				content := fmt.Sprintf("content-%d\n", i)
-				// ~10KB per file: well above the 1024-byte threshold, so every
-				// file is fingerprinted with a final SHA-256 on every scan.
-				err := os.WriteFile(filename, []byte(strings.Repeat(content, 1024)), 0777)
-				require.NoError(b, err)
-			}
-			paths := []string{filepath.Join(dir, "*.log")}
+			paths := writeBenchmarkFiles(b, b.TempDir(), benchmarkFileCount)
 			cfg := fileScannerConfig{
 				Fingerprint: fingerprintConfig{
 					Enabled: true,
@@ -1667,6 +1652,88 @@ func BenchmarkGetFilesWithFingerprintGrowing(b *testing.B) {
 	}
 }
 
+// BenchmarkWatchIdle measures fileWatcher.watch over an unchanged file set: the mostly-idle steady
+// state where allocs/op is the cost of observing nothing changed (identity cache + pre-sized maps).
+func BenchmarkWatchIdle(b *testing.B) {
+	states := []struct {
+		name        string
+		ignoreOlder time.Duration
+		aged        bool
+	}{
+		{"active", 0, false},
+		{"ignored", time.Hour, true},
+	}
+	identities := []struct {
+		name        string
+		fingerprint bool
+		newID       func() fileIdentifier
+	}{
+		{"path", false, func() fileIdentifier { return mustPathIdentifier(false) }},
+		{"fingerprint", true, func() fileIdentifier {
+			fi, _ := newFingerprintIdentifier(nil, nil)
+			return fi
+		}},
+	}
+
+	for _, st := range states {
+		for _, id := range identities {
+			for _, fileCount := range []int{100, 1000, 10000} {
+				b.Run(fmt.Sprintf("%s/%s/%d_files", st.name, id.name, fileCount), func(b *testing.B) {
+					paths := writeBenchmarkFiles(b, b.TempDir(), fileCount)
+
+					if st.aged {
+						// Age every file well past ignoreOlder so it is excluded.
+						old := time.Now().Add(-48 * time.Hour)
+						matches, err := filepath.Glob(paths[0])
+						require.NoError(b, err)
+						for _, m := range matches {
+							require.NoError(b, os.Chtimes(m, old, old))
+						}
+					}
+
+					cfg := defaultFileWatcherConfig()
+					cfg.Scanner.Fingerprint.Enabled = id.fingerprint
+					cfg.Scanner.Fingerprint.Growing = id.fingerprint
+
+					fw, err := newFileWatcher(
+						logp.NewNopLogger(),
+						paths,
+						cfg,
+						CompressionNone,
+						false,
+						id.newID(),
+						mustSourceIdentifier("bench-id"),
+					)
+					require.NoError(b, err)
+
+					// A create event is emitted for every file on the first scan; idle rescans emit
+					// nothing. Drain in the background so watch's sends never block.
+					ctx := b.Context()
+					go func() {
+						for {
+							select {
+							case <-ctx.Done():
+								return
+							case <-fw.events:
+							}
+						}
+					}()
+
+					metrics := newTestMetrics()
+					// Prime w.prev so the benchmarked scans are pure steady state.
+					fw.watch(ctx, metrics, st.ignoreOlder, time.Time{})
+
+					b.ReportAllocs()
+					b.ResetTimer()
+					for b.Loop() {
+						fw.watch(ctx, metrics, st.ignoreOlder, time.Time{})
+					}
+				})
+			}
+		}
+	}
+}
+
 func createWatcherWithConfig(t *testing.T, logger *logp.Logger, paths []string, cfgStr string) *fileWatcher {
 	tmpCfg := struct {
 		Scaner fileWatcherConfig `config:"scanner"`