Skip to content

Commit 7ce32d4

Browse files
orestisflmergify[bot]
orestisfl
authored and
mergify[bot]
committed
filestream: reduce per-scan allocations (#51863)
Two per-file, per-scan allocations are removed: - fileWatcher.watch resolved the file identity for every file up front. It is now resolved lazily on first use. - fileScanner.GetFiles rebuilt its per-scan maps from empty, forcing repeated grow/rehash allocations on a stable file set. They are now pre-sized from the previous scan's unique-file count. Combined effect vs main (go test -bench 'BenchmarkGetFiles|BenchmarkWatchIdle' -benchmem -benchtime=200ms -count=10): │ main │ this-commit │ │ sec/op │ sec/op vs base │ GetFiles-14 1.660m ± 16% 1.302m ± 5% -21.55% (p=0.000 n=10) GetFilesWithFingerprint-14 5.657m ± 9% 4.898m ± 15% -13.42% (p=0.019 n=10) GetFilesWithFingerprintGrowing/static-14 5.026m ± 14% 4.449m ± 6% -11.49% (p=0.001 n=10) GetFilesWithFingerprintGrowing/growing-14 6.823m ± 11% 7.775m ± 19% +13.96% (p=0.023 n=10) WatchIdle/active/path/100_files-14 160.6µ ± 12% 155.9µ ± 6% ~ (p=0.280 n=10) WatchIdle/active/path/1000_files-14 1.938m ± 12% 1.805m ± 21% ~ (p=0.218 n=10) WatchIdle/active/path/10000_files-14 27.05m ± 12% 24.65m ± 14% -8.87% (p=0.009 n=10) WatchIdle/active/fingerprint/100_files-14 439.9µ ± 2% 413.3µ ± 3% -6.04% (p=0.004 n=10) WatchIdle/active/fingerprint/1000_files-14 5.352m ± 11% 5.300m ± 20% ~ (p=0.853 n=10) WatchIdle/active/fingerprint/10000_files-14 58.20m ± 6% 59.59m ± 12% ~ (p=0.190 n=10) WatchIdle/ignored/path/100_files-14 170.0µ ± 9% 140.3µ ± 10% -17.47% (p=0.000 n=10) WatchIdle/ignored/path/1000_files-14 2.270m ± 9% 1.652m ± 1% -27.23% (p=0.000 n=10) WatchIdle/ignored/path/10000_files-14 25.86m ± 12% 25.09m ± 10% ~ (p=0.218 n=10) WatchIdle/ignored/fingerprint/100_files-14 428.7µ ± 14% 398.9µ ± 7% ~ (p=0.218 n=10) WatchIdle/ignored/fingerprint/1000_files-14 5.107m ± 10% 5.325m ± 15% ~ (p=0.971 n=10) WatchIdle/ignored/fingerprint/10000_files-14 58.54m ± 10% 56.75m ± 5% ~ (p=0.353 n=10) geomean 3.486m 3.230m -7.35% │ main │ this-commit │ │ B/op │ B/op vs base │ GetFiles-14 1.394Mi ± 3% 1.032Mi ± 1% -25.93% (p=0.000 n=10) GetFilesWithFingerprint-14 2.199Mi ± 3% 1.757Mi ± 4% -20.10% (p=0.000 n=10) GetFilesWithFingerprintGrowing/static-14 1.611Mi ± 0% 1.275Mi ± 0% -20.87% (p=0.000 n=10) GetFilesWithFingerprintGrowing/growing-14 5.521Mi ± 0% 5.190Mi ± 0% -5.99% (p=0.000 n=10) WatchIdle/active/path/100_files-14 121.7Ki ± 0% 101.7Ki ± 0% -16.42% (p=0.000 n=10) WatchIdle/active/path/1000_files-14 1.452Mi ± 0% 1.108Mi ± 0% -23.64% (p=0.000 n=10) WatchIdle/active/path/10000_files-14 13.73Mi ± 0% 10.94Mi ± 0% -20.29% (p=0.000 n=10) WatchIdle/active/fingerprint/100_files-14 172.1Ki ± 0% 152.2Ki ± 0% -11.59% (p=0.000 n=10) WatchIdle/active/fingerprint/1000_files-14 1.963Mi ± 0% 1.620Mi ± 0% -17.49% (p=0.000 n=10) WatchIdle/active/fingerprint/10000_files-14 18.72Mi ± 0% 15.94Mi ± 0% -14.88% (p=0.000 n=10) WatchIdle/ignored/path/100_files-14 121.73Ki ± 0% 98.59Ki ± 0% -19.01% (p=0.000 n=10) WatchIdle/ignored/path/1000_files-14 1.452Mi ± 0% 1.078Mi ± 0% -25.76% (p=0.000 n=10) WatchIdle/ignored/path/10000_files-14 13.73Mi ± 0% 10.64Mi ± 0% -22.51% (p=0.000 n=10) WatchIdle/ignored/fingerprint/100_files-14 172.2Ki ± 0% 133.4Ki ± 0% -22.54% (p=0.000 n=10) WatchIdle/ignored/fingerprint/1000_files-14 1.964Mi ± 0% 1.436Mi ± 0% -26.86% (p=0.000 n=10) WatchIdle/ignored/fingerprint/10000_files-14 19.00Mi ± 0% 14.38Mi ± 0% -24.30% (p=0.000 n=10) geomean 1.721Mi 1.376Mi -20.06% │ main │ this-commit │ │ allocs/op │ allocs/op vs base │ GetFiles-14 9.155k ± 0% 9.098k ± 0% -0.62% (p=0.000 n=10) GetFilesWithFingerprint-14 16.34k ± 0% 16.25k ± 0% -0.54% (p=0.000 n=10) GetFilesWithFingerprintGrowing/static-14 16.09k ± 0% 16.05k ± 0% -0.28% (p=0.000 n=10) GetFilesWithFingerprintGrowing/growing-14 18.09k ± 0% 18.05k ± 0% -0.25% (p=0.000 n=10) WatchIdle/active/path/100_files-14 1.452k ± 0% 1.433k ± 0% -1.31% (p=0.000 n=10) WatchIdle/active/path/1000_files-14 14.09k ± 0% 14.05k ± 0% -0.33% (p=0.000 n=10) WatchIdle/active/path/10000_files-14 140.3k ± 0% 140.1k ± 0% -0.10% (p=0.000 n=10) WatchIdle/active/fingerprint/100_files-14 1.956k ± 0% 1.937k ± 0% -0.97% (p=0.000 n=10) WatchIdle/active/fingerprint/1000_files-14 19.10k ± 0% 19.05k ± 0% -0.25% (p=0.000 n=10) WatchIdle/active/fingerprint/10000_files-14 190.3k ± 0% 190.2k ± 0% -0.07% (p=0.000 n=10) WatchIdle/ignored/path/100_files-14 1.452k ± 0% 1.233k ± 0% -15.08% (p=0.000 n=10) WatchIdle/ignored/path/1000_files-14 14.09k ± 0% 12.05k ± 0% -14.52% (p=0.000 n=10) WatchIdle/ignored/path/10000_files-14 140.3k ± 0% 120.1k ± 0% -14.36% (p=0.000 n=10) WatchIdle/ignored/fingerprint/100_files-14 1.956k ± 0% 1.737k ± 0% -11.20% (p=0.000 n=10) WatchIdle/ignored/fingerprint/1000_files-14 19.10k ± 0% 17.05k ± 0% -10.72% (p=0.000 n=10) WatchIdle/ignored/fingerprint/10000_files-14 190.3k ± 0% 170.2k ± 0% -10.58% (p=0.000 n=10) geomean 15.98k 15.14k -5.27% The map pre-sizing (fileScanner.lastCount) in isolation, benchmarked as this commit with and without the field: │ without-lastCount │ with-lastCount │ │ sec/op │ sec/op vs base │ GetFiles-14 1.743m ± 16% 1.293m ± 29% -25.81% (p=0.002 n=10) GetFilesWithFingerprint-14 5.056m ± 16% 4.905m ± 20% ~ (p=0.529 n=10) GetFilesWithFingerprintGrowing/static-14 4.642m ± 16% 4.282m ± 19% ~ (p=0.089 n=10) GetFilesWithFingerprintGrowing/growing-14 8.157m ± 20% 6.981m ± 13% -14.41% (p=0.009 n=10) WatchIdle/active/path/100_files-14 160.9µ ± 3% 142.1µ ± 4% -11.69% (p=0.000 n=10) WatchIdle/active/path/1000_files-14 1.997m ± 15% 1.753m ± 8% -12.21% (p=0.001 n=10) WatchIdle/active/path/10000_files-14 28.54m ± 13% 28.13m ± 17% ~ (p=0.739 n=10) WatchIdle/active/fingerprint/100_files-14 495.4µ ± 10% 398.5µ ± 7% -19.55% (p=0.000 n=10) WatchIdle/active/fingerprint/1000_files-14 6.617m ± 14% 5.502m ± 19% -16.86% (p=0.009 n=10) WatchIdle/active/fingerprint/10000_files-14 58.58m ± 10% 58.93m ± 6% ~ (p=0.796 n=10) WatchIdle/ignored/path/100_files-14 161.9µ ± 10% 151.9µ ± 10% ~ (p=0.089 n=10) WatchIdle/ignored/path/1000_files-14 1.915m ± 7% 1.774m ± 8% -7.38% (p=0.015 n=10) WatchIdle/ignored/path/10000_files-14 25.68m ± 11% 24.15m ± 13% ~ (p=0.218 n=10) WatchIdle/ignored/fingerprint/100_files-14 431.7µ ± 4% 407.7µ ± 5% -5.54% (p=0.019 n=10) WatchIdle/ignored/fingerprint/1000_files-14 5.428m ± 21% 4.832m ± 31% -10.98% (p=0.035 n=10) WatchIdle/ignored/fingerprint/10000_files-14 58.22m ± 7% 55.88m ± 7% ~ (p=0.063 n=10) geomean 3.551m 3.203m -9.78% │ without-lastCount │ with-lastCount │ │ B/op │ B/op vs base │ GetFiles-14 1.401Mi ± 3% 1.037Mi ± 5% -25.96% (p=0.000 n=10) GetFilesWithFingerprint-14 2.117Mi ± 2% 1.755Mi ± 7% -17.11% (p=0.000 n=10) GetFilesWithFingerprintGrowing/static-14 1.611Mi ± 0% 1.275Mi ± 0% -20.86% (p=0.000 n=10) GetFilesWithFingerprintGrowing/growing-14 5.521Mi ± 0% 5.190Mi ± 0% -6.00% (p=0.000 n=10) WatchIdle/active/path/100_files-14 121.7Ki ± 0% 101.7Ki ± 0% -16.45% (p=0.000 n=10) WatchIdle/active/path/1000_files-14 1.452Mi ± 0% 1.108Mi ± 0% -23.65% (p=0.000 n=10) WatchIdle/active/path/10000_files-14 13.73Mi ± 0% 10.94Mi ± 0% -20.29% (p=0.000 n=10) WatchIdle/active/fingerprint/100_files-14 172.1Ki ± 0% 152.2Ki ± 0% -11.59% (p=0.000 n=10) WatchIdle/active/fingerprint/1000_files-14 1.964Mi ± 0% 1.621Mi ± 0% -17.48% (p=0.000 n=10) WatchIdle/active/fingerprint/10000_files-14 18.72Mi ± 0% 15.94Mi ± 0% -14.88% (p=0.000 n=10) WatchIdle/ignored/path/100_files-14 118.60Ki ± 0% 98.60Ki ± 0% -16.86% (p=0.000 n=10) WatchIdle/ignored/path/1000_files-14 1.421Mi ± 0% 1.078Mi ± 0% -24.14% (p=0.000 n=10) WatchIdle/ignored/path/10000_files-14 13.42Mi ± 0% 10.64Mi ± 0% -20.75% (p=0.000 n=10) WatchIdle/ignored/fingerprint/100_files-14 153.3Ki ± 0% 133.4Ki ± 0% -13.00% (p=0.000 n=10) WatchIdle/ignored/fingerprint/1000_files-14 1.779Mi ± 0% 1.436Mi ± 0% -19.29% (p=0.000 n=10) WatchIdle/ignored/fingerprint/10000_files-14 17.17Mi ± 2% 14.38Mi ± 2% -16.23% (p=0.000 n=10) geomean 1.677Mi 1.376Mi -17.93% │ without-lastCount │ with-lastCount │ │ allocs/op │ allocs/op vs base │ GetFiles-14 9.155k ± 0% 9.100k ± 0% -0.61% (p=0.000 n=10) GetFilesWithFingerprint-14 16.31k ± 0% 16.25k ± 0% -0.33% (p=0.009 n=10) GetFilesWithFingerprintGrowing/static-14 16.09k ± 0% 16.05k ± 0% -0.29% (p=0.000 n=10) GetFilesWithFingerprintGrowing/growing-14 18.09k ± 0% 18.05k ± 0% -0.25% (p=0.000 n=10) WatchIdle/active/path/100_files-14 1.452k ± 0% 1.433k ± 0% -1.31% (p=0.000 n=10) WatchIdle/active/path/1000_files-14 14.09k ± 0% 14.05k ± 0% -0.33% (p=0.000 n=10) WatchIdle/active/path/10000_files-14 140.3k ± 0% 140.1k ± 0% -0.10% (p=0.000 n=10) WatchIdle/active/fingerprint/100_files-14 1.956k ± 0% 1.937k ± 0% -0.97% (p=0.000 n=10) WatchIdle/active/fingerprint/1000_files-14 19.10k ± 0% 19.05k ± 0% -0.25% (p=0.000 n=10) WatchIdle/active/fingerprint/10000_files-14 190.3k ± 0% 190.2k ± 0% -0.07% (p=0.000 n=10) WatchIdle/ignored/path/100_files-14 1.252k ± 0% 1.233k ± 0% -1.52% (p=0.000 n=10) WatchIdle/ignored/path/1000_files-14 12.09k ± 0% 12.05k ± 0% -0.38% (p=0.000 n=10) WatchIdle/ignored/path/10000_files-14 120.3k ± 0% 120.1k ± 0% -0.12% (p=0.000 n=10) WatchIdle/ignored/fingerprint/100_files-14 1.756k ± 0% 1.737k ± 0% -1.08% (p=0.000 n=10) WatchIdle/ignored/fingerprint/1000_files-14 17.10k ± 0% 17.05k ± 0% -0.27% (p=0.000 n=10) WatchIdle/ignored/fingerprint/10000_files-14 170.3k ± 0% 170.2k ± 0% -0.08% (p=0.000 n=10) geomean 15.22k 15.14k -0.50% (cherry picked from commit c05890c) # Conflicts: # filebeat/input/filestream/fswatch.go # filebeat/input/filestream/fswatch_test.go
1 parent 11a8246 commit 7ce32d4

3 files changed

Lines changed: 286 additions & 59 deletions

File tree

‎changelog/fragments/1783612408-filestream-scanner-reduce-scan-allocations.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
kind: enhancement
2+
summary: Reduce filestream scanner per-scan memory allocations for large idle file sets
3+
component: filebeat

‎filebeat/input/filestream/fswatch.go‎

Lines changed: 140 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,19 @@ func (w *fileWatcher) processNotification(evt loginp.HarvesterStatus) {
172172
func (w *fileWatcher) watch(ctx unison.Canceler) {
173173
w.log.Debug("Start next scan")
174174

175+
<<<<<<< HEAD
175176
paths := w.scanner.GetFiles()
177+
=======
178+
// file identity is updated in GetFiles
179+
now := time.Now()
180+
scanOpts := loginp.FileScanOptions{
181+
CurrentTime: now,
182+
IgnoreOlder: ignoreOlder,
183+
IgnoreInactiveSince: ignoreInactiveSince,
184+
}
185+
paths, scanMetrics := w.scanner.GetFiles(scanOpts)
186+
metrics.UpdateFileScanMetrics(scanMetrics)
187+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
176188

177189
// for debugging purposes
178190
writtenCount := 0
@@ -185,10 +197,6 @@ func (w *fileWatcher) watch(ctx unison.Canceler) {
185197
newFilesByID := make(map[string]*loginp.FileDescriptor)
186198

187199
for path, fd := range paths {
188-
// srcID is the file identity, it is the same value used to identify
189-
// the harvester and as registry key for the file's state
190-
srcID := w.getFileIdentity(fd)
191-
192200
// if the scanner found a new path or an existing path
193201
// with a different file, it is a new file
194202
prevDesc, ok := w.prev[path]
@@ -199,61 +207,47 @@ func (w *fileWatcher) watch(ctx unison.Canceler) {
199207
continue
200208
}
201209

202-
// If we got notifications about harvesters being closed, update
203-
// the state accordingly.
204-
//
205-
// This is used to prevent a sort of race condition:
206-
// When the reader/harvester reaches EOF, it blocks on a backoff,
207-
// if during this time [logFile.shouldBeClosed] is called, marks the
208-
// file as inactive and closes the reader context, once the backoff
209-
// time expires the reader and harvester are closed without ingesting
210-
// any more data.
211-
//
212-
// If the [fileWatcher] sends a write event while the harvester was blocked
213-
// no new harvester is started because one is already running, however the
214-
// [fileWatcher] updates its internal state and won't send write events until
215-
// more data is added to the file.
216-
//
217-
// This can cause some lines to be missed because the harvester closed
218-
// and the write event was lost.
219-
//
220-
// To prevent this from happening we get notified the offset of the file
221-
// (data ingested) when the harvester closes. If we have this data we
222-
// update our state to the same as the harvester, therefore starting
223-
// a new harvester if needed.
224-
w.closedHarvestersMutex.Lock()
225-
if size, harvesterClosed := w.closedHarvesters[srcID]; harvesterClosed {
226-
w.log.Debugf("Updating previous state because harvester was closed. '%s': %d", srcID, size)
227-
prevDesc.SetBytesIngested(size)
210+
// srcID is the file identity (harvester ID/registry key), resolved lazily via ensureSrcID:
211+
// an unchanged, untracked file (gzip, empty, ignore_older) never needs one, saving allocs.
212+
var srcID string
213+
ensureSrcID := func() string {
214+
if srcID == "" { // getFileIdentity never returns ""
215+
srcID = w.getFileIdentity(fd)
216+
}
217+
return srcID
218+
}
219+
220+
// closedHarvesters is empty in the steady state; this reconciliation is usually skipped.
221+
if w.hasClosedHarvesters() {
222+
w.reconcileClosedHarvester(&prevDesc, ensureSrcID())
228223
}
229-
w.closedHarvestersMutex.Unlock()
230224

231225
var e loginp.FSEvent
232226
switch {
233227
// the new size is smaller, the file was truncated
234228
case prevDesc.Info.Size() > fd.Info.Size():
235-
e = truncateEvent(path, fd, srcID)
229+
e = truncateEvent(path, fd, ensureSrcID())
236230
truncatedCount++
237231

238232
// the size is the same, timestamps are different, the file was touched
239233
case prevDesc.Info.Size() == fd.Info.Size() && prevDesc.Info.ModTime() != fd.Info.ModTime():
240234
if w.cfg.ResendOnModTime {
241-
e = truncateEvent(path, fd, srcID)
235+
e = truncateEvent(path, fd, ensureSrcID())
242236
truncatedCount++
243237
}
244238

245239
// the new size is larger, something was written.
246240
// If a harvester for this file was closed recently,
247241
// we use its state instead of the one we have cached.
248242
case prevDesc.SizeOrBytesIngested() < fd.Info.Size():
249-
e = writeEvent(path, fd, srcID)
243+
e = writeEvent(path, fd, ensureSrcID())
250244
writtenCount++
251245

252246
default:
253247
// For the delete feature we need to run the harvester for
254248
// files that have not changed until they're deleted.
255249
if w.cfg.SendNotChanged {
256-
e = notChangedEvent(path, fd, srcID)
250+
e = notChangedEvent(path, fd, ensureSrcID())
257251
}
258252
}
259253

@@ -266,12 +260,16 @@ func (w *fileWatcher) watch(ctx unison.Canceler) {
266260
}
267261
}
268262

263+
<<<<<<< HEAD
264+
=======
265+
// Record progress metrics for trackable, non-truncated files (tracksHarvesterProgress).
266+
if e.Op != loginp.OpTruncate && tracksHarvesterProgress(&fd, scanOpts) {
267+
harvesterFiles = append(harvesterFiles, loginp.HarvesterFile{ID: ensureSrcID(), Size: fd.Info.Size()})
268+
}
269+
270+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
269271
// delete from previous state to mark that we've seen the existing file again
270272
delete(w.prev, path)
271-
// Delete used state from closedHarvesters
272-
w.closedHarvestersMutex.Lock()
273-
delete(w.closedHarvesters, srcID)
274-
w.closedHarvestersMutex.Unlock()
275273
}
276274

277275
// remaining files in the prev map are the ones that are missing
@@ -309,6 +307,14 @@ func (w *fileWatcher) watch(ctx unison.Canceler) {
309307
case w.events <- createEvent(path, *fd, w.getFileIdentity(*fd)):
310308
createdCount++
311309
}
310+
<<<<<<< HEAD
311+
=======
312+
313+
// New files skip the main loop via early continue, so collect their metrics here.
314+
if tracksHarvesterProgress(fd, scanOpts) {
315+
harvesterFiles = append(harvesterFiles, loginp.HarvesterFile{ID: srcID, Size: fd.Info.Size()})
316+
}
317+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
312318
}
313319

314320
w.log.Debugw("File scan complete",
@@ -323,6 +329,54 @@ func (w *fileWatcher) watch(ctx unison.Canceler) {
323329
w.prev = paths
324330
}
325331

332+
<<<<<<< HEAD
333+
=======
334+
// hasClosedHarvesters reports whether any harvester-close notification awaits reconciliation.
335+
func (w *fileWatcher) hasClosedHarvesters() bool {
336+
w.closedHarvestersMutex.Lock()
337+
defer w.closedHarvestersMutex.Unlock()
338+
return len(w.closedHarvesters) > 0
339+
}
340+
341+
// reconcileClosedHarvester folds a recently-closed harvester's ingested offset (from
342+
// closedHarvesters) into prevDesc so a restarted harvester resumes from the right position.
343+
// It guards a close-during-backoff race that would otherwise withhold writes and lose lines.
344+
func (w *fileWatcher) reconcileClosedHarvester(prevDesc *loginp.FileDescriptor, id string) {
345+
w.closedHarvestersMutex.Lock()
346+
defer w.closedHarvestersMutex.Unlock()
347+
size, ok := w.closedHarvesters[id]
348+
if !ok {
349+
return
350+
}
351+
w.log.Debugf("Updating previous state because harvester was closed. '%s': %d", id, size)
352+
prevDesc.SetBytesIngested(size)
353+
delete(w.closedHarvesters, id)
354+
}
355+
356+
// tracksHarvesterProgress reports whether a file contributes to the harvester progress metrics.
357+
func tracksHarvesterProgress(fd *loginp.FileDescriptor, opts loginp.FileScanOptions) bool {
358+
return !fd.GZIP && fd.Info.Size() > 0 && !isFileIgnored(*fd, opts)
359+
}
360+
361+
// isFileIgnored returns true when a file is ignored, no matter the reason.
362+
func isFileIgnored(
363+
fd loginp.FileDescriptor,
364+
opts loginp.FileScanOptions,
365+
) bool {
366+
modTime := fd.Info.ModTime()
367+
368+
if opts.IgnoreOlder > 0 && opts.CurrentTime.Sub(modTime) > opts.IgnoreOlder {
369+
return true
370+
}
371+
372+
if !opts.IgnoreInactiveSince.IsZero() && modTime.Sub(opts.IgnoreInactiveSince) <= 0 {
373+
return true
374+
}
375+
376+
return false
377+
}
378+
379+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
326380
// getFileIdentity mimics the same algorithm used by the harvester to generate
327381
// the file identity to any given file.
328382
// See 'startHarvester' on internal/input-logfile/harvester.go.
@@ -359,8 +413,18 @@ func (w *fileWatcher) Event() loginp.FSEvent {
359413
return <-w.events
360414
}
361415

416+
<<<<<<< HEAD
362417
func (w *fileWatcher) GetFiles() map[string]loginp.FileDescriptor {
363418
return w.scanner.GetFiles()
419+
=======
420+
// GetFiles runs a one-off enumeration scan for the prospector's Init and
421+
// TakeOver phases. Unlike the watch loop it does not advance the scanner's
422+
// completedFingerprints set, so these pre-watch scans cannot suppress the
423+
// bridging raw header a still-growing entry needs to migrate its registry key
424+
// after a restart.
425+
func (w *fileWatcher) GetFiles(opts loginp.FileScanOptions) (map[string]loginp.FileDescriptor, loginp.FileScanMetrics) {
426+
return w.scanner.GetFiles(opts)
427+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
364428
}
365429

366430
type fingerprintConfig struct {
@@ -399,6 +463,21 @@ type fileScanner struct {
399463
hasher hash.Hash
400464
readBuffer []byte
401465
compression string
466+
<<<<<<< HEAD
467+
=======
468+
// completedFingerprints holds the paths whose fingerprint was already a
469+
// final SHA-256 on the previous watch-loop scan (growing mode only). The
470+
// bridging raw header is only useful on the scan a file crosses the
471+
// threshold, so for paths in this set toFileDescriptor skips recomputing it.
472+
// GetFiles itself is pure with respect to this set: only fileWatcher.watch
473+
// advances it (after each scan), so the enumeration-only scans the
474+
// prospector runs in Init/TakeOver cannot suppress the header a
475+
// still-growing entry needs to migrate across a restart.
476+
completedFingerprints map[string]struct{}
477+
478+
// lastCount is the number of unique files the previous scan produced.
479+
lastCount int
480+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
402481
}
403482

404483
func newFileScanner(logger *logp.Logger, paths []string, config fileScannerConfig, compression string) (*fileScanner, error) {
@@ -470,12 +549,27 @@ func (s *fileScanner) normalizeGlobPatterns() error {
470549

471550
// GetFiles returns a map of file descriptors by filenames that
472551
// match the configured paths.
552+
<<<<<<< HEAD
473553
func (s *fileScanner) GetFiles() map[string]loginp.FileDescriptor {
474554
fdByName := map[string]loginp.FileDescriptor{}
555+
=======
556+
func (s *fileScanner) GetFiles(opts loginp.FileScanOptions) (map[string]loginp.FileDescriptor, loginp.FileScanMetrics) {
557+
if opts.CurrentTime.IsZero() {
558+
opts.CurrentTime = time.Now()
559+
}
560+
561+
// Pre-size the per-scan maps from the previous scan's count.
562+
fdByName := make(map[string]loginp.FileDescriptor, s.lastCount)
563+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
475564
// used to determine if a symlink resolves in a already known target
476-
uniqueIDs := map[string]string{}
565+
uniqueIDs := make(map[string]string, s.lastCount)
477566
// used to filter out duplicate matches
567+
<<<<<<< HEAD
478568
uniqueFiles := map[string]struct{}{}
569+
=======
570+
uniqueFiles := make(map[string]struct{}, s.lastCount)
571+
scanMetrics := loginp.FileScanMetrics{}
572+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
479573

480574
for _, path := range s.paths {
481575
matches, err := filepath.Glob(path)
@@ -527,7 +621,13 @@ func (s *fileScanner) GetFiles() map[string]loginp.FileDescriptor {
527621
}
528622
}
529623

624+
<<<<<<< HEAD
530625
return fdByName
626+
=======
627+
scanMetrics.FilesUnique = int64(len(fdByName))
628+
s.lastCount = len(fdByName)
629+
return fdByName, scanMetrics
630+
>>>>>>> c05890c43 (filestream: reduce per-scan allocations (#51863))
531631
}
532632

533633
type ingestTarget struct {

0 commit comments

Comments
 (0)