filebeat/filestream: default include_file_fingerprint to true (#51793) (#51839)

Soft-revert of #50129, which made log.file.fingerprint in published
events opt-in via include_file_fingerprint (default false). Removing the
field by default is technically a breaking change that we don't
understand fully yet.

Users who want to save network, indexing, and storage costs at scale can
set include_file_fingerprint to false. A still-growing fingerprint is
never published.

After further benchmarks we can re-consider if we want to make the
breaking change and flip the default to false.

Reclassify the changelog fragment from breaking-change to enhancement.

(cherry picked from commit 8e39946)

Co-authored-by: Orestis Floros <orestis.floros@elastic.co>

Author: mergify[bot]

changelog/fragments/1776242399-filestream-include-file-fingerprint-opt-in.yaml

@@ -0,0 +1,12 @@
+kind: enhancement
+summary: "Filestream: include_file_fingerprint controls log.file.fingerprint in events."
+description: |
+  Add an `include_file_fingerprint` option (default: `true`) to the filestream
+  input. By default `log.file.fingerprint` is included in published events when
+  `file_identity.fingerprint` is configured, preserving the previous behaviour.
+  Set `include_file_fingerprint: false` to omit the field and reduce network,
+  indexing, and storage costs at scale. A still-growing fingerprint is never
+  added to events; only the completed SHA-256 is published.
+component: filebeat
+pr: https://github.com/elastic/beats/pull/50129
+issue: https://github.com/elastic/beats/issues/50724

changelog/fragments/1776242399-filestream-include-file-fingerprint.yaml

@@ -723,7 +723,7 @@ This option is not supported on Windows.
 ```yaml {applies_to}
 stack: ga 9.5.0
 ```
-Controls whether `log.file.fingerprint` is added to published events. Only takes effect when `file_identity.fingerprint` is configured. Defaults to `false`. The file path (`log.file.path`) is always present in events regardless of this setting.
+Controls whether `log.file.fingerprint` is added to published events. Only takes effect when `file_identity.fingerprint` is configured. Defaults to `true`. The file path (`log.file.path`) is always present in events regardless of this setting.
 
 ### `exclude_lines` [filebeat-input-filestream-exclude-lines]
 

docs/reference/filebeat/filebeat-input-filestream.md

@@ -463,10 +463,10 @@ filebeat.inputs:
   #include_file_owner_group_name: false
 
   # Whether to add log.file.fingerprint to published events. Only takes
-  # effect when file_identity.fingerprint is configured. Defaults to false.
+  # effect when file_identity.fingerprint is configured. Defaults to true.
   # The file path (log.file.path) is always present in events regardless
   # of this setting.
-  #include_file_fingerprint: false
+  #include_file_fingerprint: true
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering

filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl

@@ -882,10 +882,10 @@ filebeat.inputs:
   #include_file_owner_group_name: false
 
   # Whether to add log.file.fingerprint to published events. Only takes
-  # effect when file_identity.fingerprint is configured. Defaults to false.
+  # effect when file_identity.fingerprint is configured. Defaults to true.
   # The file path (log.file.path) is always present in events regardless
   # of this setting.
-  #include_file_fingerprint: false
+  #include_file_fingerprint: true
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering

filebeat/filebeat.reference.yml

@@ -71,7 +71,7 @@ type config struct {
 	IncludeFileOwnerName      bool `config:"include_file_owner_name"`
 	IncludeFileOwnerGroupName bool `config:"include_file_owner_group_name"`
 	// IncludeFileFingerprint controls whether log.file.fingerprint is added
-	// to published events. Disabled by default.
+	// to published events. Enabled by default.
 	IncludeFileFingerprint bool `config:"include_file_fingerprint"`
 
 	// -1 means that registry will never be cleaned, disabling clean_inactive.
@@ -162,7 +162,7 @@ func defaultConfig() config {
 		ReadUntilEOF:              loginp.DefaultReadUntilEOFConfig(),
 		IncludeFileOwnerName:      false,
 		IncludeFileOwnerGroupName: false,
-		IncludeFileFingerprint:    false,
+		IncludeFileFingerprint:    true,
 		CleanInactive:             -1,
 		CleanRemoved:              true,
 		HarvesterLimit:            0,

filebeat/input/filestream/config.go

@@ -558,12 +558,9 @@ func (inp *filestream) open(
 
 	r = readfile.NewStripNewline(r, inp.readerConfig.LineTerminator)
 
-	// log.file.fingerprint is opt-in (include_file_fingerprint, default false).
-	// A growing file's raw fingerprint material is the RAW hex of the file
-	// header, not a SHA-256, so publishing it would expose file content. Only
-	// publish the SHA-256 once the fingerprint is complete; below the threshold
-	// we publish no fingerprint at all. A renamed file may still carry the old
-	// path until the next open — that pre-existing limitation is unchanged here.
+	// Only publish the completed SHA-256. A still-growing fingerprint's material
+	// is the raw hex of the file header, not a hash, so publishing it would
+	// expose file content.
 	var fingerprint string
 	if inp.includeFileFingerprint && fs.desc.Fingerprint.Complete() {
 		fingerprint = fs.desc.Fingerprint.Sum

filebeat/input/filestream/input.go

@@ -117,29 +117,27 @@ func TestFilestreamIncludeFileFingerprint(t *testing.T) {
 		} `json:"log"`
 	}
 
+	const fingerprintIdentity = "file_identity.fingerprint: ~\n    prospector.scanner:\n      fingerprint.enabled: true"
+
 	tests := []struct {
 		name                   string
-		identityConfig         string
 		includeFileFingerprint string
 		expectFingerprint      bool
 	}{
 		{
 			name:                   "include_file_fingerprint_true",
-			identityConfig:         "file_identity.fingerprint: ~\n    prospector.scanner:\n      fingerprint.enabled: true",
 			includeFileFingerprint: "include_file_fingerprint: true",
 			expectFingerprint:      true,
 		},
 		{
 			name:                   "include_file_fingerprint_false",
-			identityConfig:         "file_identity.fingerprint: ~\n    prospector.scanner:\n      fingerprint.enabled: true",
 			includeFileFingerprint: "include_file_fingerprint: false",
 			expectFingerprint:      false,
 		},
 		{
 			name:                   "include_file_fingerprint_absent",
-			identityConfig:         "file_identity.fingerprint: ~\n    prospector.scanner:\n      fingerprint.enabled: true",
 			includeFileFingerprint: "",
-			expectFingerprint:      false,
+			expectFingerprint:      true,
 		},
 	}
 
@@ -174,7 +172,7 @@ output:
     path: ${path.home}
     filename: "output"
     rotate_on_startup: false
-`, tc.name, logFilePath, tc.identityConfig, tc.includeFileFingerprint)
+`, tc.name, logFilePath, fingerprintIdentity, tc.includeFileFingerprint)
 
 			filebeat.WriteConfigFile(cfg)
 			filebeat.Start()
@@ -186,10 +184,7 @@ output:
 			var failedFilePath []int
 			var failedFingerprint []int
 			for i, event := range events {
-				if tc.expectFingerprint && event.Log.File.Fingerprint == nil {
-					failedFingerprint = append(failedFingerprint, i)
-				}
-				if !tc.expectFingerprint && event.Log.File.Fingerprint != nil {
+				if (event.Log.File.Fingerprint != nil) != tc.expectFingerprint {
 					failedFingerprint = append(failedFingerprint, i)
 				}
 

filebeat/tests/integration/filestream_other_test.go

@@ -6,7 +6,6 @@ filebeat.inputs:
       fingerprint.enabled: true
     paths:
       - {{ .logfile }}
-    include_file_fingerprint: true
 
 output.file:
   path: ${path.home}

filebeat/tests/integration/testdata/run_as_filestream/fingerprint.yml

@@ -2582,10 +2582,10 @@ filebeat.inputs:
   #include_file_owner_group_name: false
 
   # Whether to add log.file.fingerprint to published events. Only takes
-  # effect when file_identity.fingerprint is configured. Defaults to false.
+  # effect when file_identity.fingerprint is configured. Defaults to true.
   # The file path (log.file.path) is always present in events regardless
   # of this setting.
-  #include_file_fingerprint: false
+  #include_file_fingerprint: true
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering