[AutoOps] Flatten Cluster Settings (#48454) (#48474)

This removes the unnecessary exporting of `cluster_settings` by
only sending the settings _that apply_. For instance, if a setting
is both set as a transient and persistent setting, then only the
transient value matters.

(cherry picked from commit 93dc66d)

Co-authored-by: Chris Earle <pickypg@users.noreply.github.com>

Author: mergify[bot]

changelog/fragments/1768602570-autoops-flatten-settings.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Flatten AutoOps Cluster Settings to avoid unnecessary nesting and information
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: metricbeat
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/beats/issues/48453

x-pack/metricbeat/module/autoops_es/cluster_settings/_meta/test/cluster_settings.my-cluster.7.17.0.json

@@ -9,6 +9,7 @@
                     "enable": "true"
                 }
             },
+            "max_shards_per_node": "2000",
             "metadata": {
                 "display_name": "my-cluster"
             }
@@ -17,6 +18,9 @@
     "transient": {
         "action": {
             "auto_create_index": "false"
+        },
+        "cluster": {
+            "max_shards_per_node": "4000"
         }
     },
     "defaults": {
@@ -198,6 +202,11 @@
                 "rewrite_to_filter_by_filter": "true"
             }
         },
+        "serverless": {
+            "search": {
+                "search_power_min": "3"
+            }
+        },
         "path": {
             "data": ["/app/data"]
         },

x-pack/metricbeat/module/autoops_es/cluster_settings/_meta/test/cluster_settings.my-cluster.8.15.3.json

@@ -12,7 +12,7 @@
                 }
             },
             "metadata": {
-                "display_name": "my-cluster"
+                "display_name": "my-overridden-cluster-name"
             },
             "max_shards_per_node": "4000"
         },
@@ -23,11 +23,21 @@
         },
         "action": {
             "auto_create_index": ".ent-search-*-logs-*,-.ent-search-*,+*"
+        },
+        "serverless": {
+            "search": {
+                "search_power_min": "3"
+            }
         }
     },
     "transient": {
         "action": {
             "auto_create_index": ".ent-search-*-logs-*,-.ent-search-*,+*"
+        },
+        "cluster": {
+            "metadata": {
+                "display_name": "my-cluster"
+            }
         }
     },
     "defaults": {

x-pack/metricbeat/module/autoops_es/cluster_settings/data.go

@@ -7,6 +7,8 @@ package cluster_settings
 import (
 	"fmt"
 
+	"github.com/elastic/elastic-agent-libs/mapstr"
+
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/autoops_es/events"
 
 	s "github.com/elastic/beats/v7/libbeat/common/schema"
@@ -213,6 +215,25 @@ var (
 	}
 )
 
+// flattenSettings merges transient, persistent, and defaults settings
+// with precedence: transient > persistent > defaults
+func flattenSettings(metricSetFields mapstr.M) mapstr.M {
+	result := mapstr.M{}
+
+	// Apply in reverse precedence order (defaults first, transient last)
+	if defaults, ok := metricSetFields["defaults"].(mapstr.M); ok {
+		result.DeepUpdate(defaults)
+	}
+	if persistent, ok := metricSetFields["persistent"].(mapstr.M); ok {
+		result.DeepUpdate(persistent)
+	}
+	if transient, ok := metricSetFields["transient"].(mapstr.M); ok {
+		result.DeepUpdate(transient)
+	}
+
+	return result
+}
+
 func eventsMapping(r mb.ReporterV2, info *utils.ClusterInfo, settings *map[string]interface{}) error {
 	metricSetFields, err := schema.Apply(*settings)
 
@@ -222,7 +243,8 @@ func eventsMapping(r mb.ReporterV2, info *utils.ClusterInfo, settings *map[strin
 		return nil
 	}
 
-	r.Event(events.CreateEventWithRandomTransactionId(info, metricSetFields))
+	// Flatten settings with precedence: transient > persistent > defaults
+	r.Event(events.CreateEventWithRandomTransactionId(info, flattenSettings(metricSetFields)))
 
 	return nil
 }

x-pack/metricbeat/module/autoops_es/cluster_settings/data_test.go

@@ -29,13 +29,15 @@ func expectValidParsedData(t *testing.T, data metricset.FetcherData[map[string]i
 
 	// metrics exist
 	require.True(t, len(*event.MetricSetFields.FlattenKeys()) > 3)
-	require.NotNil(t, auto_ops_testing.GetObjectValue(event.MetricSetFields, "defaults"))
 
 	// filename includes the "display_name" as the second part
 	displayName := strings.Split(data.File, ".")[1]
 
-	require.Equal(t, displayName, auto_ops_testing.GetObjectValue(event.MetricSetFields, "persistent.cluster.metadata.display_name"))
-	require.ElementsMatch(t, []string{"/app/data"}, auto_ops_testing.GetObjectValue(event.MetricSetFields, "defaults.path.data"))
+	// Settings are flattened with precedence: transient > persistent > defaults
+	require.Equal(t, "4000", auto_ops_testing.GetObjectValue(event.MetricSetFields, "cluster.max_shards_per_node"))
+	require.Equal(t, displayName, auto_ops_testing.GetObjectValue(event.MetricSetFields, "cluster.metadata.display_name"))
+	require.ElementsMatch(t, []string{"/app/data"}, auto_ops_testing.GetObjectValue(event.MetricSetFields, "path.data"))
+	require.Equal(t, "3", auto_ops_testing.GetObjectValue(event.MetricSetFields, "serverless.search.search_power_min"))
 
 	// schema is expected to drop this field if it appears (it does in one file)
 	require.Nil(t, auto_ops_testing.GetObjectValue(event.MetricSetFields, "ignored_field"))