[Backport 7.82.x] autodiscovery: skip config discovery when a generic openmetrics/prometheus config already exists (#54270)

Backport 75b1f56 from #54263.

 ___

## What does this PR do?

Config discovery currently only skips proposing a discovered integration's check when another config *for the same integration name* already covers the service or host. In practice, many customers point the generic `openmetrics`/`prometheus` checks at a service instead of using the specific integration. Discovery doesn't recognize this as a conflict (different `Name`), so it still proposes the specific integration's check alongside the generic one, which can lead to duplicate — and for counters, doubled — metrics.

This implements the "quickest fix" option from [Config discovery conflict with generic integrations](https://datadoghq.atlassian.net/wiki/spaces/DSCVR/pages/7031522288/Config+discovery+conflict+with+generic+integrations): if an `openmetrics` or `prometheus` config already matches the service, or is scheduled host-wide, skip config discovery for *every* integration on that service/host, regardless of `Name`.

## Motivation

Avoid duplicate/doubled metrics when a generic `openmetrics`/`prometheus` config already covers the same service that config discovery would otherwise also configure via a specific integration.

https://datadoghq.atlassian.net/browse/DSCVR-626

## Describe how to test/verify your changes

Added unit tests in `comp/core/autodiscovery/listeners/service_test.go` (`TestServiceFilterTemplatesDiscovery`) covering:
- a discovery template is dropped when an `openmetrics` sibling config matches the same service, even though it has a different `Name`
- a discovery template is dropped when `prometheus` is configured host-wide (in the static config index)
- an unrelated discovery template on the same service/host is also dropped, since a generic integration match is treated as covering every integration

## Possible Drawbacks / Trade-offs

This can cause false-positive skips: e.g. if a generic `openmetrics`/`prometheus` config targets one container/host but doesn't actually scrape the same metrics as the discoverable integration, or targets a different service, discovery will still be skipped. See the wiki page for the full discussion of alternatives that reduce this risk at the cost of being slower to ship.

Co-authored-by: ali.benabdallah <ali.benabdallah@datadoghq.com>

Author: dd-octo-sts[bot]

comp/core/autodiscovery/listeners/common_filter.go

@@ -29,12 +29,24 @@ func filterTemplatesMatched(svc FilterableService, configs map[string]integratio
 	}
 }
 
+// genericIntegrationNames are check names for generic metric-scraping
+// integrations that customers commonly point at any service, potentially
+// under a different check name than the one discovery would configure. Their
+// presence for a service (or host) is treated as covering every integration,
+// since we can't tell whether they already scrape the same metrics.
+var genericIntegrationNames = map[string]struct{}{
+	"openmetrics": {},
+	"prometheus":  {},
+}
+
 // filterTemplatesDiscovery drops configuration-discovery templates that are
 // redundant with another config source for the same integration. Dropped when:
 //  1. another check template (Instances > 0) for the same integration Name has
 //     matched this same service (present in configs), or
 //  2. a scheduled non-template (static) config exists for the same Name
-//     (tracked in staticIdx).
+//     (tracked in staticIdx), or
+//  3. a generic integration (openmetrics/prometheus) config matched this
+//     service or is scheduled host-wide, regardless of its Name.
 //
 // Logs-only sibling templates (no Instances) are ignored — discovery covers
 // metric-check configuration and shouldn't be suppressed by an integration's
@@ -44,17 +56,28 @@ func filterTemplatesDiscovery(staticIdx *StaticConfigIndex, configs map[string]i
 		return
 	}
 	nonDiscoveryNames := map[string]struct{}{}
+	hasGenericSibling := false
 	for _, cfg := range configs {
 		if !cfg.IsDiscovery() && len(cfg.Instances) > 0 {
 			nonDiscoveryNames[cfg.Name] = struct{}{}
+			if _, ok := genericIntegrationNames[cfg.Name]; ok {
+				hasGenericSibling = true
+			}
+		}
+	}
+	hasGenericStatic := false
+	for name := range genericIntegrationNames {
+		if staticIdx.Has(name) {
+			hasGenericStatic = true
+			break
 		}
 	}
 	for digest, cfg := range configs {
 		if !cfg.IsDiscovery() {
 			continue
 		}
 		_, hasSibling := nonDiscoveryNames[cfg.Name]
-		if hasSibling || staticIdx.Has(cfg.Name) {
+		if hasGenericSibling || hasGenericStatic || hasSibling || staticIdx.Has(cfg.Name) {
 			log.Debugf("Ignoring discovery template %s from %s: another config source already covers this integration",
 				cfg.Name, cfg.Source)
 			delete(configs, digest)

comp/core/autodiscovery/listeners/service_test.go

@@ -194,6 +194,13 @@ func TestServiceFilterTemplatesDiscovery(t *testing.T) {
 		Instances:     []integration.Data{[]byte("port: 80")},
 		Source:        "file:nginx/auto_conf.yaml",
 	}
+	unrelatedDiscoveryTpl := integration.Config{
+		Name:          "nginx",
+		Provider:      names.File,
+		ADIdentifiers: []string{"nginx"},
+		Discovery:     &integration.DiscoveryConfig{},
+		Source:        "file:nginx/auto_conf.yaml",
+	}
 
 	containsDigests := func(configs map[string]integration.Config, want ...integration.Config) []string {
 		t.Helper()
@@ -268,6 +275,42 @@ func TestServiceFilterTemplatesDiscovery(t *testing.T) {
 			"logs-only sibling should be kept")
 	})
 
+	t.Run("discovery dropped when generic integration sibling matches same service", func(t *testing.T) {
+		openmetricsSibling := integration.Config{
+			Name:          "openmetrics",
+			Provider:      names.File,
+			ADIdentifiers: []string{"redis"},
+			Instances:     []integration.Data{[]byte("openmetrics_endpoint: http://%%host%%:9121/metrics")},
+			Source:        "file:openmetrics/conf.yaml",
+		}
+		configs := map[string]integration.Config{
+			discoveryTpl.Digest():          discoveryTpl,
+			openmetricsSibling.Digest():    openmetricsSibling,
+			unrelatedDiscoveryTpl.Digest(): unrelatedDiscoveryTpl,
+		}
+		mkSvc(NewStaticConfigIndex()).FilterTemplates(configs)
+		assert.NotContains(t, configs, discoveryTpl.Digest(),
+			"discovery template should be dropped when an openmetrics config matches the same service, even under a different Name")
+		assert.Contains(t, configs, openmetricsSibling.Digest(), "openmetrics config should be kept")
+		assert.NotContains(t, configs, unrelatedDiscoveryTpl.Digest(),
+			"unrelated discovery template should also be dropped: a generic integration match covers every integration")
+	})
+
+	t.Run("discovery dropped when generic integration is configured host-wide", func(t *testing.T) {
+		idx := NewStaticConfigIndex()
+		idx.Add("prometheus")
+
+		configs := map[string]integration.Config{
+			discoveryTpl.Digest():          discoveryTpl,
+			unrelatedDiscoveryTpl.Digest(): unrelatedDiscoveryTpl,
+		}
+		mkSvc(idx).FilterTemplates(configs)
+		assert.NotContains(t, configs, discoveryTpl.Digest(),
+			"discovery template should be dropped when a prometheus config is scheduled host-wide")
+		assert.NotContains(t, configs, unrelatedDiscoveryTpl.Digest(),
+			"unrelated discovery template should also be dropped: a generic integration match covers every integration")
+	})
+
 	t.Run("instrumentation check overrides matched file check", func(t *testing.T) {
 		fileTpl := integration.Config{
 			Name:      "redis",