Skip to content

Commit 02dbfc6

Browse files
vitkyrkavitkyrka
authored
autodiscovery: fold namespace tracking into StaticConfigIndex, drop namespace override map
Reuse the existing StaticConfigIndex for host-wide generic-integration namespace tracking (storing each static openmetrics/prometheus config's namespace root alongside check names) instead of a separate GenericIntegrationNamespaceIndex, removing a parallel type and all of its plumbing through ServiceListernerDeps/autoconfig/configmgr. Also drop the hard-coded check-name-to-namespace override map (zk, gearmand): per https://datadoghq.atlassian.net/wiki/spaces/DSCVR/pages/7031522288/Config+discovery+conflict+with+generic+integrations this was abandoned in favor of assuming an integration's namespace root equals its check name, with the one known 7.83 exception (gearmand) handled separately rather than via a hard-coded map. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Co-authored-by: Vincent Whitchurch <vincent.whitchurch@datadoghq.com>
1 parent 0a56e69 commit 02dbfc6

16 files changed

Lines changed: 308 additions & 524 deletions

‎comp/core/autodiscovery/impl/autoconfig.go‎

Lines changed: 53 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -81,30 +81,29 @@ type Requires struct {
8181
// and then "schedule" or "unschedule" them by notifying subscribers. See the
8282
// module README for details.
8383
type AutoConfig struct {
84-
configPollers []*configPoller
85-
listeners []listeners.ServiceListener
86-
listenerCandidates map[string]*listenerCandidate
87-
listenerRetryStop chan struct{}
88-
schedulerController *scheduler.Controller
89-
listenerStop chan struct{}
90-
discoveryStop chan struct{}
91-
healthListening *health.Handle
92-
newService chan listeners.Service
93-
delService chan listeners.Service
94-
refreshConfig chan string
95-
store *store
96-
cfgMgr configManager
97-
serviceListenerFactories map[string]listeners.ServiceListenerFactory
98-
providerCatalog map[string]providerTypes.ConfigProviderFactory
99-
wmeta option.Option[workloadmeta.Component]
100-
taggerComp tagger.Component
101-
logs logComp.Component
102-
filterStore workloadfilter.Component
103-
telemetryStore *acTelemetry.Store
104-
healthPlatform healthplatformdef.Component
105-
staticConfigIndex *listeners.StaticConfigIndex
106-
genericIntegrationNamespaceIndex *listeners.GenericIntegrationNamespaceIndex
107-
serviceTracker adtypes.ServiceTracker
84+
configPollers []*configPoller
85+
listeners []listeners.ServiceListener
86+
listenerCandidates map[string]*listenerCandidate
87+
listenerRetryStop chan struct{}
88+
schedulerController *scheduler.Controller
89+
listenerStop chan struct{}
90+
discoveryStop chan struct{}
91+
healthListening *health.Handle
92+
newService chan listeners.Service
93+
delService chan listeners.Service
94+
refreshConfig chan string
95+
store *store
96+
cfgMgr configManager
97+
serviceListenerFactories map[string]listeners.ServiceListenerFactory
98+
providerCatalog map[string]providerTypes.ConfigProviderFactory
99+
wmeta option.Option[workloadmeta.Component]
100+
taggerComp tagger.Component
101+
logs logComp.Component
102+
filterStore workloadfilter.Component
103+
telemetryStore *acTelemetry.Store
104+
healthPlatform healthplatformdef.Component
105+
staticConfigIndex *listeners.StaticConfigIndex
106+
serviceTracker adtypes.ServiceTracker
108107

109108
// m covers the `configPollers`, `listenerCandidates`, `listeners`, and `listenerRetryStop`, but
110109
// not the values they point to.
@@ -205,33 +204,31 @@ func NewAutoConfigFromDeps(schedulerController *scheduler.Controller, secretReso
205204
// createNewAutoConfig creates an AutoConfig instance (without starting).
206205
func createNewAutoConfig(schedulerController *scheduler.Controller, secretResolver secrets.Component, wmeta option.Option[workloadmeta.Component], taggerComp tagger.Component, logs logComp.Component, telemetryComp telemetry.Component, filterStore workloadfilter.Component, hp healthplatformdef.Component, tracker adtypes.ServiceTracker) *AutoConfig {
207206
staticConfigIndex := listeners.NewStaticConfigIndex()
208-
genericIntegrationNamespaceIndex := listeners.NewGenericIntegrationNamespaceIndex()
209207
telStore := acTelemetry.NewStore(telemetryComp)
210-
cfgMgr := newReconcilingConfigManager(secretResolver, hp, staticConfigIndex, genericIntegrationNamespaceIndex, discovererPkg.NewPythonBridge(), telStore)
208+
cfgMgr := newReconcilingConfigManager(secretResolver, hp, staticConfigIndex, discovererPkg.NewPythonBridge(), telStore)
211209
ac := &AutoConfig{
212-
configPollers: make([]*configPoller, 0, 9),
213-
listenerCandidates: make(map[string]*listenerCandidate),
214-
listenerRetryStop: nil, // We'll open it if needed
215-
listenerStop: make(chan struct{}),
216-
discoveryStop: make(chan struct{}),
217-
healthListening: health.RegisterLiveness("ad-servicelistening"),
218-
newService: make(chan listeners.Service),
219-
delService: make(chan listeners.Service),
220-
refreshConfig: make(chan string, 100),
221-
store: newStore(),
222-
cfgMgr: cfgMgr,
223-
schedulerController: schedulerController,
224-
serviceListenerFactories: make(map[string]listeners.ServiceListenerFactory),
225-
providerCatalog: make(map[string]providerTypes.ConfigProviderFactory),
226-
wmeta: wmeta,
227-
taggerComp: taggerComp,
228-
logs: logs,
229-
filterStore: filterStore,
230-
telemetryStore: telStore,
231-
healthPlatform: hp,
232-
staticConfigIndex: staticConfigIndex,
233-
genericIntegrationNamespaceIndex: genericIntegrationNamespaceIndex,
234-
serviceTracker: tracker,
210+
configPollers: make([]*configPoller, 0, 9),
211+
listenerCandidates: make(map[string]*listenerCandidate),
212+
listenerRetryStop: nil, // We'll open it if needed
213+
listenerStop: make(chan struct{}),
214+
discoveryStop: make(chan struct{}),
215+
healthListening: health.RegisterLiveness("ad-servicelistening"),
216+
newService: make(chan listeners.Service),
217+
delService: make(chan listeners.Service),
218+
refreshConfig: make(chan string, 100),
219+
store: newStore(),
220+
cfgMgr: cfgMgr,
221+
schedulerController: schedulerController,
222+
serviceListenerFactories: make(map[string]listeners.ServiceListenerFactory),
223+
providerCatalog: make(map[string]providerTypes.ConfigProviderFactory),
224+
wmeta: wmeta,
225+
taggerComp: taggerComp,
226+
logs: logs,
227+
filterStore: filterStore,
228+
telemetryStore: telStore,
229+
healthPlatform: hp,
230+
staticConfigIndex: staticConfigIndex,
231+
serviceTracker: tracker,
235232
}
236233

237234
secretResolver.SubscribeToChanges(func(_, origin string, _ []string, oldValue, _ any) {
@@ -593,14 +590,13 @@ func (ac *AutoConfig) addListenerCandidates(listenerConfigs []pkgconfigsetup.Lis
593590
}
594591
log.Debugf("Listener %s was registered", c.Name)
595592
factoryOptions := listeners.ServiceListernerDeps{
596-
Config: &c,
597-
Telemetry: ac.telemetryStore,
598-
Filter: ac.filterStore,
599-
Tagger: ac.taggerComp,
600-
Wmeta: ac.wmeta,
601-
StaticConfigIndex: ac.staticConfigIndex,
602-
GenericIntegrationNamespaceIndex: ac.genericIntegrationNamespaceIndex,
603-
ServiceTracker: ac.serviceTracker,
593+
Config: &c,
594+
Telemetry: ac.telemetryStore,
595+
Filter: ac.filterStore,
596+
Tagger: ac.taggerComp,
597+
Wmeta: ac.wmeta,
598+
StaticConfigIndex: ac.staticConfigIndex,
599+
ServiceTracker: ac.serviceTracker,
604600
}
605601

606602
ac.listenerCandidates[c.Name] = &listenerCandidate{factory: factory, options: factoryOptions}

‎comp/core/autodiscovery/impl/configmgr.go‎

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -142,15 +142,13 @@ type reconcilingConfigManager struct {
142142

143143
// staticConfigIndex is a shared name set published to listeners so they
144144
// can deduplicate templates against static configs (see ProcessService).
145+
// It also tracks the namespace root (see listeners.NamespaceRoot) of every
146+
// scheduled static openmetrics/prometheus config, so a configuration-
147+
// discovery template can be suppressed when a host-wide generic-scraper
148+
// config already claims its metric namespace (see filterTemplatesDiscovery).
145149
// May be nil; callers that don't need cross-listener dedup can omit it.
146150
staticConfigIndex *listeners.StaticConfigIndex
147151

148-
// genericIntegrationNamespaceIndex is a shared namespace set published to
149-
// listeners so they can suppress configuration-discovery templates that
150-
// are redundant with a host-wide static openmetrics/prometheus config
151-
// (see filterTemplatesDiscovery). May be nil.
152-
genericIntegrationNamespaceIndex *listeners.GenericIntegrationNamespaceIndex
153-
154152
secretResolver secrets.Component
155153
healthPlatform healthplatformdef.Component
156154
telemetryStore *actelemetry.Store
@@ -161,19 +159,18 @@ type reconcilingConfigManager struct {
161159
var _ configManager = &reconcilingConfigManager{}
162160

163161
// newReconcilingConfigManager creates a new, empty reconcilingConfigManager.
164-
func newReconcilingConfigManager(secretResolver secrets.Component, healthPlatform healthplatformdef.Component, staticConfigIndex *listeners.StaticConfigIndex, genericIntegrationNamespaceIndex *listeners.GenericIntegrationNamespaceIndex, disco discoverer.ConfigDiscoverer, telStore *actelemetry.Store) configManager {
162+
func newReconcilingConfigManager(secretResolver secrets.Component, healthPlatform healthplatformdef.Component, staticConfigIndex *listeners.StaticConfigIndex, disco discoverer.ConfigDiscoverer, telStore *actelemetry.Store) configManager {
165163
cm := &reconcilingConfigManager{
166-
activeConfigs: map[string]integration.Config{},
167-
activeServices: map[string]serviceAndADIDs{},
168-
templatesByADID: newMultimap(),
169-
servicesByADID: newMultimap(),
170-
serviceResolutions: map[string]map[string]string{},
171-
scheduledConfigs: map[string]integration.Config{},
172-
staticConfigIndex: staticConfigIndex,
173-
genericIntegrationNamespaceIndex: genericIntegrationNamespaceIndex,
174-
secretResolver: secretResolver,
175-
healthPlatform: healthPlatform,
176-
telemetryStore: telStore,
164+
activeConfigs: map[string]integration.Config{},
165+
activeServices: map[string]serviceAndADIDs{},
166+
templatesByADID: newMultimap(),
167+
servicesByADID: newMultimap(),
168+
serviceResolutions: map[string]map[string]string{},
169+
scheduledConfigs: map[string]integration.Config{},
170+
staticConfigIndex: staticConfigIndex,
171+
secretResolver: secretResolver,
172+
healthPlatform: healthPlatform,
173+
telemetryStore: telStore,
177174
}
178175
initDiscoveryWorker(cm, disco)
179176
return cm
@@ -309,14 +306,14 @@ func (cm *reconcilingConfigManager) processNewConfig(config integration.Config)
309306
if len(decryptedConfig.Instances) > 0 {
310307
cm.staticConfigIndex.Add(config.Name)
311308

312-
// Also publish the namespace(s) of host-wide static
313-
// openmetrics/prometheus configs, so discovery templates for a
314-
// dedicated integration can be suppressed when such a config is
315-
// already claiming the same metric namespace (see
316-
// filterTemplatesDiscovery).
309+
// Also index the namespace root(s) of host-wide static
310+
// openmetrics/prometheus configs under the same set, so discovery
311+
// templates for a dedicated integration can be suppressed when
312+
// such a config is already claiming the same metric namespace
313+
// (see filterTemplatesDiscovery).
317314
if listeners.IsGenericIntegrationCheckName(config.Name) {
318315
for _, ns := range listeners.InstanceNamespaces(decryptedConfig) {
319-
cm.genericIntegrationNamespaceIndex.Add(ns)
316+
cm.staticConfigIndex.Add(listeners.NamespaceRoot(ns))
320317
}
321318
}
322319
}
@@ -378,7 +375,7 @@ func (cm *reconcilingConfigManager) processDelConfigs(configs []integration.Conf
378375

379376
if listeners.IsGenericIntegrationCheckName(config.Name) {
380377
for _, ns := range listeners.InstanceNamespaces(config) {
381-
cm.genericIntegrationNamespaceIndex.Remove(ns)
378+
cm.staticConfigIndex.Remove(listeners.NamespaceRoot(ns))
382379
}
383380
}
384381
}

‎comp/core/autodiscovery/impl/configmgr_discovery.go‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -198,17 +198,15 @@ func (cm *reconcilingConfigManager) applyDiscoveredConfigsLocked(svcID, tplDiges
198198
}
199199

200200
// conflictingGenericIntegrationLocked reports whether a generic-integration
201-
// (openmetrics/prometheus) config already claims a metric namespace matching
202-
// (or rooted in) checkName's expected namespace — either a sibling already
203-
// resolved for the same service (svcID, excluding tplDigest itself), or a
204-
// host-wide static config tracked in genericIntegrationNamespaceIndex. On a
205-
// match, it returns the conflicting config's Name (or "a static config" for
206-
// the host-wide case) and true. Must be called with cm.m held.
201+
// (openmetrics/prometheus) config already claims a metric namespace whose
202+
// root matches checkName — either a sibling already resolved for the same
203+
// service (svcID, excluding tplDigest itself), or a host-wide static config
204+
// tracked (by namespace root) in cm.staticConfigIndex. On a match, it returns
205+
// the conflicting config's Name (or "a static config" for the host-wide case)
206+
// and true. Must be called with cm.m held.
207207
func (cm *reconcilingConfigManager) conflictingGenericIntegrationLocked(svcID, tplDigest, checkName string) (string, bool) {
208-
expected := listeners.ExpectedNamespace(checkName)
209-
210-
if cm.genericIntegrationNamespaceIndex.HasMatch(expected) {
211-
return "a host-wide static config", true
208+
if cm.staticConfigIndex.Has(checkName) {
209+
return "a static config", true
212210
}
213211

214212
for otherDigest, otherResolvedDigest := range cm.serviceResolutions[svcID] {
@@ -220,7 +218,7 @@ func (cm *reconcilingConfigManager) conflictingGenericIntegrationLocked(svcID, t
220218
continue
221219
}
222220
for _, ns := range listeners.InstanceNamespaces(otherCfg) {
223-
if listeners.NamespaceMatches(ns, expected) {
221+
if listeners.NamespaceRoot(ns) == checkName {
224222
return otherCfg.Name, true
225223
}
226224
}

‎comp/core/autodiscovery/impl/configmgr_discovery_test.go‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ func TestConfigMgr_DiscoveryTemplate_RoutesThroughDiscoverer(t *testing.T) {
117117
disco := newStubDiscoverer(func(_, _ string) (string, error) {
118118
return `[{"instances":[{"openmetrics_endpoint":"http://%%host%%:8080/metrics"}]}]`, nil
119119
})
120-
cm := newReconcilingConfigManager(&mockResolver, nil, nil, nil, disco, nil).(*reconcilingConfigManager)
120+
cm := newReconcilingConfigManager(&mockResolver, nil, nil, disco, nil).(*reconcilingConfigManager)
121121
cm.start()
122122
defer cm.stop()
123123

@@ -172,7 +172,7 @@ func TestConfigMgr_DiscoveryTemplate_ServiceDeletionCancels(t *testing.T) {
172172
// forgotten.
173173
return "", assert.AnError
174174
})
175-
cm := newReconcilingConfigManager(&mockResolver, nil, nil, nil, disco, nil).(*reconcilingConfigManager)
175+
cm := newReconcilingConfigManager(&mockResolver, nil, nil, disco, nil).(*reconcilingConfigManager)
176176
cm.start()
177177
defer cm.stop()
178178

@@ -402,7 +402,7 @@ func makeDiscoveryCM(t *testing.T, payload string) (*reconcilingConfigManager, *
402402
t.Helper()
403403
mockResolver := MockSecretResolver{}
404404
disco := newStubDiscoverer(func(_, _ string) (string, error) { return payload, nil })
405-
cm := newReconcilingConfigManager(&mockResolver, nil, nil, nil, disco, nil).(*reconcilingConfigManager)
405+
cm := newReconcilingConfigManager(&mockResolver, nil, nil, disco, nil).(*reconcilingConfigManager)
406406
cm.start()
407407
t.Cleanup(cm.stop)
408408
return cm, disco
@@ -618,7 +618,7 @@ func TestConfigMgr_Lifecycle_HostPortsPassedToDiscoverer(t *testing.T) {
618618
capturedJSON.Store(serviceJSON)
619619
return `[{"instances":[{"port":8080}]}]`, nil
620620
})
621-
cm := newReconcilingConfigManager(&mockResolver, nil, nil, nil, disco, nil).(*reconcilingConfigManager)
621+
cm := newReconcilingConfigManager(&mockResolver, nil, nil, disco, nil).(*reconcilingConfigManager)
622622
cm.start()
623623
t.Cleanup(cm.stop)
624624

@@ -671,7 +671,7 @@ func TestConfigMgr_Lifecycle_HostMultiNetworkBridge(t *testing.T) {
671671
capturedJSON.Store(serviceJSON)
672672
return `[{"instances":[{"port":8080}]}]`, nil
673673
})
674-
cm := newReconcilingConfigManager(&mockResolver, nil, nil, nil, disco, nil).(*reconcilingConfigManager)
674+
cm := newReconcilingConfigManager(&mockResolver, nil, nil, disco, nil).(*reconcilingConfigManager)
675675
cm.start()
676676
t.Cleanup(cm.stop)
677677

@@ -716,7 +716,7 @@ func TestConfigMgr_Discovery_SuppressedBySiblingAlreadyResolved(t *testing.T) {
716716
disco := newStubDiscoverer(func(_, _ string) (string, error) {
717717
return `[{"instances":[{"openmetrics_endpoint":"http://%%host%%:9091/metrics"}]}]`, nil
718718
})
719-
cm := newReconcilingConfigManager(&mockResolver, nil, nil, listeners.NewGenericIntegrationNamespaceIndex(), disco, nil).(*reconcilingConfigManager)
719+
cm := newReconcilingConfigManager(&mockResolver, nil, nil, disco, nil).(*reconcilingConfigManager)
720720
cm.start()
721721
t.Cleanup(cm.stop)
722722

@@ -769,7 +769,7 @@ func TestConfigMgr_Discovery_SuppressedBySiblingAlreadyResolved(t *testing.T) {
769769
// TestConfigMgr_Discovery_SuppressedByHostWideStaticNamespace mirrors the
770770
// above but for the non-service-scoped case: a static (non-template)
771771
// openmetrics config claiming a matching namespace anywhere on the host,
772-
// tracked in genericIntegrationNamespaceIndex, must also suppress a slow
772+
// tracked (by namespace root) in staticConfigIndex, must also suppress a slow
773773
// discovery probe result at apply time.
774774
func TestConfigMgr_Discovery_SuppressedByHostWideStaticNamespace(t *testing.T) {
775775
errorStats = newAcErrorStats()
@@ -778,9 +778,12 @@ func TestConfigMgr_Discovery_SuppressedByHostWideStaticNamespace(t *testing.T) {
778778
disco := newStubDiscoverer(func(_, _ string) (string, error) {
779779
return `[{"instances":[{"openmetrics_endpoint":"http://%%host%%:9091/metrics"}]}]`, nil
780780
})
781-
nsIdx := listeners.NewGenericIntegrationNamespaceIndex()
782-
nsIdx.Add("krakend.api")
783-
cm := newReconcilingConfigManager(&mockResolver, nil, nil, nsIdx, disco, nil).(*reconcilingConfigManager)
781+
// The config manager would normally add the *root* of a static
782+
// generic-integration config's namespace (see listeners.NamespaceRoot),
783+
// e.g. "krakend" for "krakend.api" — simulate that directly here.
784+
idx := listeners.NewStaticConfigIndex()
785+
idx.Add("krakend")
786+
cm := newReconcilingConfigManager(&mockResolver, nil, idx, disco, nil).(*reconcilingConfigManager)
784787
cm.start()
785788
t.Cleanup(cm.stop)
786789

0 commit comments

Comments
 (0)