Skip to content

Commit 441711e

Browse files
authored
Skip service discovery system-probe calls in cluster check runners (#54926)
### What does this PR do? Stops the Agent from attempting to reach a local system-probe socket when running as a Cluster Checks Runner (CCR), caused by: #48000 Cluster Checks Runners are Deployment replicas, not DaemonSets, so they are never colocated with a system-probe on the same node and have no `hostPath` mount for its socket (`/opt/datadog-agent/run/sysprobe.sock`). Despite that, the workloadmeta process collector's service-discovery goroutine unconditionally queried system-probe's `/services` endpoint on CCR, which can never succeed there, and kept retrying on every collection interval, logging a `no such file or directory` error each time. This PR adds a `helper.IsCLCRunner(...)` guard to `isServiceDiscoveryEnabled()` in `comp/core/workloadmeta/collectors/internal/process/process_collector.go`, so service discovery is treated as disabled on CCR regardless of the `discovery.enabled` system-probe config setting. This only gates the service-discovery goroutine and its system-probe calls; process collection, language detection, and GPU monitoring are unaffected. There is no change in behavior for the node Agent or Cluster Agent. The guard is scoped to `helper.IsCLCRunner(...)`. ### Motivation CONS-8502 Cluster Checks Runners have always attempted to query system-probe for service discovery on startup and on every collection interval, even though they're fundamentally incapable of reaching it (no local system-probe socket, since CCR isn't colocated per-node). This produces recurring `ERROR` logs about the missing socket, with no chance of ever succeeding. ### Describe how you validated your changes - Added a unit test case in `TestStartConfiguration` (`process_collector_test.go`) verifying that `clc_runner_enabled: true` with `config_providers: [clusterchecks]` disables the collector even when `discovery.enabled: true`. - Built a test image and deployed it as a real CCR pod in a cluster to reproduce and validate. Co-authored-by: patrick.liang <patrick.liang@datadoghq.com>
1 parent 2d00390 commit 441711e

3 files changed

Lines changed: 23 additions & 0 deletions

File tree

comp/core/workloadmeta/collectors/internal/process/BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ go_library(
2020
"//comp/core/telemetry/def",
2121
"//comp/core/telemetry/impl",
2222
"//comp/core/workloadmeta/def",
23+
"//pkg/config/helper",
2324
"//pkg/config/model",
2425
"//pkg/discovery/core",
2526
"//pkg/discovery/language",
@@ -63,6 +64,7 @@ go_library(
6364
"//comp/core/telemetry/def",
6465
"//comp/core/telemetry/impl",
6566
"//comp/core/workloadmeta/def",
67+
"//pkg/config/helper",
6668
"//pkg/config/model",
6769
"//pkg/discovery/core",
6870
"//pkg/discovery/language",

comp/core/workloadmeta/collectors/internal/process/process_collector.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828

2929
"github.com/DataDog/datadog-agent/comp/core/config"
3030
sysprobeconfig "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig/def"
31+
"github.com/DataDog/datadog-agent/pkg/config/helper"
3132
pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model"
3233
"github.com/DataDog/datadog-agent/pkg/languagedetection/languagemodels"
3334
"github.com/DataDog/datadog-agent/pkg/process/procutil"
@@ -195,6 +196,14 @@ func (c *collector) isProcessCollectionEnabled() bool {
195196

196197
// isServiceDiscoveryEnabled returns a boolean indicating if service discovery is enabled
197198
func (c *collector) isServiceDiscoveryEnabled() bool {
199+
// Cluster Checks Runners are Deployment replicas, not DaemonSets, so they
200+
// are never colocated with a system-probe on the same node and have no
201+
// hostPath mount for its socket. Service discovery can never succeed
202+
// there, so skip it entirely to avoid endlessly retrying system-probe
203+
// requests and spamming "no such file or directory" error logs.
204+
if helper.IsCLCRunner(c.config) {
205+
return false
206+
}
198207
return serviceDiscoveryEnabled(c.systemProbeConfig)
199208
}
200209

comp/core/workloadmeta/collectors/internal/process/process_collector_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,18 @@ func TestStartConfiguration(t *testing.T) {
570570
},
571571
expectedError: errors.NewDisabled(componentName, "process collection, service discovery, language collection, and GPU monitoring are disabled"),
572572
},
573+
{
574+
description: "service discovery enabled but disabled on CLC runner",
575+
configOverrides: map[string]interface{}{
576+
"process_config.process_collection.enabled": false,
577+
"clc_runner_enabled": true,
578+
"config_providers": []map[string]interface{}{{"name": "clusterchecks"}},
579+
},
580+
sysConfigOverrides: map[string]interface{}{
581+
"discovery.enabled": true,
582+
},
583+
expectedError: errors.NewDisabled(componentName, "process collection, service discovery, language collection, and GPU monitoring are disabled"),
584+
},
573585
} {
574586
t.Run(tc.description, func(t *testing.T) {
575587
cfg := config.NewMock(t)

0 commit comments

Comments
 (0)