Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ go_library(
"//comp/core/workloadmeta/collectors/util",
"//comp/core/workloadmeta/def",
"//pkg/config/env",
"//pkg/config/helper",
"//pkg/errors",
"//pkg/util/containers",
"//pkg/util/kubernetes/kubelet",
Expand All @@ -37,10 +38,13 @@ dd_agent_go_test(
include_default = False,
deps = [
"//comp/core",
"//comp/core/config",
"//comp/core/workloadmeta/collectors/util",
"//comp/core/workloadmeta/def",
"//comp/core/workloadmeta/fx-mock",
"//comp/core/workloadmeta/mock",
"//pkg/config/env",
"//pkg/errors",
"//pkg/util/fxutil",
"//pkg/util/kubernetes",
"//pkg/util/kubernetes/kubelet",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util"
workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
"github.com/DataDog/datadog-agent/pkg/config/env"
"github.com/DataDog/datadog-agent/pkg/config/helper"
"github.com/DataDog/datadog-agent/pkg/errors"
"github.com/DataDog/datadog-agent/pkg/util/containers"
"github.com/DataDog/datadog-agent/pkg/util/kubernetes/kubelet"
Expand All @@ -39,6 +40,7 @@ type dependencies struct {

type collector struct {
id string
cfg config.Component
catalog workloadmeta.AgentType
store workloadmeta.Component
collectEphemeralContainers bool
Expand All @@ -57,6 +59,7 @@ func NewCollector(deps dependencies) (workloadmeta.CollectorProvider, error) {
return workloadmeta.CollectorProvider{
Collector: &collector{
id: collectorID,
cfg: deps.Config,
catalog: workloadmeta.NodeAgent,
collectEphemeralContainers: deps.Config.GetBool("include_ephemeral_containers"),
pullInterval: time.Duration(deps.Config.GetInt("kubelet_collector_pull_interval")) * time.Second,
Expand All @@ -78,6 +81,10 @@ func (c *collector) Start(_ context.Context, store workloadmeta.Component) error
return errors.NewDisabled(componentName, "Agent is not running on Kubernetes")
}

if helper.IsCLCRunner(c.cfg) {
return errors.NewDisabled(componentName, "Agent is a Cluster Checks Runner and has no reachable local Kubelet")
}

c.store = store

var err error
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package kubelet

import (
"context"
"maps"
"slices"
"sort"
Expand All @@ -21,16 +22,42 @@ import (
"k8s.io/apimachinery/pkg/api/resource"

"github.com/DataDog/datadog-agent/comp/core"
config "github.com/DataDog/datadog-agent/comp/core/config"
"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util"
workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
workloadmetafxmock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx-mock"
workloadmetamock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock"
pkgconfigenv "github.com/DataDog/datadog-agent/pkg/config/env"
pkgerrors "github.com/DataDog/datadog-agent/pkg/errors"
"github.com/DataDog/datadog-agent/pkg/util/fxutil"
"github.com/DataDog/datadog-agent/pkg/util/kubernetes"
"github.com/DataDog/datadog-agent/pkg/util/kubernetes/kubelet"
"github.com/DataDog/datadog-agent/pkg/util/pointer"
)

// TestDisabledCLCRunner verifies that the collector refuses to start (with a
// non-retriable "disabled" error) on Cluster Checks Runners, since they are
// never scheduled with a locally-reachable kubelet and would otherwise retry
// kubelet.GetKubeUtil() forever, generating noisy WARN logs.
func TestDisabledCLCRunner(t *testing.T) {
pkgconfigenv.SetFeatures(t, pkgconfigenv.Kubernetes)

cfg := config.NewMockWithOverrides(t, map[string]interface{}{
"clc_runner_enabled": true,
"config_providers": []map[string]interface{}{{"name": "clusterchecks"}},
})

c := &collector{
id: collectorID,
cfg: cfg,
catalog: workloadmeta.NodeAgent,
}

err := c.Start(context.Background(), nil)
require.Error(t, err)
assert.True(t, pkgerrors.IsDisabled(err))
}

func TestPodParser(t *testing.T) {
creationTimestamp := time.Date(2025, time.January, 1, 12, 0, 0, 0, time.UTC)
startTime := creationTimestamp.Add(time.Minute)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ go_library(
"//pkg/api/util",
"//pkg/clusteragent/api/v1:api",
"//pkg/config/env",
"//pkg/config/helper",
"//pkg/config/model",
"//pkg/config/utils",
"//pkg/errors",
Expand Down Expand Up @@ -62,12 +63,15 @@ dd_agent_go_test(
include_default = False,
deps = [
"//comp/core",
"//comp/core/config",
"//comp/core/workloadmeta/def",
"//comp/core/workloadmeta/fx-mock",
"//comp/core/workloadmeta/mock",
"//pkg/clusteragent/api/v1:api",
"//pkg/clusteragent/clusterchecks/types",
"//pkg/config/env",
"//pkg/config/mock",
"//pkg/errors",
"//pkg/proto/pbgo/core",
"//pkg/proto/pbgo/process",
"//pkg/util/cache",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util"
workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
"github.com/DataDog/datadog-agent/pkg/config/env"
"github.com/DataDog/datadog-agent/pkg/config/helper"
configutils "github.com/DataDog/datadog-agent/pkg/config/utils"
"github.com/DataDog/datadog-agent/pkg/errors"
"github.com/DataDog/datadog-agent/pkg/util/clusteragent"
Expand Down Expand Up @@ -83,6 +84,10 @@ func (c *collector) Start(ctx context.Context, store workloadmeta.Component) err
return errors.NewDisabled(componentName, "Agent is not running on Kubernetes")
}

if helper.IsCLCRunner(c.cfg) {
return errors.NewDisabled(componentName, "Agent is a Cluster Checks Runner and has no reachable local Kubelet")
}

c.store = store

var err error
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,17 @@ import (
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8sschema "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"

compconfig "github.com/DataDog/datadog-agent/comp/core/config"
workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
apiv1 "github.com/DataDog/datadog-agent/pkg/clusteragent/api/v1"
"github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks/types"
pkgconfigenv "github.com/DataDog/datadog-agent/pkg/config/env"
configmock "github.com/DataDog/datadog-agent/pkg/config/mock"
pkgerrors "github.com/DataDog/datadog-agent/pkg/errors"
pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/process"
"github.com/DataDog/datadog-agent/pkg/util/cache"
"github.com/DataDog/datadog-agent/pkg/util/clusteragent"
Expand Down Expand Up @@ -143,6 +147,28 @@ func (f *FakeDCAClient) SupportsNamespaceMetadataCollection() bool {
return f.LocalVersion.Major >= 7 && f.LocalVersion.Minor >= 55
}

// TestDisabledCLCRunner verifies that the collector refuses to start (with a
// non-retriable "disabled" error) on Cluster Checks Runners, since they are
// never scheduled with a locally-reachable kubelet and would otherwise retry
// kubelet.GetKubeUtil() forever, generating noisy WARN logs.
func TestDisabledCLCRunner(t *testing.T) {
pkgconfigenv.SetFeatures(t, pkgconfigenv.Kubernetes)

cfg := compconfig.NewMockWithOverrides(t, map[string]interface{}{
"clc_runner_enabled": true,
"config_providers": []map[string]interface{}{{"name": "clusterchecks"}},
})

c := &collector{
id: collectorID,
cfg: cfg,
}

err := c.Start(context.Background(), nil)
require.Error(t, err)
assert.True(t, pkgerrors.IsDisabled(err))
}

func TestCollector_selectPullBasedProvider(t *testing.T) {
tests := []struct {
name string
Expand Down
2 changes: 2 additions & 0 deletions comp/metadata/host/impl/hosttags/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ go_library(
deps = [
"//comp/core/tagger/tags",
"//pkg/config/env",
"//pkg/config/helper",
"//pkg/config/model",
"//pkg/config/utils",
"//pkg/gohai/cpu",
Expand All @@ -38,6 +39,7 @@ dd_agent_go_test(
srcs = ["tags_test.go"],
embed = [":hosttags"],
deps = [
"//pkg/config/env",
"//pkg/config/mock",
"//pkg/config/model",
"@com_github_stretchr_testify//assert",
Expand Down
7 changes: 6 additions & 1 deletion comp/metadata/host/impl/hosttags/tags.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/DataDog/datadog-agent/comp/core/tagger/tags"
"github.com/DataDog/datadog-agent/pkg/config/env"
"github.com/DataDog/datadog-agent/pkg/config/helper"
"github.com/DataDog/datadog-agent/pkg/config/model"
configUtils "github.com/DataDog/datadog-agent/pkg/config/utils"
gpu "github.com/DataDog/datadog-agent/pkg/gpu/tags"
Expand Down Expand Up @@ -66,7 +67,11 @@ func getProvidersDefinitions(conf model.Reader) map[string]*providerDef {
}

if env.IsFeaturePresent(env.Kubernetes) {
providers["kubernetes"] = &providerDef{10, k8s.NewKubeNodeTagsProvider(conf).GetTags}
// Cluster Checks Runners have no reachable local Kubelet, so node-label-based
// tags can never be retrieved and would otherwise retry (and log WARNs) forever.
if !helper.IsCLCRunner(conf) {
providers["kubernetes"] = &providerDef{10, k8s.NewKubeNodeTagsProvider(conf).GetTags}
}
providers["kubernetes_cluster_agent_tags"] = &providerDef{10, clusterinfo.GetClusterAgentStaticTags}
}

Expand Down
25 changes: 25 additions & 0 deletions comp/metadata/host/impl/hosttags/tags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/stretchr/testify/assert"

"github.com/DataDog/datadog-agent/pkg/config/env"
configmock "github.com/DataDog/datadog-agent/pkg/config/mock"
"github.com/DataDog/datadog-agent/pkg/config/model"
)
Expand Down Expand Up @@ -169,6 +170,30 @@ func TestSanitizeEUDMTagValue(t *testing.T) {
assert.Equal(t, "trim_me", sanitizeEUDMTagValue(" trim me "))
}

func TestGetProvidersDefinitionsSkipsKubernetesNodeTagsOnCLCRunner(t *testing.T) {
mockConfig, _ := setupTest(t)
env.SetFeatures(t, env.Kubernetes)

mockConfig.SetInTest("clc_runner_enabled", true)
mockConfig.SetInTest("config_providers", []map[string]interface{}{{"name": "clusterchecks"}})

providers := getProvidersDefinitions(mockConfig)
_, hasKubernetesNodeTags := providers["kubernetes"]
assert.False(t, hasKubernetesNodeTags, "kubernetes node-tags provider should be skipped on Cluster Checks Runners, which have no reachable local Kubelet")

_, hasClusterAgentTags := providers["kubernetes_cluster_agent_tags"]
assert.True(t, hasClusterAgentTags, "kubernetes_cluster_agent_tags provider should still be registered on Cluster Checks Runners")
}

func TestGetProvidersDefinitionsIncludesKubernetesNodeTagsOnNodeAgent(t *testing.T) {
mockConfig, _ := setupTest(t)
env.SetFeatures(t, env.Kubernetes)

providers := getProvidersDefinitions(mockConfig)
_, hasKubernetesNodeTags := providers["kubernetes"]
assert.True(t, hasKubernetesNodeTags, "kubernetes node-tags provider should be registered on a regular node Agent")
}

func TestHostTagsCache(t *testing.T) {
mockConfig, ctx := setupTest(t)
mockConfig.SetInTest("collect_gce_tags", false)
Expand Down
1 change: 1 addition & 0 deletions pkg/util/cloudproviders/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ go_library(
importpath = "github.com/DataDog/datadog-agent/pkg/util/cloudproviders",
visibility = ["//visibility:public"],
deps = [
"//pkg/config/helper",
"//pkg/config/setup",
"//pkg/util/cloudproviders/alibaba",
"//pkg/util/cloudproviders/azure",
Expand Down
19 changes: 19 additions & 0 deletions pkg/util/cloudproviders/cloudproviders.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"sync"
"time"

"github.com/DataDog/datadog-agent/pkg/config/helper"
configsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
"github.com/DataDog/datadog-agent/pkg/util/hostname/validate"
"github.com/DataDog/datadog-agent/pkg/util/kubelet"
Expand Down Expand Up @@ -134,16 +135,34 @@ var (
hostAliasLogOnce = true
)

// kubeletDependentHostAliasDetectors are the host-alias detectors that route through
// the shared kubelet client singleton. Cluster Checks Runners are Deployment
// replicas, not DaemonSets, so they are never colocated with a node's kubelet and can
// never reach it.
var kubeletDependentHostAliasDetectors = map[string]bool{
"kubelet": true,
kubernetes.CloudProviderName: true,
}

// GetHostAliases returns the hostname aliases and the name of the possible cloud providers
func GetHostAliases(ctx context.Context) ([]string, string) {
aliases := []string{}
cloudprovider := ""
isCLCRunner := helper.IsCLCRunner(configsetup.Datadog())

// cloud providers endpoints can take a few seconds to answer. We're using a WaitGroup to call all of them
// concurrently since GetHostAliases is called during the agent startup and is blocking.
var wg sync.WaitGroup

for _, hostAliasesDetector := range hostAliasesDetectors {
if isCLCRunner && kubeletDependentHostAliasDetectors[hostAliasesDetector.name] {
// Skip probing the kubelet client singleton: it can never succeed on a
// CCR and would otherwise trigger its exponential-backoff retrier and
// its "Impossible to reach Kubelet" warning for no benefit.
log.Debugf("Skipping %s Host Alias: Agent is a Cluster Checks Runner and has no reachable local Kubelet", hostAliasesDetector.name)
continue
}

wg.Add(1)
go func(hostAliasesDetector cloudProviderAliasesDetector) {
defer wg.Done()
Expand Down
49 changes: 49 additions & 0 deletions pkg/util/cloudproviders/cloudproviders_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,55 @@ func TestCloudProviderAliases(t *testing.T) {
assert.Contains(t, []string{"detector1", "detector3"}, cloudprovider)
}

// TestCloudProviderAliasesSkipsKubeletDependentDetectorsOnCLCRunner ensures
// that GetHostAliases never invokes the "kubelet" or "kubernetes" detectors
// on a Cluster Checks Runner, since CCRs are Deployment replicas (not
// DaemonSets) and can never reach a local kubelet.
func TestCloudProviderAliasesSkipsKubeletDependentDetectorsOnCLCRunner(t *testing.T) {
origDetectors := hostAliasesDetectors
defer func() { hostAliasesDetectors = origDetectors }()

config := configmock.New(t)
config.SetInTest("clc_runner_enabled", true)
config.SetInTest("config_providers", []map[string]interface{}{{"name": "clusterchecks"}})

kubeletCalled := false
kubernetesDetectorCalled := false
otherDetectorCalled := false

hostAliasesDetectors = []cloudProviderAliasesDetector{
{
name: "kubelet",
callback: func(_ context.Context) ([]string, error) {
kubeletCalled = true
return []string{"kubelet-alias"}, nil
},
},
{
name: "kubernetes",
callback: func(_ context.Context) ([]string, error) {
kubernetesDetectorCalled = true
return []string{"kubernetes-alias"}, nil
},
},
{
name: "other",
isCloudEnv: true,
callback: func(_ context.Context) ([]string, error) {
otherDetectorCalled = true
return []string{"other-alias"}, nil
},
},
}

aliases, cloudprovider := GetHostAliases(context.TODO())
assert.False(t, kubeletCalled, "kubelet host alias detector should be skipped on a Cluster Checks Runner")
assert.False(t, kubernetesDetectorCalled, "kubernetes host alias detector should be skipped on a Cluster Checks Runner")
assert.True(t, otherDetectorCalled, "non-kubelet-dependent host alias detectors should still run on a Cluster Checks Runner")
assert.Equal(t, []string{"other-alias"}, aliases)
assert.Equal(t, "other", cloudprovider)
}

func TestCloudProviderHostCCRID(t *testing.T) {
origDetectors := hostCCRIDDetectors
defer func() { hostCCRIDDetectors = origDetectors }()
Expand Down
2 changes: 2 additions & 0 deletions pkg/util/containers/metrics/kubelet/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ go_library(
deps = [
"//comp/core/workloadmeta/def",
"//pkg/config/env",
"//pkg/config/helper",
"//pkg/config/setup",
"//pkg/errors",
"//pkg/util/containers/metrics/provider",
"//pkg/util/kubernetes/kubelet",
Expand Down
Loading
Loading