Merge branch 'main' into mrafi/remove-update-install-source

Author: mrafi97

.github/CODEOWNERS

@@ -29,6 +29,9 @@
 /MODULE.bazel*                          @DataDog/agent-build
 # Temporary during Bazel migration
 /**/BUILD.bazel                         @DataDog/agent-build
+# This should really be to a team that does not exist yet. It would cover supply chain in general.
+# For now, agent-build are the experts in that.
+/third_party/                           @DataDog/agent-build
 
 /CHANGELOG.rst                          @DataDog/agent-delivery
 /CHANGELOG-DCA.rst                      @DataDog/container-integrations @DataDog/container-platform
@@ -379,6 +382,7 @@
 /comp/snmpscan @DataDog/ndm-core
 /comp/softwareinventory @DataDog/windows-products
 /comp/syntheticstestscheduler @DataDog/synthetics-executing
+/comp/workloadselection @DataDog/injection-platform
 # END COMPONENTS
 
 # Additional notification to  @iglendd about Agent Telemetry changes for optional approval and governance acknowledgement

.gitignore

@@ -172,6 +172,8 @@ devenv/packer_cache
 test_output.json
 module_test_output.json
 e2e_test_output.json
+e2e_test_output.json.*.part
+junit-out-AgentFlavor.base-*.xml
 
 # doxygen doc & error log
 rtloader/doc

Dockerfiles/agent/Dockerfile

@@ -161,6 +161,7 @@ RUN --mount=from=artifacts,target=/artifacts \
  && rm -rf usr etc/init lib \
     go/ \
     opt/datadog-agent/embedded/bin/installer \
+    opt/datadog-agent/embedded/bin/dd-compile-policy \
     opt/datadog-agent/sources \
     opt/datadog-agent/embedded/share/doc \
     opt/datadog-agent/embedded/share/man \

cmd/agent/subcommands/run/command.go

@@ -34,6 +34,7 @@ import (
 	agenttelemetryfx "github.com/DataDog/datadog-agent/comp/core/agenttelemetry/fx"
 	"github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/datastreams"
 	ssistatusfx "github.com/DataDog/datadog-agent/comp/updater/ssistatus/fx"
+	workloadselectionfx "github.com/DataDog/datadog-agent/comp/workloadselection/fx"
 
 	haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx"
 	snmpscanfx "github.com/DataDog/datadog-agent/comp/snmpscan/fx"
@@ -534,6 +535,7 @@ func getSharedFxOption() fx.Option {
 		diagnosefx.Module(),
 		ipcfx.ModuleReadWrite(),
 		ssistatusfx.Module(),
+		workloadselectionfx.Module(),
 		workloadfilterfx.Module(),
 		connectivitycheckerfx.Module(),
 		configstreamfx.Module(),

comp/README.md

@@ -806,3 +806,9 @@ versions, installation dates, and other relevant details for inventory tracking.
 
 Package syntheticstestscheduler defines a synthetics scheduler component to run
 network tests based on remote config.
+
+### [comp/workloadselection](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/workloadselection)
+
+*Datadog Team*: injection-platform
+
+Package workloadselection listens to Remote Config to receive & apply workload selection configuration

comp/api/api/apiimpl/server.go

@@ -20,7 +20,7 @@ import (
 
 func startServer(listener net.Listener, srv *http.Server, name string) {
 	// Use a stack depth of 4 on top of the default one to get a relevant filename in the stdlib
-	logWriter, _ := pkglogsetup.NewLogWriter(5, log.ErrorLvl)
+	logWriter, _ := pkglogsetup.NewTLSHandshakeErrorWriter(5, log.ErrorLvl)
 
 	srv.ErrorLog = stdLog.New(logWriter, fmt.Sprintf("Error from the Agent HTTP server '%s': ", name), 0) // log errors to seelog
 

comp/core/config/setup.go

@@ -88,3 +88,8 @@ func setupConfig(config pkgconfigmodel.BuildableConfig, secretComp secrets.Compo
 
 	return nil
 }
+
+// GetInstallPath returns the install path for the agent
+func GetInstallPath() string {
+	return pkgconfigsetup.InstallPath
+}

comp/core/workloadfilter/catalog/common.go

@@ -9,7 +9,9 @@ package catalog
 // This file contains filter programs that can be shared across different entity types.
 
 import (
+	log "github.com/DataDog/datadog-agent/comp/core/log/def"
 	"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
+	"github.com/DataDog/datadog-agent/pkg/util/containers"
 )
 
 // AutodiscoveryAnnotations creates a CEL program for autodiscovery annotations.
@@ -35,3 +37,49 @@ func AutodiscoveryLogsAnnotations() program.FilterProgram {
 		ExcludePrefix: "logs_",
 	}
 }
+
+// LegacyContainerGlobalProgram creates a legacy filter program for global containerized filtering
+func LegacyContainerGlobalProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
+	return createLegacyContainerProgram("LegacyContainerGlobalProgram", cfg.ContainerInclude, cfg.ContainerExclude, logger)
+}
+
+// LegacyContainerMetricsProgram creates a legacy filter program for containerized metrics filtering
+func LegacyContainerMetricsProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
+	return createLegacyContainerProgram("LegacyContainerMetricsProgram", cfg.ContainerIncludeMetrics, cfg.ContainerExcludeMetrics, logger)
+}
+
+// LegacyContainerLogsProgram creates a legacy filter program for containerized logs filtering
+func LegacyContainerLogsProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
+	return createLegacyContainerProgram("LegacyContainerLogsProgram", cfg.ContainerIncludeLogs, cfg.ContainerExcludeLogs, logger)
+}
+
+// LegacyContainerACExcludeProgram creates a legacy filter program for containerized AC exclusion filtering
+func LegacyContainerACExcludeProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
+	return createLegacyContainerProgram("LegacyContainerACExcludeProgram", nil, cfg.ACExclude, logger)
+}
+
+// LegacyContainerACIncludeProgram creates a legacy filter program for containerized AC inclusion filtering
+func LegacyContainerACIncludeProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
+	return createLegacyContainerProgram("LegacyContainerACIncludeProgram", cfg.ACInclude, nil, logger)
+}
+
+// LegacyContainerSBOMProgram creates a legacy filter program for containerized SBOM filtering
+func LegacyContainerSBOMProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
+	return createLegacyContainerProgram("LegacyContainerSBOMProgram", cfg.SBOMContainerInclude, cfg.SBOMContainerExclude, logger)
+}
+
+func createLegacyContainerProgram(programName string, include, exclude []string, logger log.Component) program.FilterProgram {
+	var initErrors []error
+
+	filter, err := containers.NewFilter(containers.GlobalFilter, include, exclude)
+	if err != nil {
+		initErrors = append(initErrors, err)
+		logger.Warnf("Failed to create filter '%s': %v", programName, err)
+	}
+
+	return program.LegacyFilterProgram{
+		Name:                 programName,
+		Filter:               filter,
+		InitializationErrors: initErrors,
+	}
+}

comp/core/workloadfilter/catalog/container.go

@@ -7,92 +7,15 @@
 package catalog
 
 import (
-	"regexp"
-	"strings"
-
 	log "github.com/DataDog/datadog-agent/comp/core/log/def"
 	workloadfilter "github.com/DataDog/datadog-agent/comp/core/workloadfilter/def"
 	"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
 	"github.com/DataDog/datadog-agent/pkg/util/containers"
 )
 
-// LegacyContainerMetricsProgram creates a program for filtering container metrics
-func LegacyContainerMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyContainerMetricsProgram"
-	include := filterConfig.ContainerIncludeMetrics
-	exclude := filterConfig.ContainerExcludeMetrics
-	return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
-}
-
-// LegacyContainerLogsProgram creates a program for filtering container logs
-func LegacyContainerLogsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyContainerLogsProgram"
-	include := filterConfig.ContainerIncludeLogs
-	exclude := filterConfig.ContainerExcludeLogs
-	return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
-}
-
-// LegacyContainerACExcludeProgram creates a program for excluding containers via legacy `AC` filters
-func LegacyContainerACExcludeProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyContainerACExcludeProgram"
-	exclude := filterConfig.ACExclude
-	return createFromOldFilters(programName, nil, exclude, workloadfilter.ContainerType, logger)
-}
-
-// LegacyContainerACIncludeProgram creates a program for including containers via legacy `AC` filters
-func LegacyContainerACIncludeProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyContainerACIncludeProgram"
-	include := filterConfig.ACInclude
-	return createFromOldFilters(programName, include, nil, workloadfilter.ContainerType, logger)
-}
-
-// LegacyContainerGlobalProgram creates a program for filtering container globally
-func LegacyContainerGlobalProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyContainerGlobalProgram"
-	include := filterConfig.ContainerInclude
-	exclude := filterConfig.ContainerExclude
-	return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
-}
-
-// LegacyContainerSBOMProgram creates a program for filtering container SBOMs
-func LegacyContainerSBOMProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyContainerSBOMProgram"
-	include := filterConfig.SBOMContainerInclude
-	exclude := filterConfig.SBOMContainerExclude
-	return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
-}
-
 // ContainerPausedProgram creates a program for filtering paused containers
 func ContainerPausedProgram(_ *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "ContainerPausedProgram"
-	var initErrors []error
-
-	exclude := containers.GetPauseContainerExcludeList()
-
-	excludeRegex := make([]*regexp.Regexp, 0, len(exclude))
-	for _, pattern := range exclude {
-		pattern = strings.TrimPrefix(pattern, "image:")
-		regex, err := regexp.Compile(pattern)
-		if err != nil {
-			initErrors = append(initErrors, err)
-			logger.Warnf("Error compiling regex pattern for %s: %v", programName, err)
-			continue
-		}
-		excludeRegex = append(excludeRegex, regex)
-	}
-
-	return &program.RegexProgram{
-		Name:         programName,
-		ExcludeRegex: excludeRegex,
-		ExtractField: func(entity workloadfilter.Filterable) string {
-			container, ok := entity.(*workloadfilter.Container)
-			if !ok {
-				return ""
-			}
-			return container.GetImage()
-		},
-		InitializationErrors: initErrors,
-	}
+	return createLegacyContainerProgram("ContainerPausedProgram", nil, containers.GetPauseContainerExcludeList(), logger)
 }
 
 // ContainerCELMetricsProgram creates a program for filtering container metrics via CEL rules

comp/core/workloadfilter/catalog/kube_endpoints.go

@@ -12,22 +12,6 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
 )
 
-// LegacyEndpointsMetricsProgram creates a program for filtering endpoints metrics
-func LegacyEndpointsMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyEndpointsMetricsProgram"
-	include := filterConfig.ContainerIncludeMetrics
-	exclude := filterConfig.ContainerExcludeMetrics
-	return createFromOldFilters(programName, include, exclude, workloadfilter.EndpointType, logger)
-}
-
-// LegacyEndpointsGlobalProgram creates a program for filtering endpoints globally
-func LegacyEndpointsGlobalProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
-	programName := "LegacyEndpointsGlobalProgram"
-	includeList := filterConfig.GetLegacyContainerInclude()
-	excludeList := filterConfig.GetLegacyContainerExclude()
-	return createFromOldFilters(programName, includeList, excludeList, workloadfilter.EndpointType, logger)
-}
-
 // EndpointCELMetricsProgram creates a program for filtering endpoints metrics via CEL rules
 func EndpointCELMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
 	programName := "EndpointCELMetricsProgram"