process autodiscovery for krakend

Adds process-based autodiscovery for krakend, on top of the existing
container-based autodiscovery, using a CEL selector to match the
krakend process by name. Both flavors run as parametrized variants of
the same e2e test in a single environment: the process variant scopes
DD_AUTOCONFIG_EXCLUDE_FEATURES=docker to just that discovery invocation
(via the new ddev --env support) so that invocation's autodiscovery run
never learns about containers, letting the otherwise container-bound
krakend process be matched by process-based autodiscovery instead.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: vitkyrka

datadog_checks_dev/changelog.d/24238.added

@@ -0,0 +1 @@
+Add process autodiscovery E2E testing helpers.

datadog_checks_dev/datadog_checks/dev/docker.py

@@ -229,27 +229,45 @@ def _assert_no_log_patterns(logs: str, patterns: Sequence[str], candidate_index:
             )
 
 
+def _get_auto_conf_volume(check_root: str | os.PathLike[str] | None = None) -> str:
+    check_root = os.fspath(check_root or find_check_root(depth=2))
+    check_name = os.path.basename(check_root)
+    check_pkg = os.path.join(check_root, 'datadog_checks', check_name)
+    return f'{check_pkg}/data/auto_conf.yaml:/etc/datadog-agent/conf.d/{check_name}.d/auto_conf.yaml:ro'
+
+
 def get_e2e_discovery_metadata(
     check_root: str | os.PathLike[str] | None = None,
-) -> dict[str, list[str]]:
-    """Return Docker volume metadata for an e2e discovery run.
+) -> dict[str, list[str] | dict[str, str]]:
+    """Return metadata for an e2e discovery run.
 
-    Mounts the integration's ``auto_conf.yaml`` into the agent container.
+    Mounts the integration's ``auto_conf.yaml`` into the agent container, and grants
+    the capabilities needed for process-based autodiscovery to see processes running
+    in sibling containers (on top of the container-based autodiscovery already
+    enabled via the Docker socket mount).
 
     Use ``dd_agent_check_discovery`` alongside this metadata so that the static
     per-env config is temporarily replaced with an empty-instances file, leaving
-    ``auto_conf.yaml`` as the sole AD template driving config-discovery.
+    ``auto_conf.yaml`` as the sole AD template driving config-discovery. To exercise
+    process-based (rather than container-based) discovery specifically, pass
+    ``process=True`` to that single ``dd_agent_check_discovery`` call so only that
+    invocation's autodiscovery run stops seeing containers.
     """
-    check_root = os.fspath(check_root or find_check_root(depth=1))
-    check_name = os.path.basename(check_root)
-    check_pkg = os.path.join(check_root, 'datadog_checks', check_name)
-    auto_conf = os.path.join(check_pkg, 'data', 'auto_conf.yaml')
-
     return {
         'docker_volumes': [
-            f'{auto_conf}:/etc/datadog-agent/conf.d/{check_name}.d/auto_conf.yaml:ro',
+            _get_auto_conf_volume(check_root),
             '/var/run/docker.sock:/var/run/docker.sock:ro',
         ],
+        'env_vars': {
+            # Reduce the default service collection interval from 60s to speed
+            # up tests.
+            'DD_DISCOVERY_SERVICE_COLLECTION_INTERVAL': '10s',
+        },
+        # system-probe(-lite) needs these capabilities to read /proc entries of
+        # other processes for service discovery. Without them it falls back to
+        # only scanning the agent's own PID namespace, which finds no host
+        # processes.
+        'cap_add': ['SYS_PTRACE', 'DAC_READ_SEARCH'],
     }
 
 

datadog_checks_dev/datadog_checks/dev/plugin/pytest.py

@@ -203,6 +203,9 @@ def run_check(config=None, **kwargs):
         if 'times' in kwargs:
             kwargs['check_times'] = kwargs.pop('times')
 
+        for env_key, env_value in (kwargs.pop('env_vars', None) or {}).items():
+            check_command.extend(['--env', '{}={}'.format(env_key, env_value)])
+
         for key, value in kwargs.items():
             if value is not False:
                 check_command.append('--{}'.format(key.replace('_', '-')))
@@ -244,11 +247,30 @@ def dd_agent_check_discovery(dd_agent_check):
     Passes the empty-instances config required to let ``auto_conf.yaml`` drive
     autodiscovery, and sets sensible defaults for ``discovery_min_instances`` and
     ``discovery_timeout`` — all of which can be overridden per call.
+
+    Pass ``process=True`` to exercise process-based (rather than container-based)
+    autodiscovery for a process that is otherwise container-bound: this excludes
+    the ``docker`` feature for this invocation only (so this run's autodiscovery
+    never learns about any containers) and, unless overridden, extends the
+    default timeout to account for the minimum process age that process-based
+    autodiscovery requires before it recognizes a process as a service.
     """
     if not e2e_testing():
         pytest.skip('Not running E2E tests')
 
-    def run(*, discovery_min_instances=1, discovery_timeout=30, **kwargs):
+    def run(*, discovery_min_instances=1, discovery_timeout=None, process=False, **kwargs):
+        if discovery_timeout is None:
+            # Process autodiscovery needs to wait for the agent to recognize the
+            # process as a service which happens after a minimum process age of 1
+            # minute. This time can be reduced significantly once agent-side
+            # support for reducing the minimum age via configuration is available.
+            discovery_timeout = 90 if process else 30
+
+        if process:
+            env_vars = kwargs.pop('env_vars', None) or {}
+            env_vars.setdefault('DD_AUTOCONFIG_EXCLUDE_FEATURES', 'docker')
+            kwargs['env_vars'] = env_vars
+
         return dd_agent_check(
             {'init_config': {}, 'instances': []},
             discovery_min_instances=discovery_min_instances,

datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/auto_conf/cel_selector.yaml

@@ -0,0 +1,4 @@
+- name: cel_selector
+  description: CEL selector for autodiscovery.
+  enabled: true
+  example: {}

datadog_checks_dev/tests/test_docker.py

@@ -67,6 +67,10 @@ def test_get_e2e_discovery_metadata(tmp_path):
             f'{check_package_root}/data/auto_conf.yaml:/etc/datadog-agent/conf.d/test_check.d/auto_conf.yaml:ro',
             '/var/run/docker.sock:/var/run/docker.sock:ro',
         ],
+        'env_vars': {
+            'DD_DISCOVERY_SERVICE_COLLECTION_INTERVAL': '10s',
+        },
+        'cap_add': ['SYS_PTRACE', 'DAC_READ_SEARCH'],
     }
 
 

ddev/src/ddev/e2e/agent/docker.py

@@ -277,6 +277,9 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env
         for key, value in sorted(env_vars.items()):
             command.extend(['-e', f'{key}={value}'])
 
+        for cap in self.metadata.get('cap_add', []):
+            command.extend(['--cap-add', cap])
+
         # The docker `--add-host` command will reliably create entries in the `/etc/hosts` file,
         # otherwise, edits to that file will be overwritten on container restarts
         for host, ip in self.metadata.get('custom_hosts', []):

krakend/assets/configuration/spec.yaml

@@ -80,4 +80,9 @@ files:
     overrides:
       value.example:
       - krakend
+  - template: auto_conf/cel_selector
+    overrides:
+      cel_selector.example:
+        processes:
+          - "process.name == 'krakend'"
   - template: auto_conf/discovery

krakend/changelog.d/24238.added

@@ -0,0 +1 @@
+Add process autodiscovery support.

krakend/datadog_checks/krakend/data/auto_conf.yaml

@@ -6,6 +6,12 @@
 ad_identifiers:
   - krakend
 
+## CEL selector for autodiscovery.
+#
+cel_selector:
+  processes:
+  - process.name == 'krakend'
+
 ## Enables configuration discovery
 #
 discovery: {}

krakend/tests/test_e2e.py

@@ -25,14 +25,15 @@ def test_e2e(dd_agent_check, instance: InstanceBuilder):
 
 
 @pytest.mark.e2e
-def test_e2e_discovery(dd_agent_check_discovery, is_lab):
+@pytest.mark.parametrize('process', [False, True], ids=['container', 'process'])
+def test_e2e_discovery(dd_agent_check_discovery, is_lab, process):
     # In the lab environment we currently do not mount auto_conf.yaml into the
     # Agent container, so the Agent has no Autodiscovery template to trigger
     # config discovery.
     if is_lab:
         pytest.skip('lab does not currently support configuration discovery')
 
-    aggregator = dd_agent_check_discovery(check_rate=True)
+    aggregator = dd_agent_check_discovery(check_rate=True, process=process)
 
     metadata_metrics = get_metrics_from_metadata()