[pull] master from DataDog:master

.ddev/config.toml

@@ -94,9 +94,6 @@ runners = { windows = ["windows-2022"] }
 [overrides.ci.tcp_check]
 platforms = ["linux", "windows"]
 
-[overrides.ci.tokumx]
-exclude = true
-
 [overrides.dependencies.licenses]
 # https://github.com/aerospike/aerospike-client-python/blob/master/LICENSE
 aerospike = ['Apache-2.0']

datadog_checks_base/tests/base/utils/discovery/test_discovery.py

@@ -209,6 +209,20 @@ def test_candidate_ports_prefers_hints_and_deduplicates():
     ]
 
 
+def test_dev_placeholder_field_constants_match_models():
+    """Guard the one fact datadog_checks_dev must hand-copy from base.
+
+    The discovery tooling cannot import datadog_checks_base, so it keeps the
+    Service/Port field names as constants used to validate candidate-template
+    placeholders. This test fails if those constants drift from the real models.
+    """
+    pytest.importorskip('datadog_checks.dev.tooling.configuration.discovery.registry')
+    from datadog_checks.dev.tooling.configuration.discovery.registry import PORT_FIELDS, SERVICE_FIELDS
+
+    assert SERVICE_FIELDS == set(Service.model_fields)
+    assert PORT_FIELDS == set(Port.model_fields)
+
+
 def test_discovery_strategy_passes_complete_contexts():
     from datadog_checks.base.utils.discovery import discovery_strategy
 

datadog_checks_dev/changelog.d/24126.added

@@ -0,0 +1 @@
+Add tooling support for generating configuration discovery files.

datadog_checks_dev/changelog.d/24145.fixed

@@ -0,0 +1 @@
+Remove obsolete TokuMX-specific validation workarounds.

datadog_checks_dev/changelog.d/24197.fixed

@@ -0,0 +1 @@
+Compare license headers against the actual base branch instead of always against ``origin/master``, fixing false validation failures for files that exist on a release branch but were deleted from master.

datadog_checks_dev/datadog_checks/dev/__init__.py

@@ -3,7 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 from .__about__ import __version__
 from .conditions import WaitFor
-from .docker import docker_run, get_docker_hostname
+from .docker import docker_run, get_docker_hostname, get_e2e_discovery_metadata
 from .env import environment_run
 from .errors import RetryError
 from .fs import chdir, get_here, temp_chdir, temp_dir

datadog_checks_dev/datadog_checks/dev/docker.py

@@ -1,9 +1,14 @@
 # (C) Datadog, Inc. 2018-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+import json
+import logging
 import os
+import re
+from collections.abc import Callable, Mapping, Sequence
 from contextlib import contextmanager
-from typing import Iterator  # noqa: F401
+from types import SimpleNamespace
+from typing import Any, Iterator  # noqa: F401
 from urllib.parse import urlparse
 
 from .conditions import CheckDockerLogs
@@ -20,6 +25,16 @@
     from contextlib2 import ExitStack
 
 
+CONTAINER_STABILITY_LOG_PATTERNS = (
+    r'error',
+    r'panic',
+    r'fatal',
+    r'segmentation fault',
+    r'core dumped',
+    r'Traceback',
+)
+
+
 def get_docker_hostname():
     """
     Determine the hostname Docker uses based on the environment, defaulting to `localhost`.
@@ -42,6 +57,202 @@ def get_container_ip(container_id_or_name):
     return run_command(command, capture='out', check=True).stdout.strip()
 
 
+def assert_all_discovery_candidates_stable(
+    dd_agent_check: Callable[..., Any],
+    check_cls: type[Any],
+    compose_file: str | os.PathLike[str] | None = None,
+    compose_service: str | None = None,
+    *,
+    project_name: str | None = None,
+    service_id: str | None = None,
+    dd_agent_check_kwargs: Mapping[str, Any] | None = None,
+    log_patterns: Sequence[str] = CONTAINER_STABILITY_LOG_PATTERNS,
+) -> None:
+    """Run generated discovery candidates directly and assert the target container stays stable."""
+    compose_service = compose_service or _get_default_compose_service()
+    container_id = _get_compose_container_id(compose_file, compose_service, project_name=project_name)
+    initial_state = _inspect_container(container_id)
+    previous_logs = _get_container_logs(container_id)
+
+    ports = tuple(SimpleNamespace(number=port, name='') for port in _get_container_ports(initial_state))
+    service = SimpleNamespace(
+        id=service_id or compose_service,
+        host=_get_container_ip_from_inspect(initial_state),
+        ports=ports,
+    )
+    candidates = tuple(check_cls.generate_configs(service))
+    if not candidates:
+        raise AssertionError(f'No discovery candidates generated for service {service.id!r}')
+
+    check_kwargs = {'check_rate': True}
+    if dd_agent_check_kwargs:
+        check_kwargs.update(dd_agent_check_kwargs)
+
+    for index, candidate in enumerate(candidates, 1):
+        logging.debug('Probing candidate #%d: %r', index, candidate)
+        try:
+            dd_agent_check(candidate, **check_kwargs)
+        except Exception:
+            logging.debug('Error probing candidate #%d: %r', index, candidate)
+            pass
+
+        current_container_id = _get_compose_container_id(compose_file, compose_service, project_name=project_name)
+        current_state = _inspect_container(current_container_id)
+        _assert_container_stable(initial_state, current_state, index)
+
+        current_logs = _get_container_logs(current_container_id)
+        new_logs = current_logs[len(previous_logs) :] if current_logs.startswith(previous_logs) else current_logs
+        for line in new_logs.splitlines():
+            logging.debug('New log line: %s', line)
+        _assert_no_log_patterns(new_logs, log_patterns, index)
+        previous_logs = current_logs
+
+
+def _get_compose_container_id(
+    compose_file: str | os.PathLike[str] | None, compose_service: str, *, project_name: str | None = None
+) -> str:
+    compose_file = compose_file or _get_default_compose_file()
+    project_name = project_name or _get_default_compose_project_name()
+
+    command = ['docker', 'compose']
+    if project_name:
+        command.extend(['-p', project_name])
+    command.extend(['-f', os.fspath(compose_file), 'ps', '-q', compose_service])
+
+    container_id = run_command(command, capture='out', check=True).stdout.strip()
+    if not container_id:
+        raise AssertionError(f'No container found for compose service {compose_service!r}')
+
+    return container_id
+
+
+def _get_default_compose_service() -> str:
+    docker_metadata = get_state('docker_compose_metadata', {})
+    if docker_metadata.get('service_name'):
+        return docker_metadata['service_name']
+
+    return os.path.basename(find_check_root(depth=2))
+
+
+def _get_default_compose_file() -> str:
+    docker_metadata = get_state('docker_compose_metadata', {})
+    if docker_metadata.get('compose_file'):
+        return docker_metadata['compose_file']
+
+    compose_file = os.path.join(find_check_root(depth=3), 'tests', 'docker', 'docker-compose.yml')
+    if os.path.exists(compose_file):
+        return compose_file
+
+    raise AssertionError(
+        'Could not determine the compose file. Pass compose_file explicitly or use docker_run with a compose file.'
+    )
+
+
+def _get_default_compose_project_name() -> str | None:
+    docker_metadata = get_state('docker_compose_metadata', {})
+    return docker_metadata.get('project_name') or os.getenv('COMPOSE_PROJECT_NAME')
+
+
+def _inspect_container(container_id: str) -> dict[str, Any]:
+    raw_inspect = run_command(['docker', 'inspect', container_id], capture='out', check=True).stdout
+    return json.loads(raw_inspect)[0]
+
+
+def _get_container_ip_from_inspect(inspect_data: Mapping[str, Any]) -> str:
+    networks = inspect_data.get('NetworkSettings', {}).get('Networks', {})
+    for network in networks.values():
+        ip_address = network.get('IPAddress')
+        if ip_address:
+            return ip_address
+
+    raise AssertionError(f"Could not determine container IP for {inspect_data.get('Name', '<unknown>')}")
+
+
+def _get_container_ports(inspect_data: Mapping[str, Any]) -> list[int]:
+    ports: set[int] = set()
+    exposed_ports = inspect_data.get('Config', {}).get('ExposedPorts') or {}
+    network_ports = inspect_data.get('NetworkSettings', {}).get('Ports') or {}
+
+    for raw_port in list(exposed_ports) + list(network_ports):
+        port, _, protocol = raw_port.partition('/')
+        if protocol and protocol != 'tcp':
+            continue
+        try:
+            ports.add(int(port))
+        except ValueError:
+            continue
+
+    if not ports:
+        raise AssertionError(f"No TCP ports found for container {inspect_data.get('Name', '<unknown>')}")
+
+    return sorted(ports)
+
+
+def _get_container_logs(container_id: str) -> str:
+    result = run_command(['docker', 'logs', container_id], capture=True)
+    return result.stdout + result.stderr
+
+
+def _assert_container_stable(
+    initial_state: Mapping[str, Any], current_state: Mapping[str, Any], candidate_index: int
+) -> None:
+    initial_id = initial_state['Id']
+    current_id = current_state['Id']
+    if current_id != initial_id:
+        raise AssertionError(
+            f'Container changed while probing candidate #{candidate_index}: {initial_id} -> {current_id}'
+        )
+
+    state = current_state.get('State', {})
+    if not state.get('Running'):
+        raise AssertionError(f'Container is not running after probing candidate #{candidate_index}')
+
+    initial_restart_count = initial_state.get('RestartCount', 0)
+    current_restart_count = current_state.get('RestartCount', 0)
+    if current_restart_count != initial_restart_count:
+        raise AssertionError(
+            f'Container restart count changed while probing candidate #{candidate_index}: '
+            f'{initial_restart_count} -> {current_restart_count}'
+        )
+
+    health = state.get('Health')
+    if health and health.get('Status') != 'healthy':
+        raise AssertionError(f"Container health is {health.get('Status')!r} after probing candidate #{candidate_index}")
+
+
+def _assert_no_log_patterns(logs: str, patterns: Sequence[str], candidate_index: int) -> None:
+    for pattern in patterns:
+        match = re.search(pattern, logs, re.IGNORECASE)
+        if match:
+            raise AssertionError(
+                f'Container logs matched {pattern!r} after probing candidate #{candidate_index}: {match.group(0)!r}'
+            )
+
+
+def get_e2e_discovery_metadata(
+    check_root: str | os.PathLike[str] | None = None,
+) -> dict[str, list[str]]:
+    """Return Docker volume metadata for an e2e discovery run.
+
+    Mounts the integration's ``auto_conf.yaml`` into the agent container.
+
+    Use ``dd_agent_check_discovery`` alongside this metadata so that the static
+    per-env config is temporarily replaced with an empty-instances file, leaving
+    ``auto_conf.yaml`` as the sole AD template driving config-discovery.
+    """
+    check_root = os.fspath(check_root or find_check_root(depth=1))
+    check_name = os.path.basename(check_root)
+    check_pkg = os.path.join(check_root, 'datadog_checks', check_name)
+    auto_conf = os.path.join(check_pkg, 'data', 'auto_conf.yaml')
+
+    return {
+        'docker_volumes': [
+            f'{auto_conf}:/etc/datadog-agent/conf.d/{check_name}.d/auto_conf.yaml:ro',
+            '/var/run/docker.sock:/var/run/docker.sock:ro',
+        ],
+    }
+
+
 def compose_file_active(compose_file):
     """
     Returns a `bool` indicating whether or not a compose file has any active services.
@@ -221,6 +432,16 @@ def docker_run(
                     'mount_logs: expected True, a list or a set, but got {}'.format(type(mount_logs).__name__)
                 )
 
+    if compose_file is not None:
+        save_state(
+            'docker_compose_metadata',
+            {
+                'compose_file': compose_file,
+                'project_name': (env_vars or {}).get('COMPOSE_PROJECT_NAME') or os.getenv('COMPOSE_PROJECT_NAME'),
+                'service_name': service_name,
+            },
+        )
+
     with environment_run(
         up=set_up,
         down=tear_down,

datadog_checks_dev/datadog_checks/dev/plugin/pytest.py

@@ -237,6 +237,28 @@ def run_check(config=None, **kwargs):
         yield run_check
 
 
+@pytest.fixture
+def dd_agent_check_discovery(dd_agent_check):
+    """Wrapper around ``dd_agent_check`` for config-discovery e2e tests.
+
+    Passes the empty-instances config required to let ``auto_conf.yaml`` drive
+    autodiscovery, and sets sensible defaults for ``discovery_min_instances`` and
+    ``discovery_timeout`` — all of which can be overridden per call.
+    """
+    if not e2e_testing():
+        pytest.skip('Not running E2E tests')
+
+    def run(*, discovery_min_instances=1, discovery_timeout=30, **kwargs):
+        return dd_agent_check(
+            {'init_config': {}, 'instances': []},
+            discovery_min_instances=discovery_min_instances,
+            discovery_timeout=discovery_timeout,
+            **kwargs,
+        )
+
+    return run
+
+
 @pytest.fixture
 def dd_run_check():
     checks = {}

datadog_checks_dev/datadog_checks/dev/tooling/catalog_const.py

@@ -67,7 +67,6 @@
     'system_swap',  # system
     'tcp_check',  # remote connection
     'tls',  # remote connection
-    'tokumx',  # eoled, only available in py2
     'windows_service',  # OS
     'wmi_check',  # base class
 }

datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/license_headers.py

@@ -15,7 +15,7 @@
     echo_warning,
 )
 from datadog_checks.dev.tooling.constants import get_root
-from datadog_checks.dev.tooling.license_headers import validate_license_headers
+from datadog_checks.dev.tooling.license_headers import build_get_previous, validate_license_headers
 from datadog_checks.dev.tooling.testing import process_checks_option
 from datadog_checks.dev.tooling.utils import complete_valid_checks
 
@@ -25,15 +25,14 @@
     "mysql": ["datadog_checks/mysql/databases_data.py"],  # deleted in master but still in release branches
     "php_fpm": ["datadog_checks/php_fpm/vendor"],
     "snmp": ["tests/mibs"],
-    "tokumx": ["datadog_checks/tokumx/vendor"],
 }
 
 
 @click.command(context_settings=CONTEXT_SETTINGS, short_help='Validate license headers in python files')
 @click.argument('check', shell_complete=complete_valid_checks, required=False)
 @click.option('--fix', is_flag=True, help='Attempt to fix errors')
 @click.pass_context
-def license_headers(ctx, check, fix):
+def license_headers(ctx: click.Context, check: str | None, fix: bool) -> None:
     """Validate license headers in python code files.
 
     If `check` is specified, only the check will be validated, if check value is 'changed' will only apply to changed
@@ -52,11 +51,13 @@ def license_headers(ctx, check, fix):
     total_errors = 0
     total_fixes = 0
 
+    get_previous = build_get_previous()
+
     for check_name in checks:
         path_to_check = root / check_name
         ignores = [pathlib.Path(p) for p in IGNORES.get(check_name, [])]
         ignores.extend([pathlib.Path(p) for p in IGNORES.get("all")])
-        errors = validate_license_headers(path_to_check, ignore=ignores, repo_root=root)
+        errors = validate_license_headers(path_to_check, ignore=ignores, repo_root=root, get_previous=get_previous)
 
         for err in errors:
             echo_failure(f'{check_name}/{err.path}: {err.message}')

datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/models.py

@@ -40,7 +40,6 @@
 
 INTEGRATIONS_WITHOUT_MODELS = {
     'snmp',  # Deprecated
-    'tokumx',  # Python 2 only
 }
 
 
@@ -166,7 +165,7 @@ def models(ctx, check, sync, verbose):
                 current_model_file_lines = read_file_lines(model_file_path)
 
                 if model_file in CUSTOM_FILES and (len(current_model_file_lines) + 1) > len(license_header_lines):
-                    # validators.py and deprecations.py are custom files, they should only be rendered the first time
+                    # Custom files should only be rendered the first time.
                     continue
 
             if not is_community_check: