Introduce the global DD_AGENTLESS_ENABLED flag

Author: bwoebi

.gitlab/package.yml

@@ -284,6 +284,8 @@ variables:
         WINDOWS_ARCH: ["amd64", "x86"]
   variables:
     WINDOWS_BUILD_IMAGE: "registry.ddbuild.io/images/mirror/dd-trace-py/windows-build:d4d18a67d43d400d3ecc6bda777a5e233a24f434@sha256:58c3e28646ddf8a1021a079a380c8840e51f0b88c1489a522beee6ec99413019"
+  before_script:
+    - git config --global core.longpaths true
   script:
     - bash .gitlab/scripts/build-wheel-windows.sh
   artifacts:

ddtrace/internal/native/_native.pyi

@@ -1363,6 +1363,9 @@ class RemoteConfigClient:
     """Native single-target remote config client (origin process).
 
     Children consume published configs via :class:`RemoteConfigReader` instead.
+
+    Passing ``api_key`` (together with ``site`` and ``hostname``) selects agentless
+    mode: configs are fetched from ``config.<site>`` rather than from the agent.
     """
 
     def __new__(
@@ -1381,6 +1384,9 @@ class RemoteConfigClient:
         process_tags: Optional[list[tuple[str, str]]] = None,
         timeout_ms: int = 5000,
         test_session_token: Optional[str] = None,
+        site: Optional[str] = None,
+        api_key: Optional[str] = None,
+        hostname: Optional[str] = None,
     ) -> "RemoteConfigClient": ...
     def add_capabilities(self, capabilities: list[RemoteConfigCapabilities]) -> None:
         """Add capabilities the client advertises to the agent."""
@@ -1401,6 +1407,13 @@ class RemoteConfigClient:
     def get_client_id(self) -> str:
         """The remote config client id (a UUID); stable for the process lifetime."""
         ...
+    def get_refresh_interval(self) -> float:
+        """Seconds to wait before the next poll.
+
+        Agentless mode follows the interval the backend recommends, refreshed on
+        every successful fetch; against the agent this is a fixed default.
+        """
+        ...
     def enable_shared_memory(self) -> None:
         """Enable cross-process broadcast. Call on the origin before forking."""
         ...

ddtrace/internal/remoteconfig/client.py

@@ -85,6 +85,7 @@ class RemoteConfigClient:
 
     def __init__(self) -> None:
         self.id = str(uuid.uuid4())
+        self.agentless = ddtrace.config._agentless_enabled
         self.agent_url = agent_config.trace_agent_url
 
         # Product callbacks for single subscriber architecture
@@ -104,6 +105,17 @@ def ensure_native(self) -> Any:
             from ddtrace.internal.native_runtime import get_native_runtime
 
             tracer_version = _pep440_to_semver()
+            # In agentless mode the API key selects the direct-to-backend fetcher;
+            # site and hostname identify the intake and this client to it.
+            agentless_kwargs = (
+                {
+                    "site": ddtrace.config._dd_site,
+                    "api_key": ddtrace.config._dd_api_key,
+                    "hostname": get_hostname(),
+                }
+                if self.agentless
+                else {}
+            )
             self._native = _NativeClient(
                 get_native_runtime(),
                 agent_url=str(self.agent_url),
@@ -117,6 +129,7 @@ def ensure_native(self) -> Any:
                 process_tags=_build_process_tags(),
                 timeout_ms=int(agent_config.trace_agent_timeout_seconds * 1000),
                 test_session_token=get_test_session_token(),
+                **agentless_kwargs,
             )
             if self._capability_values:
                 self._native.add_capabilities(self._capability_values)
@@ -125,6 +138,16 @@ def ensure_native(self) -> Any:
     def renew_id(self) -> None:
         self.id = str(uuid.uuid4())
 
+    def refresh_interval(self) -> Optional[float]:
+        """Seconds the backend wants us to wait before polling again.
+
+        Only agentless fetches carry a server-recommended interval; None means
+        "keep whatever interval the poller was configured with".
+        """
+        if self._native is None or not self.agentless:
+            return None
+        return self._native.get_refresh_interval()
+
     def register_callback(self, product_name: "RemoteConfigProduct", callback: RCCallback) -> None:
         self._product_callbacks[product_name] = callback
         log.debug("[%s][P: %s] Registered callback for product %s", os.getpid(), os.getppid(), product_name)

ddtrace/internal/remoteconfig/worker.py

@@ -40,7 +40,9 @@ def __init__(self) -> None:
             interval=ddconfig._remote_config_poll_interval, no_wait_at_start=True, autorestart=False
         )
         self._client = RemoteConfigClient()
-        self._state = self._agent_check
+        # Agentless fetches go straight to the Remote Config backend, so there is
+        # no agent to negotiate the v0.7/config endpoint with.
+        self._state = self._online if self._client.agentless else self._agent_check
         self._parent_id = os.getpid()
         self._capabilities_map: "dict[RemoteConfigCapabilities, RemoteConfigProduct]" = dict()
         self._consecutive_failures = 0
@@ -74,21 +76,31 @@ def _agent_check(self) -> None:
 
     def _online(self) -> None:
         with StopWatch() as sw:
-            if not self._client.request():
-                self._consecutive_failures += 1
-                if self._consecutive_failures >= self._MAX_CONSECUTIVE_FAILURES:
-                    self._state = self._agent_check
-                    self._consecutive_failures = 0
-                return
+            succeeded = self._client.request()
+
+        # The interval is the backend's to decide in agentless mode: it recommends
+        # a cadence on every successful fetch and a backoff after a failed one.
+        interval = self._client.refresh_interval()
+        if interval is not None and interval != self.interval:
+            log.debug("Remote Config poll interval set to %.3fs by the backend", interval)
+            self.interval = interval
+
+        if not succeeded:
+            self._consecutive_failures += 1
+            # Without an agent there is nothing to fall back to, so keep retrying
+            # on the backoff the native client asked for.
+            if self._consecutive_failures >= self._MAX_CONSECUTIVE_FAILURES and not self._client.agentless:
+                self._state = self._agent_check
+                self._consecutive_failures = 0
+            return
 
         self._consecutive_failures = 0
-        elapsed = sw.elapsed()
         log.debug(
             "[%d][P: %d] Datadog Remote Config Poller sent request to %s in %.5fs",
             os.getpid(),
             os.getppid(),
-            self._client.agent_url,
-            elapsed,
+            "the Remote Config intake" if self._client.agentless else self._client.agent_url,
+            sw.elapsed(),
         )
 
     def periodic(self) -> None:

ddtrace/internal/settings/_agentless.py

@@ -0,0 +1,59 @@
+import typing as t
+
+from ddtrace.internal.settings._core import DDConfig
+
+
+class AgentlessConfig(DDConfig):
+    # No __prefix__: _DD_APM_TRACING_AGENTLESS_ENABLED start with _.
+
+    api_key = DDConfig.v(t.Optional[str], "dd.api_key", default=None)
+    site = DDConfig.v(str, "dd.site", default="datadoghq.com")
+
+    #: The global switch. Every product setting below defaults to it.
+    enabled = DDConfig.v(bool, "dd.agentless.enabled", default=False)
+
+    # Raw per-product overrides; None means "follow the global switch". Read the
+    # resolved values below instead of these.
+    _apm_tracing = DDConfig.v(t.Optional[bool], "_dd.apm.tracing.agentless.enabled", default=None)
+    _ci_visibility = DDConfig.v(t.Optional[bool], "dd.civisibility.agentless.enabled", default=None)
+    _llmobs = DDConfig.v(t.Optional[bool], "dd.llmobs.agentless.enabled", default=None)
+
+    apm_tracing = DDConfig.d(bool, lambda c: c.enabled if c._apm_tracing is None else c._apm_tracing)
+    ci_visibility = DDConfig.d(bool, lambda c: c.enabled if c._ci_visibility is None else c._ci_visibility)
+    # LLM Observability keeps a third state: left unset (and with no global switch) it
+    # probes the agent at startup and decides then, so it must not collapse to False.
+    llmobs = DDConfig.d(t.Optional[bool], lambda c: True if c.enabled and c._llmobs is None else c._llmobs)
+
+    #: Whether anything at all submits agentlessly. Products without a transport
+    #: setting of their own (instrumentation telemetry) follow this.
+    any_enabled = DDConfig.d(bool, lambda c: bool(c.enabled or c.apm_tracing or c.ci_visibility or c.llmobs))
+
+    def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
+        super().__init__(*args, **kwargs)
+
+        if self.enabled and not self.api_key:
+            msg = (
+                "DD_AGENTLESS_ENABLED is set but DD_API_KEY is not. Agentless mode submits data "
+                "straight to the Datadog intake, which is not possible without an API key. "
+                "Set DD_API_KEY, or unset DD_AGENTLESS_ENABLED to submit through the agent."
+            )
+            raise ValueError(msg)
+
+    def reported_configuration(self) -> "list[tuple[str, t.Any, str]]":
+        """The (environment variable, effective value, origin) triples to report as telemetry.
+
+        Agentless config is resolved early, and we thus must explicitly report our config.
+        """
+        return [
+            (env_name, value, self.value_source(env_name))
+            for env_name, value in (
+                ("DD_AGENTLESS_ENABLED", self.enabled),
+                ("DD_SITE", self.site),
+                ("_DD_APM_TRACING_AGENTLESS_ENABLED", self.apm_tracing),
+                ("DD_CIVISIBILITY_AGENTLESS_ENABLED", self.ci_visibility),
+                ("DD_LLMOBS_AGENTLESS_ENABLED", self.llmobs),
+            )
+        ]
+
+
+config = AgentlessConfig()

ddtrace/internal/settings/_config.py

@@ -32,6 +32,7 @@
 from ddtrace.internal.serverless import in_azure_function
 from ddtrace.internal.serverless import in_gcp_function
 from ddtrace.internal.settings import env
+from ddtrace.internal.settings._agentless import config as agentless_config
 from ddtrace.internal.telemetry import get_config as _get_config
 from ddtrace.internal.telemetry import telemetry_writer
 from ddtrace.internal.telemetry import validate_and_report_otel_metrics_exporter_enabled
@@ -468,6 +469,14 @@ def __init__(self) -> None:
         self._debug_mode = _get_config("DD_TRACE_DEBUG", False, asbool, "OTEL_LOG_LEVEL")
         self._startup_logs_enabled = _get_config("DD_TRACE_STARTUP_LOGS", False, asbool)
 
+        self._dd_api_key = agentless_config.api_key
+        self._dd_site = agentless_config.site
+        self._agentless_enabled = agentless_config.enabled
+        for _name, _value, _origin in agentless_config.reported_configuration():
+            telemetry_writer.add_configuration(_name, _value, _origin)
+
+        self._dd_app_key = _get_config("DD_APP_KEY", report_telemetry=False)
+
         self._trace_rate_limit: int = _get_config("DD_TRACE_RATE_LIMIT", DEFAULT_SAMPLING_RATE_LIMIT, int)
         if self._trace_rate_limit != DEFAULT_SAMPLING_RATE_LIMIT and self._trace_sampling_rules in ("", "[]"):
             log.warning(
@@ -678,7 +687,7 @@ def __init__(self) -> None:
             log.warning("Invalid obfuscation pattern, disabling query string tracing", exc_info=True)
             self._http_tag_query_string = False  # Disable query string tagging if malformed obfuscation pattern
 
-        self._ci_visibility_agentless_enabled = _get_config("DD_CIVISIBILITY_AGENTLESS_ENABLED", False, asbool)
+        self._ci_visibility_agentless_enabled = agentless_config.ci_visibility
         self._ci_visibility_agentless_url = _get_config("DD_CIVISIBILITY_AGENTLESS_URL", "")
         self._ci_visibility_intelligent_testrunner_enabled = _get_config("DD_CIVISIBILITY_ITR_ENABLED", True, asbool)
         self._ci_visibility_log_level = _get_config("DD_CIVISIBILITY_LOG_LEVEL", "info")
@@ -701,11 +710,7 @@ def __init__(self) -> None:
 
         self._trace_methods = _get_config("DD_TRACE_METHODS")
 
-        self._dd_api_key = _get_config("DD_API_KEY", report_telemetry=False)
-        self._dd_app_key = _get_config("DD_APP_KEY", report_telemetry=False)
-        self._dd_site = _get_config("DD_SITE", "datadoghq.com")
-
-        self._llmobs_agentless_enabled = _get_config("DD_LLMOBS_AGENTLESS_ENABLED", None, asbool)
+        self._llmobs_agentless_enabled = agentless_config.llmobs
         self._llmobs_instrumented_proxy_urls = _get_config(
             "DD_LLMOBS_INSTRUMENTED_PROXY_URLS", None, lambda x: set(x.strip().split(","))
         )
@@ -761,7 +766,7 @@ def __init__(self) -> None:
             "DD_TRACE_EXPERIMENTAL_LONG_RUNNING_INITIAL_FLUSH_INTERVAL", default=10.0, modifier=float
         )
         # When True, traces are sent via the JSON span intake (agentless EvP), e.g. browser-intake-*.
-        self._trace_agentless_enabled = _get_config("_DD_APM_TRACING_AGENTLESS_ENABLED", False, asbool)
+        self._trace_agentless_enabled = agentless_config.apm_tracing
         if self._trace_agentless_enabled:
             log.debug(
                 "APM Agentless enabled: health metrics and client-side stats are disabled. "

ddtrace/internal/settings/_supported_configurations.py

@@ -9,6 +9,7 @@
         "DATADOG_TAGS",
         "DATADOG_TRACE_AGENT_HOSTNAME",
         "DD_ACTION_EXECUTION_ID",
+        "DD_AGENTLESS_ENABLED",
         "DD_AGENTLESS_LOG_SUBMISSION_ENABLED",
         "DD_AGENT_HOST",
         "DD_AGENT_PORT",

ddtrace/internal/settings/_telemetry.py

@@ -8,12 +8,9 @@
 class TelemetryConfig(DDConfig):
     __prefix__ = "dd"
 
-    API_KEY = DDConfig.v(t.Optional[str], "api_key", default=None)
-    SITE = DDConfig.v(str, "site", default="datadoghq.com")
     ENV = DDConfig.v(str, "env", default="")
     SERVICE = DDConfig.v(str, "service", default=detect_service(sys.argv) or "unnamed-python-service")
     VERSION = DDConfig.v(str, "version", default="")
-    AGENTLESS_MODE = DDConfig.v(bool, "civisibility.agentless.enabled", default=False)
     DEBUG = DDConfig.v(bool, "internal.telemetry.debug.enabled", default=False)
     HEARTBEAT_INTERVAL = DDConfig.v(float, "telemetry.heartbeat_interval", default=60.0)
     TELEMETRY_ENABLED = DDConfig.v(bool, "instrumentation_telemetry.enabled", default=True)

ddtrace/internal/settings/dynamic_instrumentation.py

@@ -30,18 +30,6 @@ def _derive_tags(c: DDConfig) -> str:
     return ",".join([":".join((k, v)) for (k, v) in _tags.items() if v is not None])
 
 
-def _resolve_agentless(c: DDConfig) -> bool:
-    """Whether the APM trace writer should run in agentless mode.
-
-    Falls back when agentless is requested but ``DD_API_KEY`` is unset.
-    """
-    if not ddconfig._trace_agentless_enabled:
-        return False
-    if not ddconfig._dd_api_key:
-        return False
-    return True
-
-
 def normalize_ident(ident: str) -> str:
     return ident.strip().lower().replace("_", "")
 
@@ -77,7 +65,9 @@ class DynamicInstrumentationConfig(DDConfig):
         help="Enable Dynamic Instrumentation",
     )
 
-    _agentless = DDConfig.d(bool, _resolve_agentless)
+    # A callable, not a value: this has to be read when the config is resolved, not when the
+    # class is defined.
+    _agentless = DDConfig.d(bool, lambda _: bool(ddconfig._agentless_enabled))
 
     metrics = DDConfig.v(
         bool,

ddtrace/internal/telemetry/writer.py

@@ -14,6 +14,7 @@
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.packages import is_user_code
 from ddtrace.internal.settings._agent import config as agent_config
+from ddtrace.internal.settings._agentless import config as agentless_config
 from ddtrace.internal.settings._telemetry import config
 
 from ...internal import atexit
@@ -165,9 +166,11 @@ def __init__(self, agentless: Optional[bool] = None) -> None:
         self._enabled = config.TELEMETRY_ENABLED
 
         if agentless is None:
-            agentless = config.AGENTLESS_MODE or config.API_KEY not in (None, "")
+            # An API key on its own says nothing about how data should be submitted;
+            # only an explicit agentless setting does.
+            agentless = agentless_config.any_enabled
 
-        if agentless and not config.API_KEY:
+        if agentless and not agentless_config.api_key:
             log.debug("Disabling telemetry: no Datadog API key found in agentless mode")
             self._enabled = False
 
@@ -239,8 +242,8 @@ def _build_worker(self) -> "TelemetryWorker":
             endpoint_url = "file://" + os.path.join(self._payload_file_dir, "")
             api_key = None
         elif self._agentless:
-            endpoint_url = _agentless_endpoint_url(config.SITE)
-            api_key = config.API_KEY
+            endpoint_url = _agentless_endpoint_url(agentless_config.site)
+            api_key = agentless_config.api_key
         else:
             endpoint_url = agent_config.trace_agent_url
             api_key = None
@@ -412,7 +415,7 @@ def enable_agentless_client(self, enabled: bool = True) -> None:
 
         self._agentless = enabled
 
-        if enabled and not config.API_KEY:
+        if enabled and not agentless_config.api_key:
             log.debug("Cannot switch telemetry to agentless mode: no Datadog API key found")
             return