flink: add container-based config discovery support (#24485)

* flink: add container-based config discovery support

Adds a discovery.strategies block using discovery/openmetrics_from_ports
(port 9249), an auto_conf.yaml ad_identifiers entry for the flink image,
E2E discovery metadata wiring, and test_e2e_discovery/
test_e2e_discovery_all_candidates tests covering both the jobmanager and
taskmanager roles, which share the same container image.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Add changelog entry for #24485

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* fixup

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Author: vitkyrka

flink/assets/configuration/spec.yaml

@@ -2,6 +2,12 @@ name: flink
 fleet_configurable: true
 files:
 - name: flink.yaml
+  discovery:
+    strategies:
+    - template: discovery/openmetrics_from_ports
+      overrides:
+        port_hints:
+        - 9249
   options:
   - template: init_config
     options:
@@ -23,3 +29,10 @@ files:
       path: /var/log/flink.log
       source: flink
       service: <SERVICE>
+- name: auto_conf.yaml
+  options:
+  - template: ad_identifiers
+    overrides:
+      value.example:
+      - flink
+  - template: auto_conf/discovery

flink/changelog.d/24485.added

@@ -0,0 +1 @@
+Add container-based config discovery support.

flink/datadog_checks/flink/config_models/discovery.py

@@ -0,0 +1,42 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+# This file is autogenerated.
+# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
+#     ddev -x validate config -s <INTEGRATION_NAME>
+#     ddev -x validate models -s <INTEGRATION_NAME>
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+from datadog_checks.base.utils.discovery import Service, candidate_ports
+from datadog_checks.flink.config_models import discovery_overrides
+from datadog_checks.flink.config_models.instance import InstanceConfig
+from datadog_checks.flink.config_models.shared import SharedConfig
+
+
+def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]:
+    shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump(
+        by_alias=True, mode='json', exclude_none=True
+    )
+    # discovery[0]: from_ports
+    for port in candidate_ports(service, [9249]):
+        ctx = {'port': port}
+        instance_data = {
+            'openmetrics_endpoint': 'http://{service.host}:{port.number}/metrics'.format(service=service, **ctx),
+        }
+        instance = InstanceConfig.model_validate(
+            instance_data, context={'configured_fields': frozenset(instance_data)}
+        ).model_dump(by_alias=True, mode='json', exclude_none=True)
+        yield {'init_config': shared, 'instances': [instance]}
+
+
+def candidates(service: Service) -> Iterator[dict[str, Any]]:
+    override = getattr(discovery_overrides, 'candidates', None)
+    if override is None:
+        yield from _generated_candidates(service)
+    else:
+        yield from override(service, default=_generated_candidates)

flink/datadog_checks/flink/config_models/discovery_overrides.py

@@ -0,0 +1,12 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+# Override the generated discovery candidates() for this integration.
+#
+# Define a candidates(service, default) function to wrap or replace the generated
+# candidate generation. `default` is the generated generator; call it to reuse
+# the spec-driven candidates, or ignore it to replace them entirely.
+#
+# def candidates(service, default):
+#     yield from default(service)

flink/datadog_checks/flink/config_models/discovery_strategies.py

@@ -0,0 +1,18 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+# Here you can define custom (local:) discovery strategies for this integration.
+#
+# Decorate a generator with @discovery_strategy (imported from
+# datadog_checks.base.utils.discovery) and reference it from the spec discovery
+# stanza as `strategy: local:<function_name>`. The function receives the
+# discovered Service plus the inputs declared in the spec and yields one context
+# (ctx) mapping per candidate, exposing the keys listed in `provides`.
+#
+# from datadog_checks.base.utils.discovery import discovery_strategy
+#
+# @discovery_strategy(provides=('svc',))
+# def from_some_config(service, config_path):
+#     ...
+#     yield {'svc': ...}

flink/datadog_checks/flink/data/auto_conf.yaml

@@ -0,0 +1,19 @@
+## @param ad_identifiers - list of strings - required
+## A list of container identifiers that are used by Autodiscovery to identify
+## which container the check should be run against. For more information, see:
+## https://docs.datadoghq.com/agent/guide/ad_identifiers/
+#
+ad_identifiers:
+  - flink
+
+## Enables configuration discovery
+#
+discovery: {}
+
+## Unused init configuration
+#
+init_config:
+
+## Unused instance configuration
+#
+instances: []

flink/tests/conftest.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from datadog_checks.dev import docker_run, get_docker_hostname, get_here
+from datadog_checks.dev import docker_run, get_docker_hostname, get_e2e_discovery_metadata, get_here
 from datadog_checks.dev.conditions import CheckEndpoints
 from datadog_checks.flink import FlinkCheck
 
@@ -26,9 +26,12 @@ def dd_environment():
         ),
         sleep=15,
     ):
-        yield {
-            "openmetrics_endpoint": f"http://{get_docker_hostname()}:{JOBMANAGER_PORT}/metrics",
-        }
+        yield (
+            {
+                "openmetrics_endpoint": f"http://{get_docker_hostname()}:{JOBMANAGER_PORT}/metrics",
+            },
+            get_e2e_discovery_metadata(),
+        )
 
 
 @pytest.fixture

flink/tests/test_e2e.py

@@ -3,10 +3,18 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import pytest
 
+from datadog_checks.dev.docker import CONTAINER_STABILITY_LOG_PATTERNS, assert_all_discovery_candidates_stable
 from datadog_checks.flink import FlinkCheck
 
 pytestmark = [pytest.mark.e2e]
 
+# Flink logs a harmless startup notice ("Hadoop FS is not available ...: NoClassDefFoundError")
+# with the vanilla `flink` image. Exclude only that known-benign substring so a real error is
+# still caught.
+DISCOVERY_STABILITY_LOG_PATTERNS = tuple(
+    pattern if pattern != r'error' else r'(?<!NoClassDefFound)error' for pattern in CONTAINER_STABILITY_LOG_PATTERNS
+)
+
 # Core metrics that should be present on a freshly-started JobManager,
 # regardless of whether any Flink job has been submitted. JVM and cluster
 # metrics are reported as soon as the reporter starts.
@@ -20,9 +28,35 @@
     "flink.jobmanager.taskSlotsTotal",
 ]
 
+# Core metrics that should be present on a freshly-started TaskManager, mirroring
+# EXPECTED_CORE_METRICS above but for the other role sharing the same container image.
+EXPECTED_TASKMANAGER_CORE_METRICS = [
+    "flink.taskmanager.Status.JVM.CPU.Load",
+    "flink.taskmanager.Status.JVM.Memory.Heap.Used",
+    "flink.taskmanager.Status.JVM.Threads.Count",
+]
+
 
 def test_e2e_jobmanager_metrics(dd_agent_check, dd_environment):
     aggregator = dd_agent_check(dd_environment, rate=True)
     for metric in EXPECTED_CORE_METRICS:
         aggregator.assert_metric(metric, at_least=1)
     aggregator.assert_service_check('flink.openmetrics.health', FlinkCheck.OK)
+
+
+def test_e2e_discovery(dd_agent_check_discovery):
+    # Both the jobmanager and taskmanager containers share the same `flink` image, so
+    # Autodiscovery finds and configures one instance per container.
+    aggregator = dd_agent_check_discovery(rate=True, discovery_min_instances=2)
+
+    for metric in EXPECTED_CORE_METRICS:
+        aggregator.assert_metric(metric, at_least=1)
+    for metric in EXPECTED_TASKMANAGER_CORE_METRICS:
+        aggregator.assert_metric(metric, at_least=1)
+    aggregator.assert_service_check('flink.openmetrics.health', FlinkCheck.OK)
+
+
+def test_e2e_discovery_all_candidates(dd_agent_check):
+    assert_all_discovery_candidates_stable(
+        dd_agent_check, FlinkCheck, compose_service='jobmanager', log_patterns=DISCOVERY_STABILITY_LOG_PATTERNS
+    )