From a9f3fc7279ea8a8c65632bc66bc23db0a784f563 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 24 Jul 2026 17:50:09 -0600 Subject: [PATCH 1/2] test(ffe): cover agentless EVP topologies --- manifests/nodejs.yml | 2 - tests/ffe/test_agentless_exposures.py | 137 ++++++++++++++++- tests/ffe/test_agentless_flag_eval_evp.py | 145 +++++++++++++++++- tests/ffe/test_agentless_flag_eval_metrics.py | 1 + tests/ffe/utils/telemetry.py | 21 ++- tests/test_the_test/scenarios.json | 29 ++++ .../test_feature_flag_telemetry.py | 19 ++- tests/test_the_test/test_group_rules.py | 7 + .../test_mock_ffe_agentless_backend.py | 120 ++++++++++++++- utils/_context/_scenarios/__init__.py | 119 +++++++++++++- utils/_context/_scenarios/endtoend.py | 139 ++++++++++++++++- utils/_context/containers.py | 59 ++++++- utils/build/docker/nodejs/express/app.js | 27 +++- .../docker/nodejs/express4-typescript/app.ts | 11 ++ utils/build/docker/nodejs/express4.Dockerfile | 1 + utils/build/docker/nodejs/fastify/app.js | 11 ++ .../_mock_ffe_agentless_backend.py | 25 ++- utils/interfaces/__init__.py | 2 + utils/proxy/core.py | 5 +- utils/proxy/ports.py | 3 + 20 files changed, 842 insertions(+), 41 deletions(-) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 1c04172858b..b5cc16d007c 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1708,8 +1708,6 @@ manifest: component_version: <5.66.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: *ref_5_83_0 tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2697 tracks Node.js agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) - tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) - tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: diff --git a/tests/ffe/test_agentless_exposures.py b/tests/ffe/test_agentless_exposures.py index 8975c8efe85..c9408cf3d22 100644 --- a/tests/ffe/test_agentless_exposures.py +++ b/tests/ffe/test_agentless_exposures.py @@ -4,14 +4,22 @@ from tests.ffe.utils.evaluation import evaluate_flag from tests.ffe.utils.fixtures import JSON from tests.ffe.utils.telemetry import ( + UNEXPECTED_ROUTE_WAIT_SECONDS, assert_expected_telemetry_route, matching_telemetry, + telemetry_request_was_accepted, wait_for_telemetry, ) -from utils import features, scenarios +from utils import context, features, interfaces, scenarios +from utils.interfaces._feature_flag_telemetry import FeatureFlagTelemetryInterfaceValidator + + +RELAY_EXPOSURES_PATH_V4 = "/evp_proxy/v4/api/v2/exposures" +RELAY_EXPOSURES_PATH_V2 = "/evp_proxy/v2/api/v2/exposures" @scenarios.feature_flagging_and_experimentation_agentless_sidecar +@scenarios.feature_flagging_and_experimentation_agentless_in_process @scenarios.feature_flagging_and_experimentation_agentless_direct_fallback @features.feature_flags_exposures class Test_FFE_Agentless_Exposure: @@ -33,7 +41,9 @@ def matcher(data: JSON) -> bool: for data in matching_telemetry(matcher): if data.get("path") == EXPOSURES_PATH: events.extend(exposure_events_from_data(data, {self.flag_key}, self.targeting_key)) - assert events, f"No exposure event found for {self.flag_key} and {self.targeting_key}" + assert len(events) == 1, ( + f"Expected one exposure event for {self.flag_key} and {self.targeting_key}, got {len(events)}" + ) assert_expected_telemetry_route(matcher, "exposure event") event = events[0] @@ -41,3 +51,126 @@ def matcher(data: JSON) -> bool: assert event["variant"]["key"] == "on" assert event["allocation"]["key"] == "default-allocation" assert event["subject"]["id"] == self.targeting_key + + +def _exposure_events_from_any_route(data: JSON, flag_key: str, targeting_key: str) -> list[JSON]: + path = data.get("path") + if not isinstance(path, str) or not path.endswith(EXPOSURES_PATH): + return [] + + canonical = dict(data) + canonical["path"] = EXPOSURES_PATH + return exposure_events_from_data(canonical, {flag_key}, targeting_key) + + +def _evaluate_discovery_flag(flag_key: str, targeting_key: str) -> None: + response = evaluate_flag(flag_key, targeting_key=targeting_key) + assert response.status_code == 200, f"Flag evaluation failed: {response.text}" + + +def _matching_exposure_requests( + interface: FeatureFlagTelemetryInterfaceValidator, flag_key: str, targeting_key: str +) -> list[JSON]: + return [data for data in interface.get_data() if _exposure_events_from_any_route(data, flag_key, targeting_key)] + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_v4 +@scenarios.feature_flagging_and_experimentation_agentless_relay_v2 +@features.feature_flags_exposures +class Test_FFE_Agentless_Exposure_Discovery: + flag_key = "empty-targeting-key-flag" + targeting_key = "agentless-discovery-user" + + def setup_agentless_exposure_prefers_advertised_local_evp(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_exposure_prefers_advertised_local_evp(self) -> None: + relay_profile = getattr(context.scenario, "relay_profile") # noqa: B009 - scenario subtype capability + expected_path = { + "v4": RELAY_EXPOSURES_PATH_V4, + "v2": RELAY_EXPOSURES_PATH_V2, + }[relay_profile] + + assert interfaces.ffe_relay.wait_for( + lambda data: data.get("path") == expected_path + and bool(_exposure_events_from_any_route(data, self.flag_key, self.targeting_key)), + timeout=30, + ), f"Timed out waiting for exposure through advertised {relay_profile} EVP route" + + relay_requests = _matching_exposure_requests(interfaces.ffe_relay, self.flag_key, self.targeting_key) + accepted = [data for data in relay_requests if telemetry_request_was_accepted(data)] + assert len(accepted) == 1, f"Expected one accepted local exposure request, got {len(accepted)}" + assert accepted[0]["path"] == expected_path + assert list(interfaces.ffe_relay.get_data(path_filters="/info")), "Tracer did not discover relay capabilities" + assert not _matching_exposure_requests(interfaces.ffe_direct, self.flag_key, self.targeting_key) + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_no_evp +@features.feature_flags_exposures +class Test_FFE_Agentless_Exposure_PreSend_Fallback: + flag_key = "empty-targeting-key-flag" + targeting_key = "agentless-no-evp-user" + + def setup_agentless_exposure_falls_back_when_info_has_no_evp(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_exposure_falls_back_when_info_has_no_evp(self) -> None: + def matcher(data: JSON) -> bool: + return bool(_exposure_events_from_any_route(data, self.flag_key, self.targeting_key)) + + wait_for_telemetry(matcher, "direct exposure after unsupported /info") + assert_expected_telemetry_route(matcher, "direct exposure after unsupported /info") + assert list(interfaces.ffe_relay.get_data(path_filters="/info")), "Tracer did not query relay /info" + assert not _matching_exposure_requests(interfaces.ffe_relay, self.flag_key, self.targeting_key) + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_evp_405 +@features.feature_flags_exposures +class Test_FFE_Agentless_Exposure_Definitive_Fallback: + flag_key = "empty-targeting-key-flag" + targeting_key = "agentless-evp-405-user" + + def setup_agentless_exposure_retries_direct_after_405(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_exposure_retries_direct_after_405(self) -> None: + def matcher(data: JSON) -> bool: + return bool(_exposure_events_from_any_route(data, self.flag_key, self.targeting_key)) + + assert interfaces.ffe_relay.wait_for( + lambda data: matcher(data) and data.get("response", {}).get("status_code") == 405, + timeout=30, + ), "Timed out waiting for the definitive local EVP 405" + wait_for_telemetry(matcher, "direct exposure after local EVP 405") + assert_expected_telemetry_route(matcher, "direct exposure after local EVP 405") + + local_attempts = _matching_exposure_requests(interfaces.ffe_relay, self.flag_key, self.targeting_key) + direct_accepts = [ + data + for data in _matching_exposure_requests(interfaces.ffe_direct, self.flag_key, self.targeting_key) + if telemetry_request_was_accepted(data) + ] + assert len(local_attempts) == 1, f"Expected one local attempt before fallback, got {len(local_attempts)}" + assert len(direct_accepts) == 1, f"Expected one accepted direct fallback, got {len(direct_accepts)}" + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_evp_500 +@features.feature_flags_exposures +class Test_FFE_Agentless_Exposure_Ambiguous_Failure: + flag_key = "empty-targeting-key-flag" + targeting_key = "agentless-evp-500-user" + + def setup_agentless_exposure_does_not_retry_direct_after_500(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_exposure_does_not_retry_direct_after_500(self) -> None: + def matcher(data: JSON) -> bool: + return bool(_exposure_events_from_any_route(data, self.flag_key, self.targeting_key)) + + assert interfaces.ffe_relay.wait_for( + lambda data: matcher(data) and data.get("response", {}).get("status_code") == 500, + timeout=30, + ), "Timed out waiting for the ambiguous local EVP 500" + assert not interfaces.ffe_direct.wait_for(matcher, timeout=UNEXPECTED_ROUTE_WAIT_SECONDS), ( + "Ambiguously rejected local exposure was retried through direct intake" + ) diff --git a/tests/ffe/test_agentless_flag_eval_evp.py b/tests/ffe/test_agentless_flag_eval_evp.py index e5550bc3f62..5e8072dbe65 100644 --- a/tests/ffe/test_agentless_flag_eval_evp.py +++ b/tests/ffe/test_agentless_flag_eval_evp.py @@ -3,6 +3,7 @@ import pytest from tests.ffe.test_flag_eval_evp import ( + EVP_FLAGEVALUATIONS_PATH, assert_batch_context, assert_event_contract, evp_flagevaluation_events_from_data, @@ -11,19 +12,27 @@ from tests.ffe.utils.evaluation import evaluate_flag from tests.ffe.utils.fixtures import JSON from tests.ffe.utils.telemetry import ( + UNEXPECTED_ROUTE_WAIT_SECONDS, assert_expected_telemetry_route, matching_telemetry, + telemetry_request_was_accepted, wait_for_telemetry, ) -from utils import features, scenarios +from utils import context, features, interfaces, scenarios +from utils.interfaces._feature_flag_telemetry import FeatureFlagTelemetryInterfaceValidator + + +RELAY_FLAGEVALUATIONS_PATH_V4 = "/evp_proxy/v4/api/v2/flagevaluation" +RELAY_FLAGEVALUATIONS_PATH_V2 = "/evp_proxy/v2/api/v2/flagevaluation" @scenarios.feature_flagging_and_experimentation_agentless_sidecar +@scenarios.feature_flagging_and_experimentation_agentless_in_process @scenarios.feature_flagging_and_experimentation_agentless_direct_fallback @features.feature_flags_evp_flagevaluation @pytest.mark.skip_if_xfail class Test_FFE_Agentless_EVP_Flagevaluation: - flag_key = "empty-targeting-key-flag" + flag_key = "empty_string_flag" def setup_agentless_evp_flagevaluation(self) -> None: self.response = evaluate_flag(self.flag_key, targeting_key="agentless-evp-user") @@ -39,11 +48,137 @@ def matcher(data: JSON) -> bool: events = [] for data in matching_telemetry(matcher): events.extend(evp_flagevaluation_events_from_data(data, self.flag_key)) - assert events, f"No EVP flagevaluation event found for {self.flag_key}" + assert len(events) == 1, f"Expected one EVP flagevaluation event for {self.flag_key}, got {len(events)}" assert_expected_telemetry_route(matcher, "aggregate flag-evaluation event") batch, event = events[0] assert_batch_context(batch) assert_event_contract(event, self.flag_key) - assert object_key(event.get("variant"), "variant") == "on" - assert object_key(event.get("allocation"), "allocation") == "default-allocation" + assert event["evaluation_count"] == 1 + assert object_key(event.get("variant"), "variant") == "non_empty" + assert object_key(event.get("allocation"), "allocation") == "allocation-test" + + +def _flagevaluation_events_from_any_route(data: JSON, flag_key: str) -> list[tuple[JSON, JSON]]: + path = data.get("path") + if not isinstance(path, str) or not path.endswith(EVP_FLAGEVALUATIONS_PATH): + return [] + + canonical = dict(data) + canonical["path"] = EVP_FLAGEVALUATIONS_PATH + return evp_flagevaluation_events_from_data(canonical, flag_key) + + +def _evaluate_discovery_flag(flag_key: str, targeting_key: str) -> None: + response = evaluate_flag(flag_key, targeting_key=targeting_key) + assert response.status_code == 200, f"Flag evaluation failed: {response.text}" + + +def _matching_flagevaluation_requests(interface: FeatureFlagTelemetryInterfaceValidator, flag_key: str) -> list[JSON]: + return [data for data in interface.get_data() if _flagevaluation_events_from_any_route(data, flag_key)] + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_v4 +@scenarios.feature_flagging_and_experimentation_agentless_relay_v2 +@features.feature_flags_evp_flagevaluation +@pytest.mark.skip_if_xfail +class Test_FFE_Agentless_EVP_Flagevaluation_Discovery: + flag_key = "empty_string_flag" + targeting_key = "agentless-evaluation-discovery-user" + + def setup_agentless_flagevaluation_prefers_advertised_local_evp(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_flagevaluation_prefers_advertised_local_evp(self) -> None: + relay_profile = getattr(context.scenario, "relay_profile") # noqa: B009 - scenario subtype capability + expected_path = { + "v4": RELAY_FLAGEVALUATIONS_PATH_V4, + "v2": RELAY_FLAGEVALUATIONS_PATH_V2, + }[relay_profile] + + assert interfaces.ffe_relay.wait_for( + lambda data: data.get("path") == expected_path + and bool(_flagevaluation_events_from_any_route(data, self.flag_key)), + timeout=30, + ), f"Timed out waiting for flagevaluation through advertised {relay_profile} EVP route" + + relay_requests = _matching_flagevaluation_requests(interfaces.ffe_relay, self.flag_key) + accepted = [data for data in relay_requests if telemetry_request_was_accepted(data)] + assert len(accepted) == 1, f"Expected one accepted local flagevaluation request, got {len(accepted)}" + assert accepted[0]["path"] == expected_path + assert list(interfaces.ffe_relay.get_data(path_filters="/info")), "Tracer did not discover relay capabilities" + assert not _matching_flagevaluation_requests(interfaces.ffe_direct, self.flag_key) + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_no_evp +@features.feature_flags_evp_flagevaluation +@pytest.mark.skip_if_xfail +class Test_FFE_Agentless_EVP_Flagevaluation_PreSend_Fallback: + flag_key = "empty_string_flag" + targeting_key = "agentless-evaluation-no-evp-user" + + def setup_agentless_flagevaluation_falls_back_when_info_has_no_evp(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_flagevaluation_falls_back_when_info_has_no_evp(self) -> None: + def matcher(data: JSON) -> bool: + return bool(_flagevaluation_events_from_any_route(data, self.flag_key)) + + wait_for_telemetry(matcher, "direct flagevaluation after unsupported /info") + assert_expected_telemetry_route(matcher, "direct flagevaluation after unsupported /info") + assert list(interfaces.ffe_relay.get_data(path_filters="/info")), "Tracer did not query relay /info" + assert not _matching_flagevaluation_requests(interfaces.ffe_relay, self.flag_key) + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_evp_405 +@features.feature_flags_evp_flagevaluation +@pytest.mark.skip_if_xfail +class Test_FFE_Agentless_EVP_Flagevaluation_Definitive_Fallback: + flag_key = "empty_string_flag" + targeting_key = "agentless-evaluation-evp-405-user" + + def setup_agentless_flagevaluation_retries_direct_after_405(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_flagevaluation_retries_direct_after_405(self) -> None: + def matcher(data: JSON) -> bool: + return bool(_flagevaluation_events_from_any_route(data, self.flag_key)) + + assert interfaces.ffe_relay.wait_for( + lambda data: matcher(data) and data.get("response", {}).get("status_code") == 405, + timeout=30, + ), "Timed out waiting for the definitive local flagevaluation 405" + wait_for_telemetry(matcher, "direct flagevaluation after local EVP 405") + assert_expected_telemetry_route(matcher, "direct flagevaluation after local EVP 405") + + local_attempts = _matching_flagevaluation_requests(interfaces.ffe_relay, self.flag_key) + direct_accepts = [ + data + for data in _matching_flagevaluation_requests(interfaces.ffe_direct, self.flag_key) + if telemetry_request_was_accepted(data) + ] + assert len(local_attempts) == 1, f"Expected one local attempt before fallback, got {len(local_attempts)}" + assert len(direct_accepts) == 1, f"Expected one accepted direct fallback, got {len(direct_accepts)}" + + +@scenarios.feature_flagging_and_experimentation_agentless_relay_evp_500 +@features.feature_flags_evp_flagevaluation +@pytest.mark.skip_if_xfail +class Test_FFE_Agentless_EVP_Flagevaluation_Ambiguous_Failure: + flag_key = "empty_string_flag" + targeting_key = "agentless-evaluation-evp-500-user" + + def setup_agentless_flagevaluation_does_not_retry_direct_after_500(self) -> None: + _evaluate_discovery_flag(self.flag_key, self.targeting_key) + + def test_agentless_flagevaluation_does_not_retry_direct_after_500(self) -> None: + def matcher(data: JSON) -> bool: + return bool(_flagevaluation_events_from_any_route(data, self.flag_key)) + + assert interfaces.ffe_relay.wait_for( + lambda data: matcher(data) and data.get("response", {}).get("status_code") == 500, + timeout=30, + ), "Timed out waiting for the ambiguous local flagevaluation 500" + assert not interfaces.ffe_direct.wait_for(matcher, timeout=UNEXPECTED_ROUTE_WAIT_SECONDS), ( + "Ambiguously rejected local flagevaluation was retried through direct intake" + ) diff --git a/tests/ffe/test_agentless_flag_eval_metrics.py b/tests/ffe/test_agentless_flag_eval_metrics.py index f5e49f1b2a0..a992ad4a923 100644 --- a/tests/ffe/test_agentless_flag_eval_metrics.py +++ b/tests/ffe/test_agentless_flag_eval_metrics.py @@ -9,6 +9,7 @@ @scenarios.feature_flagging_and_experimentation_agentless_sidecar +@scenarios.feature_flagging_and_experimentation_agentless_in_process @scenarios.feature_flagging_and_experimentation_agentless_direct_fallback @features.feature_flags_eval_metrics class Test_FFE_Agentless_Eval_Metric: diff --git a/tests/ffe/utils/telemetry.py b/tests/ffe/utils/telemetry.py index e35a2ec0b23..545fbbd2826 100644 --- a/tests/ffe/utils/telemetry.py +++ b/tests/ffe/utils/telemetry.py @@ -24,7 +24,7 @@ def telemetry_interface() -> FeatureFlagTelemetryInterfaceValidator: def telemetry_route() -> str: route = getattr(context.scenario, "telemetry_route", None) - assert route in ("sidecar", "direct"), f"Scenario {context.scenario.name} has no telemetry route" + assert route in ("sidecar", "in_process", "direct"), f"Scenario {context.scenario.name} has no telemetry route" return route @@ -39,7 +39,7 @@ def matching_telemetry(matcher: Callable[[JSON], bool]) -> list[JSON]: def assert_expected_telemetry_route(matcher: Callable[[JSON], bool], description: str) -> None: - captured = matching_telemetry(matcher) + captured = [data for data in matching_telemetry(matcher) if telemetry_request_was_accepted(data)] assert captured, f"No {description} captured through {telemetry_route()} telemetry" if telemetry_route() == "direct": @@ -51,14 +51,27 @@ def assert_expected_telemetry_route(matcher: Callable[[JSON], bool], description unexpected = getattr(context.scenario, "unexpected_telemetry_interface", None) assert isinstance(unexpected, FeatureFlagTelemetryInterfaceValidator) if unexpected.replay: - duplicated = any(matcher(cast("JSON", data)) for data in unexpected.get_data()) + duplicated = any( + matcher(cast("JSON", data)) and telemetry_request_was_accepted(cast("JSON", data)) + for data in unexpected.get_data() + ) else: duplicated = unexpected.wait_for( - lambda data: matcher(cast("JSON", data)), timeout=UNEXPECTED_ROUTE_WAIT_SECONDS + lambda data: matcher(cast("JSON", data)) and telemetry_request_was_accepted(cast("JSON", data)), + timeout=UNEXPECTED_ROUTE_WAIT_SECONDS, ) assert not duplicated, f"{description} was duplicated through the non-selected telemetry route" +def telemetry_request_was_accepted(data: JSON) -> bool: + response = data.get("response") + if not isinstance(response, dict): + return False + + status_code = response.get("status_code") + return isinstance(status_code, int) and 200 <= status_code < 300 + + def _request_header(data: JSON, name: str) -> str | None: request = data.get("request") if not isinstance(request, dict): diff --git a/tests/test_the_test/scenarios.json b/tests/test_the_test/scenarios.json index 16412649051..d93c8280b4b 100644 --- a/tests/test_the_test/scenarios.json +++ b/tests/test_the_test/scenarios.json @@ -3353,14 +3353,43 @@ ], "tests/ffe/test_agentless_exposures.py::Test_FFE_Agentless_Exposure::test_agentless_exposure": [ "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT_FALLBACK", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_IN_PROCESS", "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SIDECAR" ], + "tests/ffe/test_agentless_exposures.py::Test_FFE_Agentless_Exposure_Discovery::test_agentless_exposure_prefers_advertised_local_evp": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_V2", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_V4" + ], + "tests/ffe/test_agentless_exposures.py::Test_FFE_Agentless_Exposure_PreSend_Fallback::test_agentless_exposure_falls_back_when_info_has_no_evp": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_NO_EVP" + ], + "tests/ffe/test_agentless_exposures.py::Test_FFE_Agentless_Exposure_Definitive_Fallback::test_agentless_exposure_retries_direct_after_405": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_EVP_405" + ], + "tests/ffe/test_agentless_exposures.py::Test_FFE_Agentless_Exposure_Ambiguous_Failure::test_agentless_exposure_does_not_retry_direct_after_500": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_EVP_500" + ], "tests/ffe/test_agentless_flag_eval_evp.py::Test_FFE_Agentless_EVP_Flagevaluation::test_agentless_evp_flagevaluation": [ "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT_FALLBACK", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_IN_PROCESS", "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SIDECAR" ], + "tests/ffe/test_agentless_flag_eval_evp.py::Test_FFE_Agentless_EVP_Flagevaluation_Discovery::test_agentless_flagevaluation_prefers_advertised_local_evp": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_V2", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_V4" + ], + "tests/ffe/test_agentless_flag_eval_evp.py::Test_FFE_Agentless_EVP_Flagevaluation_PreSend_Fallback::test_agentless_flagevaluation_falls_back_when_info_has_no_evp": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_NO_EVP" + ], + "tests/ffe/test_agentless_flag_eval_evp.py::Test_FFE_Agentless_EVP_Flagevaluation_Definitive_Fallback::test_agentless_flagevaluation_retries_direct_after_405": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_EVP_405" + ], + "tests/ffe/test_agentless_flag_eval_evp.py::Test_FFE_Agentless_EVP_Flagevaluation_Ambiguous_Failure::test_agentless_flagevaluation_does_not_retry_direct_after_500": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_EVP_500" + ], "tests/ffe/test_agentless_flag_eval_metrics.py::Test_FFE_Agentless_Eval_Metric::test_agentless_eval_metric": [ "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT_FALLBACK", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_IN_PROCESS", "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SIDECAR" ], "tests/integrations/test_cassandra.py::Test_Cassandra::test_main": [ diff --git a/tests/test_the_test/test_feature_flag_telemetry.py b/tests/test_the_test/test_feature_flag_telemetry.py index 269e4e6b67a..b86fd00ff96 100644 --- a/tests/test_the_test/test_feature_flag_telemetry.py +++ b/tests/test_the_test/test_feature_flag_telemetry.py @@ -4,7 +4,7 @@ import pytest -from tests.ffe.utils.telemetry import assert_expected_telemetry_route +from tests.ffe.utils.telemetry import assert_expected_telemetry_route, telemetry_request_was_accepted from utils import context, features, scenarios from utils.interfaces._feature_flag_telemetry import FeatureFlagTelemetryInterfaceValidator, metric_points_from_data @@ -86,7 +86,7 @@ def test_replay_route_assertion_checks_recorded_unexpected_route(monkeypatch: py expected.configure("unused", replay=True) unexpected.configure("unused", replay=True) - matching_data = {"path": "/api/v2/exposures"} + matching_data = {"path": "/api/v2/exposures", "response": {"status_code": 202}} expected._append_data(matching_data) # noqa: SLF001 - focused replay fixture monkeypatch.setattr( context, @@ -102,6 +102,21 @@ def test_replay_route_assertion_checks_recorded_unexpected_route(monkeypatch: py assert_expected_telemetry_route(matcher, "exposure event") + unexpected._append_data( # noqa: SLF001 - focused replay fixture + {"path": "/api/v2/exposures", "response": {"status_code": 405}} + ) + assert_expected_telemetry_route(matcher, "exposure event") + unexpected._append_data(matching_data) # noqa: SLF001 - focused replay fixture with pytest.raises(AssertionError, match="duplicated through the non-selected telemetry route"): assert_expected_telemetry_route(matcher, "exposure event") + + +@scenarios.test_the_test +@features.not_reported +def test_only_2xx_telemetry_responses_are_accepted() -> None: + assert telemetry_request_was_accepted({"response": {"status_code": 200}}) + assert telemetry_request_was_accepted({"response": {"status_code": 202}}) + assert not telemetry_request_was_accepted({"response": {"status_code": 405}}) + assert not telemetry_request_was_accepted({"response": {"status_code": 500}}) + assert not telemetry_request_was_accepted({"response": None}) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index ff092f287cb..485c2764dfe 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -20,7 +20,13 @@ def test_tracer_release(): dormant_agentless_scenarios = [ scenarios.feature_flagging_and_experimentation_agentless, scenarios.feature_flagging_and_experimentation_agentless_sidecar, + scenarios.feature_flagging_and_experimentation_agentless_in_process, scenarios.feature_flagging_and_experimentation_agentless_direct_fallback, + scenarios.feature_flagging_and_experimentation_agentless_relay_v4, + scenarios.feature_flagging_and_experimentation_agentless_relay_v2, + scenarios.feature_flagging_and_experimentation_agentless_relay_no_evp, + scenarios.feature_flagging_and_experimentation_agentless_relay_evp_405, + scenarios.feature_flagging_and_experimentation_agentless_relay_evp_500, ] for dormant_agentless_scenario in dormant_agentless_scenarios: assert dormant_agentless_scenario.include_agent is False @@ -32,6 +38,7 @@ def test_tracer_release(): assert scenarios.feature_flagging_and_experimentation_agentless.use_proxy is False assert scenarios.feature_flagging_and_experimentation_agentless._flush_weblog_on_stop is False # noqa: SLF001 assert scenarios.feature_flagging_and_experimentation_agentless_sidecar.use_proxy is True + assert scenarios.feature_flagging_and_experimentation_agentless_in_process.use_proxy is True assert scenarios.feature_flagging_and_experimentation_agentless_direct_fallback.use_proxy is True not_in_tracer_release_group = [ diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index f1cfc718aec..badeec2d054 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -1,5 +1,7 @@ """Unit coverage for the mock FFE agentless backend test fixture.""" +import json +from typing import Literal from unittest.mock import MagicMock import requests @@ -14,7 +16,7 @@ MockFFEAgentlessBackendServer, ) from utils._context._scenarios.endtoend import FeatureFlaggingAgentlessEndToEndScenario -from utils._context.containers import ServerlessSidecarContainer +from utils._context.containers import ServerlessSidecarContainer, serverless_init_wrapper_dockerfile from utils.proxy.ports import ProxyPorts @@ -25,6 +27,9 @@ def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id try: response = requests.get(server.base_url + CONFIG_PATH, headers={"dd-api-key": EXPECTED_API_KEY}, timeout=5) response.raise_for_status() + payload = response.json() + assert payload["data"]["type"] == "universal-flag-configuration" + assert payload["data"]["attributes"]["flags"] status = server.status() assert status["requests_total"] == 1 @@ -35,6 +40,22 @@ def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id server.close() +@scenarios.test_the_test +@features.not_reported +def test_mock_ffe_agentless_backend_can_serve_custom_endpoint_without_auth(worker_id: str) -> None: + server = MockFFEAgentlessBackendServer(worker_id, require_auth=False) + try: + response = requests.get(server.base_url + CONFIG_PATH, timeout=5) + response.raise_for_status() + + status = server.status() + assert status["requests_total"] == 1 + assert status["last_auth_present"] is False + assert status["last_status_code"] == 200 + finally: + server.close() + + @scenarios.test_the_test @features.not_reported def test_mock_ffe_agentless_backend_host_gateway_mapping(monkeypatch: pytest.MonkeyPatch, worker_id: str) -> None: @@ -160,6 +181,33 @@ def test_agentless_sidecar_scenario_prefers_serverless_sidecar() -> None: assert environment["DD_PROXY_HTTPS"] == f"http://proxy:{ProxyPorts.ffe_direct}" +@scenarios.test_the_test +@features.not_reported +def test_agentless_in_process_scenario_wraps_weblog() -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_IN_PROCESS", + doc="test", + include_agent=False, + telemetry_route="in_process", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + + environment = scenario.weblog_infra.library_container.environment + assert scenario.agent_container not in scenario._containers # noqa: SLF001 - focused topology test + assert scenario.proxy_container in scenario._containers # noqa: SLF001 - focused topology test + assert not scenario.weblog_infra._other_containers # noqa: SLF001 - focused topology test + assert environment["DD_TRACE_AGENT_URL"] == "http://127.0.0.1:8126" + assert environment["OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"] == "http://127.0.0.1:4318/v1/metrics" + assert environment["DD_PROXY_HTTPS"] == f"http://proxy:{ProxyPorts.ffe_sidecar}" + assert "DD_EVP_PROXY_CONFIG_ADDITIONAL_ENDPOINTS" not in environment + assert scenario.telemetry_interface is interfaces.ffe_sidecar + + dockerfile = serverless_init_wrapper_dockerfile("system_tests/weblog") + assert "FROM system_tests/weblog" in dockerfile + assert 'ENTRYPOINT ["/datadog-init"]' in dockerfile + + @scenarios.test_the_test @features.not_reported def test_agentless_end_to_end_scenario_closes_backend_when_startup_fails( @@ -175,7 +223,8 @@ def test_agentless_end_to_end_scenario_closes_backend_when_startup_fails( backend = MagicMock(spec=MockFFEAgentlessBackendServer) backend.reset.side_effect = RuntimeError("reset failed") - def create_backend(_worker_id: str) -> MagicMock: + def create_backend(_worker_id: str, *, require_auth: bool) -> MagicMock: + assert require_auth is True return backend monkeypatch.setattr(endtoend_scenarios, "MockFFEAgentlessBackendServer", create_backend) @@ -230,3 +279,70 @@ def test_agentless_direct_scenario_uses_authenticated_fallback() -> None: assert environment["DD_PROXY_HTTPS"] == f"http://proxy:{ProxyPorts.ffe_direct}" assert environment["OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"] == f"http://proxy:{ProxyPorts.ffe_direct}/v1/metrics" assert environment["OTEL_EXPORTER_OTLP_METRICS_HEADERS"] == (f"dd-api-key={EXPECTED_API_KEY},dd-protocol=otlp") + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize( + ("relay_profile", "telemetry_route", "expected_endpoints", "expected_failure"), + [ + ("v4", "relay", ["/evp_proxy/v4/", "/evp_proxy/v2/"], None), + ("v2", "relay", ["/evp_proxy/v2/"], None), + ("no_evp", "direct", [], None), + ("evp_405", "direct", ["/evp_proxy/v4/"], 405), + ("evp_500", "relay", ["/evp_proxy/v4/"], 500), + ], +) +def test_agentless_programmable_relay_profiles( + relay_profile: Literal["v4", "v2", "no_evp", "evp_405", "evp_500"], + telemetry_route: Literal["relay", "direct"], + expected_endpoints: list[str], + expected_failure: int | None, +) -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_PROGRAMMABLE_RELAY", + doc="test", + include_agent=False, + relay_profile=relay_profile, + telemetry_route=telemetry_route, + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + + environment = scenario.weblog_infra.library_container.environment + assert environment["DD_TRACE_AGENT_URL"] == f"http://proxy:{ProxyPorts.ffe_relay}" + assert environment["DD_PROXY_HTTPS"] == f"http://proxy:{ProxyPorts.ffe_direct}" + assert environment["OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"] == f"http://proxy:{ProxyPorts.ffe_direct}/v1/metrics" + + responses = scenario.proxy_container.internal_mocked_backend_responses + info = next(response for response in responses if response.path == "/info") + assert json.loads(info.content) == {"endpoints": expected_endpoints} + + failures = [response for response in responses if response.path.endswith("/api/v2/exposures")] + if expected_failure is None: + assert not failures + else: + assert len(failures) == 1 + assert failures[0].status_code == expected_failure + + if telemetry_route == "relay": + assert scenario.telemetry_interface is interfaces.ffe_relay + assert scenario.unexpected_telemetry_interface is interfaces.ffe_direct + else: + assert scenario.telemetry_interface is interfaces.ffe_direct + assert scenario.unexpected_telemetry_interface is interfaces.ffe_relay + + +@scenarios.test_the_test +@features.not_reported +def test_agentless_programmable_relay_rejects_incompatible_provenance() -> None: + with pytest.raises(ValueError, match="requires relay or direct telemetry provenance"): + FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_INVALID_RELAY", + doc="test", + include_agent=False, + relay_profile="v4", + telemetry_route="sidecar", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 83338858b9d..6fc0c3ac7d8 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -730,7 +730,7 @@ class _Scenarios: feature_flagging_and_experimentation_agentless = FeatureFlaggingAgentlessEndToEndScenario( "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", weblog_env={ - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_ENABLED": "true", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", "DD_REMOTE_CONFIGURATION_ENABLED": "false", @@ -748,7 +748,7 @@ class _Scenarios: feature_flagging_and_experimentation_agentless_sidecar = FeatureFlaggingAgentlessEndToEndScenario( "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SIDECAR", weblog_env={ - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_ENABLED": "true", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", "DD_REMOTE_CONFIGURATION_ENABLED": "false", @@ -763,10 +763,28 @@ class _Scenarios: use_proxy_for_weblog=False, ) + feature_flagging_and_experimentation_agentless_in_process = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_IN_PROCESS", + weblog_env={ + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + }, + doc="Validate auto telemetry through serverless-init wrapping the weblog process.", + include_agent=False, + include_default_scenario_groups=False, + library_interface_timeout=0, + scenario_groups=[scenario_groups.ffe], + telemetry_route="in_process", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + feature_flagging_and_experimentation_agentless_direct_fallback = FeatureFlaggingAgentlessEndToEndScenario( "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT_FALLBACK", weblog_env={ - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_ENABLED": "true", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", "DD_REMOTE_CONFIGURATION_ENABLED": "false", @@ -781,6 +799,101 @@ class _Scenarios: use_proxy_for_weblog=False, ) + feature_flagging_and_experimentation_agentless_relay_v4 = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_V4", + weblog_env={ + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + }, + doc="Validate that auto telemetry prefers EVP v4 when a local relay advertises v4 and v2.", + include_agent=False, + include_default_scenario_groups=False, + library_interface_timeout=0, + relay_profile="v4", + scenario_groups=[scenario_groups.ffe], + telemetry_route="relay", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + + feature_flagging_and_experimentation_agentless_relay_v2 = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_V2", + weblog_env={ + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + }, + doc="Validate the compatible EVP v2 route when it is the only advertised local endpoint.", + include_agent=False, + include_default_scenario_groups=False, + library_interface_timeout=0, + relay_profile="v2", + scenario_groups=[scenario_groups.ffe], + telemetry_route="relay", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + + feature_flagging_and_experimentation_agentless_relay_no_evp = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_NO_EVP", + weblog_env={ + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + }, + doc="Validate direct fallback before publish when local /info advertises no supported EVP endpoint.", + include_agent=False, + include_default_scenario_groups=False, + library_interface_timeout=0, + relay_profile="no_evp", + scenario_groups=[scenario_groups.ffe], + telemetry_route="direct", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + + feature_flagging_and_experimentation_agentless_relay_evp_405 = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_EVP_405", + weblog_env={ + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + }, + doc="Validate exactly-once direct fallback after a definitive local EVP 405 response.", + include_agent=False, + include_default_scenario_groups=False, + library_interface_timeout=0, + relay_profile="evp_405", + scenario_groups=[scenario_groups.ffe], + telemetry_route="direct", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + + feature_flagging_and_experimentation_agentless_relay_evp_500 = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_RELAY_EVP_500", + weblog_env={ + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + }, + doc="Validate that an ambiguous local EVP 500 response does not cross-route retry the same batch.", + include_agent=False, + include_default_scenario_groups=False, + library_interface_timeout=0, + relay_profile="evp_500", + scenario_groups=[scenario_groups.ffe], + telemetry_route="relay", + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario( "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE", rc_api_enabled=True, diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index e7c3f1910ec..f6de5d1ab13 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -1,5 +1,8 @@ from typing import Any, Literal +from http import HTTPStatus +import json import os +import time import pytest from docker.models.networks import Network @@ -8,10 +11,11 @@ from watchdog.observers.polling import PollingObserver from watchdog.events import FileSystemEventHandler, FileSystemEvent -from utils import interfaces +from utils import interfaces, weblog from utils.interfaces._core import ProxyBasedInterfaceValidator from utils.interfaces._feature_flag_telemetry import FeatureFlagTelemetryInterfaceValidator from utils.buddies import BuddyHostPorts +from utils.proxy.mocked_response import MockedBackendResponse from utils.proxy.ports import ProxyPorts from utils._context.component_version import Version from utils._context.docker import get_docker_client @@ -641,14 +645,19 @@ def __init__( self, name: str, *, - telemetry_route: Literal["none", "sidecar", "direct"] = "none", + telemetry_route: Literal["none", "sidecar", "in_process", "direct", "relay"] = "none", + relay_profile: Literal["v4", "v2", "no_evp", "evp_405", "evp_500"] | None = None, weblog_env: dict[str, str | None] | None = None, other_weblog_containers: tuple[type[TestedContainer], ...] = (), **kwargs: Any, # noqa: ANN401 ) -> None: self.telemetry_route = telemetry_route + self.relay_profile = relay_profile environment = dict(weblog_env or {}) + if relay_profile is not None and telemetry_route not in ("relay", "direct"): + raise ValueError("A programmable relay profile requires relay or direct telemetry provenance") + if telemetry_route != "none": environment |= { "DD_FEATURE_FLAGS_TELEMETRY_TRANSPORT": "auto", @@ -657,9 +666,23 @@ def __init__( "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", "OTEL_METRIC_EXPORT_INTERVAL": "1000", "DD_PROXY_HTTPS": f"http://proxy:{ProxyPorts.ffe_direct}", + "HTTPS_PROXY": f"http://proxy:{ProxyPorts.ffe_direct}", + "NODE_EXTRA_CA_CERTS": "/usr/local/share/ca-certificates/system-tests-mitmproxy-ca.pem", } - if telemetry_route == "sidecar": + if relay_profile is not None: + environment |= { + "DD_AGENT_HOST": "proxy", + "DD_TRACE_AGENT_PORT": str(ProxyPorts.ffe_relay), + "DD_TRACE_AGENT_URL": f"http://proxy:{ProxyPorts.ffe_relay}", + "DD_API_KEY": EXPECTED_API_KEY, + "DD_SITE": "datad0g.com", + # `/info` does not describe OTLP. Keep metrics explicitly direct + # so these profiles isolate the EVP capability state machine. + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://proxy:{ProxyPorts.ffe_direct}/v1/metrics", + "OTEL_EXPORTER_OTLP_METRICS_HEADERS": (f"dd-api-key={EXPECTED_API_KEY},dd-protocol=otlp"), + } + elif telemetry_route == "sidecar": environment |= { "DD_AGENT_HOST": "ffe-serverless-sidecar", "DD_TRACE_AGENT_PORT": "8126", @@ -667,6 +690,20 @@ def __init__( "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://ffe-serverless-sidecar:4318/v1/metrics", } other_weblog_containers += (ServerlessSidecarContainer,) + elif telemetry_route == "in_process": + environment |= { + "DD_AGENT_HOST": "127.0.0.1", + "DD_TRACE_AGENT_PORT": "8126", + "DD_TRACE_AGENT_URL": "http://127.0.0.1:8126", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://127.0.0.1:4318/v1/metrics", + "DD_APM_ENABLED": "true", + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT": "127.0.0.1:4318", + # HTTPS_PROXY preserves ffe_sidecar route provenance. + "DD_PROXY_HTTPS": f"http://proxy:{ProxyPorts.ffe_sidecar}", + "DD_PROXY_HTTP": f"http://proxy:{ProxyPorts.ffe_sidecar}", + "DD_SERVERLESS_FLUSH_STRATEGY": "periodically,100", + "DD_SKIP_SSL_VALIDATION": "true", + } elif telemetry_route == "direct": environment |= { "DD_API_KEY": EXPECTED_API_KEY, @@ -683,20 +720,85 @@ def __init__( **kwargs, ) + if relay_profile is not None: + self.proxy_container.internal_mocked_backend_responses.extend(self._relay_responses(relay_profile)) + + @staticmethod + def _wait_for_agentless_configuration() -> None: + """Wait until the initial UFC poll is applied before collecting test setup.""" + + for _ in range(50): + response = weblog.post( + "/ffe", + json={ + "flag": "numeric_flag", + "variationType": "NUMERIC", + "defaultValue": -1, + "targetingKey": "agentless-readiness-probe", + "attributes": {}, + }, + ) + if response.status_code == HTTPStatus.OK and response.text is not None: + payload = json.loads(response.text) + if payload.get("value") != -1: + return + time.sleep(0.2) + raise RuntimeError("Feature Flags agentless configuration was not applied before test collection") + + @staticmethod + def _relay_responses( + relay_profile: Literal["v4", "v2", "no_evp", "evp_405", "evp_500"], + ) -> list[MockedBackendResponse]: + endpoints = { + "v4": ["/evp_proxy/v4/", "/evp_proxy/v2/"], + "v2": ["/evp_proxy/v2/"], + "no_evp": [], + "evp_405": ["/evp_proxy/v4/"], + "evp_500": ["/evp_proxy/v4/"], + }[relay_profile] + responses = [ + MockedBackendResponse( + path="/info", + content=json.dumps({"endpoints": endpoints}).encode(), + content_type="application/json", + ) + ] + + failure_status = {"evp_405": 405, "evp_500": 500}.get(relay_profile) + if failure_status is not None: + for path in ( + "/evp_proxy/v4/api/v2/exposures", + "/evp_proxy/v4/api/v2/flagevaluation", + ): + responses.append( + MockedBackendResponse( + path=path, + content=b"local relay rejected payload", + content_type="text/plain", + status_code=failure_status, + ) + ) + + return responses + @property def telemetry_interface(self) -> FeatureFlagTelemetryInterfaceValidator: - if self.telemetry_route == "sidecar": + if self.telemetry_route in ("sidecar", "in_process"): return interfaces.ffe_sidecar if self.telemetry_route == "direct": return interfaces.ffe_direct + if self.telemetry_route == "relay": + return interfaces.ffe_relay raise ValueError("This scenario does not capture Feature Flags telemetry") @property def unexpected_telemetry_interface(self) -> FeatureFlagTelemetryInterfaceValidator: - if self.telemetry_route == "sidecar": + if self.telemetry_route in ("sidecar", "in_process"): return interfaces.ffe_direct if self.telemetry_route == "direct": - return interfaces.ffe_sidecar + return interfaces.ffe_relay if self.relay_profile is not None else interfaces.ffe_sidecar + if self.telemetry_route == "relay": + return interfaces.ffe_direct raise ValueError("This scenario does not capture Feature Flags telemetry") def configure(self, config: pytest.Config) -> None: @@ -708,11 +810,28 @@ def configure(self, config: pytest.Config) -> None: if self.telemetry_route != "none": interfaces.ffe_sidecar.configure(self.host_log_folder, replay=self.replay) interfaces.ffe_direct.configure(self.host_log_folder, replay=self.replay) + interfaces.ffe_relay.configure(self.host_log_folder, replay=self.replay) super().configure(config) + if self.telemetry_route == "in_process" and not self.replay: + library_container = self.weblog_infra.library_container + assert isinstance(library_container, WeblogContainer), ( + "The in-process serverless-init topology requires a standard weblog container" + ) + library_container.wrap_with_serverless_init() + if not self.replay: + # This must run after EndToEndScenario has started the containers + # and confirmed weblog readiness. + self.warmups.append(self._wait_for_agentless_configuration) except BaseException: self._stop_mock_backend() raise + def get_image_list(self, library: str, weblog: str) -> list[str]: + images = super().get_image_list(library, weblog) + if self.telemetry_route == "in_process": + images.extend(ServerlessSidecarContainer().get_image_list(library, weblog)) + return images + def _set_containers_dependancies(self) -> None: super()._set_containers_dependancies() if not self._use_proxy_for_telemetry: @@ -726,7 +845,7 @@ def _set_containers_dependancies(self) -> None: def _start_interfaces_watchdog(self) -> None: super()._start_interfaces_watchdog() if self.telemetry_route != "none": - self.start_interfaces_watchdog([interfaces.ffe_sidecar, interfaces.ffe_direct]) + self.start_interfaces_watchdog([interfaces.ffe_sidecar, interfaces.ffe_direct, interfaces.ffe_relay]) def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool) -> None: super()._wait_and_stop_containers(force_interface_timout_to_zero=force_interface_timout_to_zero) @@ -747,11 +866,15 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool) -> interfaces.ffe_sidecar.check_deserialization_errors() interfaces.ffe_direct.check_deserialization_errors() + interfaces.ffe_relay.check_deserialization_errors() def _start_mock_backend(self, worker_id: str) -> None: assert self._mock_backend is None, "mock FFE agentless backend is already running" - self._mock_backend = MockFFEAgentlessBackendServer(worker_id) + self._mock_backend = MockFFEAgentlessBackendServer( + worker_id, + require_auth=self.telemetry_route == "none", + ) self._mock_backend.reset() environment = self.weblog_infra.library_container.environment diff --git a/utils/_context/containers.py b/utils/_context/containers.py index f9c4a54c2d1..b0bbcb77874 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -3,7 +3,9 @@ import platform import re import stat +import subprocess import sys +import tempfile import json from typing import cast, Literal from http import HTTPStatus @@ -52,6 +54,24 @@ _DEFAULT_NETWORK_NAME = "system-tests_default" _NETWORK_NAME = "bridge" if "GITLAB_CI" in os.environ else _DEFAULT_NETWORK_NAME +_SERVERLESS_INIT_IMAGE = "datadog/serverless-init:1.9.13" + + +def serverless_init_wrapper_dockerfile(base_image: str, init_image: str | None = None) -> str: + """Build a same-container serverless-init wrapper around a prepared weblog.""" + init_image = init_image or mirror_image(_SERVERLESS_INIT_IMAGE) + return ( + f"FROM {init_image} AS serverless-init\n" + f"FROM {base_image}\n" + "ARG TARGETARCH\n" + "USER root\n" + "COPY --from=serverless-init /datadog-init /datadog-init\n" + "COPY --from=serverless-init /lib/ /lib/\n" + 'RUN if [ "$TARGETARCH" = "amd64" ]; then mkdir -p /lib64 && ' + "ln -sf /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2; fi\n" + 'ENTRYPOINT ["/datadog-init"]\n' + 'LABEL system-tests-serverless-init-wrapper="true"\n' + ) def create_network() -> Network: @@ -1041,6 +1061,43 @@ def get_image_list(self, library: str | None, weblog: str | None) -> list[str]: return result + def wrap_with_serverless_init(self) -> None: + """Replace the prepared weblog image with an in-process init wrapper.""" + source_image = self.image.name + source_id = self.image._image.id.removeprefix("sha256:")[:12] # noqa: SLF001 - prepared image identity + source_platform = f"{self.image._image.attrs['Os']}/{self.image._image.attrs['Architecture']}" # noqa: SLF001 + worker_id = re.sub(r"[^a-z0-9_.-]", "-", os.environ.get("PYTEST_XDIST_WORKER", "master").lower()) + wrapper_image = f"system_tests/weblog-serverless-init:{worker_id}-{source_id}" + + with tempfile.TemporaryDirectory(prefix="system-tests-serverless-init-") as build_context: + Path(build_context, "Dockerfile").write_text( + serverless_init_wrapper_dockerfile(source_image), + encoding="utf-8", + ) + build = subprocess.run( + [ + "docker", + "build", + "--platform", + source_platform, + "--file", + str(Path(build_context, "Dockerfile")), + "--tag", + wrapper_image, + build_context, + ], + check=False, + capture_output=True, + text=True, + ) + if build.returncode != 0: + logger.stdout(build.stdout + build.stderr) + build.check_returncode() + + self.image = ImageInfo(wrapper_image, local_image_only=True) + self.image.load() + self.image.save_image_info(self.log_folder_path) + def configure(self, *, host_log_folder: str, replay: bool): super().configure(host_log_folder=host_log_folder, replay=replay) @@ -1131,7 +1188,7 @@ def flush(self) -> None: return try: - r = weblog.get("/flush", timeout=10) + r = weblog.get("/flush", timeout=20) assert r.status_code == HTTPStatus.OK except Exception as e: self.healthy = False diff --git a/utils/build/docker/nodejs/express/app.js b/utils/build/docker/nodejs/express/app.js index 88b6610e381..2f95519d2e1 100644 --- a/utils/build/docker/nodejs/express/app.js +++ b/utils/build/docker/nodejs/express/app.js @@ -638,6 +638,18 @@ app.get('/flush', (req, res) => { // does have a callback :) const promises = [] + if (process.env.DD_FEATURE_FLAGS_TELEMETRY_TRANSPORT) { + try { + const ddTraceDir = require('path').dirname(require.resolve('dd-trace')) + require(require.resolve('dc-polyfill', { paths: [ddTraceDir] })).channel('ffe:writers:flush').publish() + // FFE writer requests are asynchronous and do not expose a callback. + // Keep serverless-init alive for one full forwarding interval. + promises.push(new Promise(resolve => setTimeout(resolve, 12000))) + } catch (err) { + console.error('Unable to flush Feature Flags writers:', err) + } + } + try { const { profiler } = require('dd-trace/packages/dd-trace/src/profiling/') if (profiler?._collect) { @@ -836,7 +848,10 @@ app.post('/ai_guard/evaluate', async (req, res) => { let openFeatureClient = null // Initialize OpenFeature provider if FFE is enabled -if (process.env.DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED === 'true') { +if ( + process.env.DD_FEATURE_FLAGS_ENABLED === 'true' || + process.env.DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED === 'true' +) { const { openfeature } = tracer OpenFeature.setProvider(openfeature) openFeatureClient = OpenFeature.getClient() @@ -856,24 +871,26 @@ app.post('/ffe', async (req, res) => { for (const key of keys) { const context = { targetingKey: key, ...attributes } + let details switch (variationType) { case 'BOOLEAN': - value = await openFeatureClient.getBooleanValue(flag, defaultValue, context) + details = await openFeatureClient.getBooleanDetails(flag, defaultValue, context) break case 'STRING': - value = await openFeatureClient.getStringValue(flag, defaultValue, context) + details = await openFeatureClient.getStringDetails(flag, defaultValue, context) break case 'INTEGER': case 'NUMERIC': - value = await openFeatureClient.getNumberValue(flag, defaultValue, context) + details = await openFeatureClient.getNumberDetails(flag, defaultValue, context) break case 'JSON': - value = await openFeatureClient.getObjectValue(flag, defaultValue, context) + details = await openFeatureClient.getObjectDetails(flag, defaultValue, context) break default: return res.status(400).json({ error: `Unknown variation type: ${variationType}` }) } + value = details.value } res.status(200).json({ value, count: keys.length }) diff --git a/utils/build/docker/nodejs/express4-typescript/app.ts b/utils/build/docker/nodejs/express4-typescript/app.ts index b13ebf95612..1d03424e63a 100644 --- a/utils/build/docker/nodejs/express4-typescript/app.ts +++ b/utils/build/docker/nodejs/express4-typescript/app.ts @@ -361,6 +361,17 @@ app.get('/flush', (req: Request, res: Response) => { // does have a callback :) const promises = [] + if (process.env.DD_FEATURE_FLAGS_TELEMETRY_TRANSPORT) { + try { + const ddTraceDir = require('path').dirname(require.resolve('dd-trace')) + require(require.resolve('dc-polyfill', { paths: [ddTraceDir] })).channel('ffe:writers:flush').publish() + // FFE writer requests are asynchronous and do not expose a callback. + promises.push(new Promise(resolve => setTimeout(resolve, 1200))) + } catch (err) { + console.error('Unable to flush Feature Flags writers:', err) + } + } + const { profiler } = require('dd-trace/packages/dd-trace/src/profiling/') if (profiler?._collect) { promises.push(profiler._collect('on_shutdown')) diff --git a/utils/build/docker/nodejs/express4.Dockerfile b/utils/build/docker/nodejs/express4.Dockerfile index 1bc999e5faa..1e0d569a7d2 100644 --- a/utils/build/docker/nodejs/express4.Dockerfile +++ b/utils/build/docker/nodejs/express4.Dockerfile @@ -4,6 +4,7 @@ FROM datadog/system-tests:express4.base-v3 # runtime path so it is never bundled) so the /spawn_child endpoint is present. COPY utils/build/docker/nodejs/express/app.js app.js COPY utils/build/docker/nodejs/express/fork_child.js fork_child.js +COPY utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem /usr/local/share/ca-certificates/system-tests-mitmproxy-ca.pem EXPOSE 7777 diff --git a/utils/build/docker/nodejs/fastify/app.js b/utils/build/docker/nodejs/fastify/app.js index 49f5bfdde75..609b9df5c54 100644 --- a/utils/build/docker/nodejs/fastify/app.js +++ b/utils/build/docker/nodejs/fastify/app.js @@ -773,6 +773,17 @@ fastify.get('/flush', async (request, reply) => { // does have a callback :) const promises = [] + if (process.env.DD_FEATURE_FLAGS_TELEMETRY_TRANSPORT) { + try { + const ddTraceDir = require('path').dirname(require.resolve('dd-trace')) + require(require.resolve('dc-polyfill', { paths: [ddTraceDir] })).channel('ffe:writers:flush').publish() + // FFE writer requests are asynchronous and do not expose a callback. + promises.push(new Promise(resolve => setTimeout(resolve, 1200))) + } catch (err) { + console.error('Unable to flush Feature Flags writers:', err) + } + } + try { const { profiler } = require('dd-trace/packages/dd-trace/src/profiling/') if (profiler?._collect) { diff --git a/utils/docker_fixtures/_mock_ffe_agentless_backend.py b/utils/docker_fixtures/_mock_ffe_agentless_backend.py index d7076eef17e..966f7d2d4d8 100644 --- a/utils/docker_fixtures/_mock_ffe_agentless_backend.py +++ b/utils/docker_fixtures/_mock_ffe_agentless_backend.py @@ -136,9 +136,10 @@ def status(self) -> MockFFEAgentlessBackendStatus: class MockFFEAgentlessBackendHTTPServer(ThreadingHTTPServer): daemon_threads = True - def __init__(self, server_address: tuple[str, int]) -> None: + def __init__(self, server_address: tuple[str, int], *, require_auth: bool) -> None: super().__init__(server_address, MockFFEAgentlessBackendRequestHandler) self.state = MockFFEAgentlessBackendState() + self.require_auth = require_auth class MockFFEAgentlessBackendRequestHandler(BaseHTTPRequestHandler): @@ -183,6 +184,7 @@ def _handle_config(self) -> None: status_code, body, headers = _response_for_response( response=response, has_auth=_has_auth(request_headers), + require_auth=self.server.require_auth, ) self.server.state.record_response(status_code) with contextlib.suppress(BrokenPipeError, ConnectionResetError): @@ -236,7 +238,15 @@ def _has_auth(headers: Mapping[str, str]) -> bool: def _valid_ufc_bytes() -> bytes: - return UFC_FIXTURE_PATH.read_bytes() + configuration = json.loads(UFC_FIXTURE_PATH.read_bytes()) + return json.dumps( + { + "data": { + "type": "universal-flag-configuration", + "attributes": configuration, + } + } + ).encode() def validate_responses(responses: object) -> list[str]: @@ -253,8 +263,8 @@ def validate_responses(responses: object) -> list[str]: return responses -def _response_for_response(response: str, *, has_auth: bool) -> tuple[int, bytes, dict[str, str]]: - if not has_auth: +def _response_for_response(response: str, *, has_auth: bool, require_auth: bool) -> tuple[int, bytes, dict[str, str]]: + if require_auth and not has_auth: return HTTPStatus.UNAUTHORIZED, b"", {} if response == "unauthorized": @@ -273,9 +283,12 @@ def _strip_config_path(url: str) -> str: class MockFFEAgentlessBackendServer: - def __init__(self, worker_id: str) -> None: + def __init__(self, worker_id: str, *, require_auth: bool = True) -> None: self.port = get_host_port(worker_id, 4900) - self._server = MockFFEAgentlessBackendHTTPServer(("0.0.0.0", self.port)) # noqa: S104 - test fixture must be container-reachable. + self._server = MockFFEAgentlessBackendHTTPServer( + ("0.0.0.0", self.port), # noqa: S104 - test fixture must be container-reachable. + require_auth=require_auth, + ) self._thread = threading.Thread( target=self._server.serve_forever, name="mock-ffe-agentless-backend", daemon=True ) diff --git a/utils/interfaces/__init__.py b/utils/interfaces/__init__.py index 587daba23ee..ed19b4f347d 100644 --- a/utils/interfaces/__init__.py +++ b/utils/interfaces/__init__.py @@ -31,6 +31,7 @@ test_agent = _TestAgentInterfaceValidator() ffe_sidecar = FeatureFlagTelemetryInterfaceValidator("ffe_sidecar") ffe_direct = FeatureFlagTelemetryInterfaceValidator("ffe_direct") +ffe_relay = FeatureFlagTelemetryInterfaceValidator("ffe_relay") python_buddy = LibraryInterfaceValidator("python_buddy") nodejs_buddy = LibraryInterfaceValidator("nodejs_buddy") @@ -45,6 +46,7 @@ "agent_stdout", "backend", "ffe_direct", + "ffe_relay", "ffe_sidecar", "golang_buddy", "java_buddy", diff --git a/utils/proxy/core.py b/utils/proxy/core.py index 524d5b2b459..9e17ccb1338 100644 --- a/utils/proxy/core.py +++ b/utils/proxy/core.py @@ -40,7 +40,7 @@ # Used to create the stub TLS server cert (mitmproxy CA is always present at startup). _MITMPROXY_CA_PEM = "/app/utils/proxy/.mitmproxy/mitmproxy-ca.pem" -_MOCKED_BACKEND_PORTS = (ProxyPorts.agent, ProxyPorts.ffe_sidecar, ProxyPorts.ffe_direct) +_MOCKED_BACKEND_PORTS = (ProxyPorts.agent, ProxyPorts.ffe_sidecar, ProxyPorts.ffe_direct, ProxyPorts.ffe_relay) class _UDPForwarder(asyncio.DatagramProtocol): @@ -337,6 +337,8 @@ def response(self, flow: HTTPFlow): interface = "ffe_sidecar" elif proxy_port == ProxyPorts.ffe_direct: interface = "ffe_direct" + elif proxy_port == ProxyPorts.ffe_relay: + interface = "ffe_relay" else: raise ValueError(f"Unknown port provenance for {flow.request}: {proxy_port}") @@ -427,6 +429,7 @@ def start_proxy() -> None: f"regular@{ProxyPorts.agent}", # from agent to backend f"regular@{ProxyPorts.ffe_sidecar}", # Feature Flags sidecar to backend f"regular@{ProxyPorts.ffe_direct}", # Feature Flags SDK direct to backend + f"regular@{ProxyPorts.ffe_relay}", # programmable local Feature Flags relay f"regular@{ProxyPorts.otel_collector}", # from otel collector to backend ] diff --git a/utils/proxy/ports.py b/utils/proxy/ports.py index b6615a4490c..b2854334512 100644 --- a/utils/proxy/ports.py +++ b/utils/proxy/ports.py @@ -23,6 +23,9 @@ class ProxyPorts(IntEnum): ffe_direct = 8202 """Feature Flags telemetry emitted directly by the SDK to Datadog intake.""" + ffe_relay = 8203 + """Programmable local Feature Flags relay used for capability and fallback contracts.""" + python_buddy = 9001 nodejs_buddy = 9002 java_buddy = 9003 From 4bb0f61971e349536c2826d0194466d453080972 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 24 Jul 2026 18:39:17 -0600 Subject: [PATCH 2/2] test(ffe): keep Node.js agentless EVP gated --- manifests/nodejs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index b5cc16d007c..1c04172858b 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1708,6 +1708,8 @@ manifest: component_version: <5.66.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: *ref_5_83_0 tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2697 tracks Node.js agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: