diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 8d9b8df0aae..782e71a74d1 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -323,6 +323,12 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS + - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT"') + run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT + - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS"') + run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"') run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 6d3051b2f52..db02d002385 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -42,7 +42,8 @@ manifest: tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature - tests/ffe/test_exposures.py: missing_feature + tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index ce7ca0a8801..d770d58a7b8 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -275,7 +275,8 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature - tests/ffe/test_exposures.py: missing_feature + tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 7fb1ca12d51..83b44d6b061 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -728,7 +728,10 @@ manifest: tests/ffe/test_dynamic_evaluation.py: v3.36.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) - tests/ffe/test_exposures.py: v3.36.0 + tests/ffe/test_exposure_egress.py: v3.36.0 + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: v3.36.0 tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: v3.44.0 tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) diff --git a/manifests/golang.yml b/manifests/golang.yml index f08c4d4f2de..f76ba47ed40 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1156,7 +1156,10 @@ manifest: tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance::test_unknown_operator_errors: bug (FFL-2182) - tests/ffe/test_exposures.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 + tests/ffe/test_exposure_egress.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 tests/ffe/test_flag_eval_evp.py: v2.10.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation::test_ffe_evp_flagevaluation_degradation: flaky (FFL-2676) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2784) diff --git a/manifests/java.yml b/manifests/java.yml index 318a6924f9f..9a2f22a9144 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3297,11 +3297,17 @@ manifest: weblog: [spring-boot] - declaration: bug (FFL-2184) excluded_weblog: [spring-boot] - tests/ffe/test_exposures.py: + tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": irrelevant spring-boot: v1.56.0 - tests/ffe/test_exposures.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1729) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: + - weblog_declaration: + "*": irrelevant + spring-boot: v1.56.0 + tests/ffe/test_exposures_datadog_agent.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1729) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index b8053404a7f..e1c589e59b7 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1749,11 +1749,15 @@ manifest: "*": incomplete_test_app express4: *ref_5_77_0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance::test_unknown_operator_errors: bug (FFL-2182) - tests/ffe/test_exposures.py: + tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": incomplete_test_app express4: *ref_5_77_0 - tests/ffe/test_exposures.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1730) + tests/ffe/test_exposures_datadog_agent.py: + - weblog_declaration: + "*": incomplete_test_app + express4: *ref_5_77_0 + tests/ffe/test_exposures_datadog_agent.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1730) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: diff --git a/manifests/php.yml b/manifests/php.yml index 5a44e2dc4fc..9745557f6fb 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -800,12 +800,19 @@ manifest: "*": v1.21.0-dev laravel11x: incomplete_test_app symfony7x: incomplete_test_app - tests/ffe/test_exposures.py: + tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": v1.21.0-dev laravel11x: incomplete_test_app symfony7x: incomplete_test_app - tests/ffe/test_exposures.py::Test_FFE_Exposure_Events::test_ffe_multiple_remote_config_files: flaky (FFL-2676) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: + - weblog_declaration: + "*": v1.21.0-dev + laravel11x: incomplete_test_app + symfony7x: incomplete_test_app + tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Events::test_ffe_multiple_remote_config_files: flaky (FFL-2676) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: bug (FFL-2676) tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation: bug (FFL-2676) diff --git a/manifests/python.yml b/manifests/python.yml index 76b5c64a600..cd94cf90207 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1382,7 +1382,10 @@ manifest: tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2699 tracks Python agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) - tests/ffe/test_exposures.py: v4.2.0-dev + tests/ffe/test_exposure_egress.py: v4.2.0-dev + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: v4.2.0-dev tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: v4.7.0 tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 115925750e6..922249799c1 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2002,7 +2002,13 @@ manifest: - weblog_declaration: "*": irrelevant rails72: v2.23.0-dev - tests/ffe/test_exposures.py: + tests/ffe/test_exposure_egress.py: + - weblog_declaration: + "*": irrelevant + rails72: v2.23.0-dev + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: - weblog_declaration: "*": irrelevant rails72: v2.23.0-dev diff --git a/manifests/rust.yml b/manifests/rust.yml index 7fc8dc4f4b1..4509b6963fc 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -22,7 +22,8 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature - tests/ffe/test_exposures.py: missing_feature + tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) + tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) diff --git a/mirror_images.lock.yaml b/mirror_images.lock.yaml index ed645a6af58..f18320ace07 100644 --- a/mirror_images.lock.yaml +++ b/mirror_images.lock.yaml @@ -26,6 +26,10 @@ images: digest: sha256:0199e251729b25276f97049cbcd39dd1997e98413a07b10a3c7e9e755d505a42 target: registry.ddbuild.io/system-tests/mirror/datadog/dd-appsec-php-ci:php-8.2-release tag: php-8.2-release + datadog/serverless-init:1.9.13: + digest: sha256:d187a970ce66d6f1ec4ba26e7c38783ad2e1c6e423fc1b1d39e81ba6332317cf + target: registry.ddbuild.io/system-tests/mirror/datadog/serverless-init:1.9.13 + tag: 1.9.13 datadog/system-tests:apache-mod-7.0-zts.base-v1: digest: sha256:bafc072734341cafb86f45673df2de4851a04b951d0717c88f7958cac1815b20 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.0-zts.base-v1 diff --git a/mirror_images.yaml b/mirror_images.yaml index ceaae4dd45b..cd32d925f87 100644 --- a/mirror_images.yaml +++ b/mirror_images.yaml @@ -13,6 +13,7 @@ - "datadog/agent:latest" - "datadog/dd-appsec-php-ci:php-8.0-release" - "datadog/dd-appsec-php-ci:php-8.2-release" +- "datadog/serverless-init:1.9.13" - "datadog/system-tests:apache-mod-7.0-zts.base-v1" - "datadog/system-tests:apache-mod-7.0.base-v1" - "datadog/system-tests:apache-mod-7.1-zts.base-v1" diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 9aca2fb8dba..e1ab63cf1a0 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -7,7 +7,8 @@ This directory contains system tests for the Feature Flags & Experimentation (FF | File | Description | |------|-------------| | `test_dynamic_evaluation.py` | Dynamic flag evaluation via Remote Config | -| `test_exposures.py` | Flag exposure tracking and reporting | +| `test_exposure_egress.py` | Shared exposure cache and payload contract across Agent, sidecar, and direct egress | +| `test_exposures_datadog_agent.py` | Flag exposure tracking and reporting through the Datadog Agent | | `test_flag_eval_metrics.py` | Evaluation metrics (OTel counter) | | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds | diff --git a/tests/ffe/test_exposure_egress.py b/tests/ffe/test_exposure_egress.py new file mode 100644 index 00000000000..f7dcf78ab7f --- /dev/null +++ b/tests/ffe/test_exposure_egress.py @@ -0,0 +1,120 @@ +"""Test one exposure contract through every supported deployment topology.""" + +from dataclasses import dataclass + +from tests.ffe.utils.exposures import assert_exposure_side_effects_contract, exposure_events_from_data +from tests.ffe.utils.fixtures import make_ufc_fixture +from utils import context, features, interfaces, remote_config as rc, scenarios, weblog +from utils._context._scenarios.endtoend import FeatureFlaggingAgentlessEndToEndScenario +from utils._context.component_version import Version +from utils.interfaces._core import ProxyBasedInterfaceValidator +from utils.mocked_backend.ffe import EXPECTED_API_KEY + +RC_PATH = "datadog/2/FFE_FLAGS" + + +@dataclass(frozen=True) +class ExposureEgress: + interface: ProxyBasedInterfaceValidator + excluded_interfaces: tuple[ProxyBasedInterfaceValidator, ...] = () + expected_api_key: str | None = None + + +def exposure_egress() -> ExposureEgress: + """Return the capture interface and route-only expectations for this topology.""" + scenario = context.scenario + if not isinstance(scenario, FeatureFlaggingAgentlessEndToEndScenario): + assert scenario.name == "FEATURE_FLAGGING_AND_EXPERIMENTATION" + return ExposureEgress(interfaces.agent) + + if scenario.exposure_egress == "sidecar": + assert scenario.components["serverless-init"] == Version("1.9.13") + expected_api_key = scenario.serverless_init_container.environment["DD_API_KEY"] + assert expected_api_key is not None + return ExposureEgress( + interfaces.datadog_sidecar, + (interfaces.datadog_direct,), + expected_api_key, + ) + + assert scenario.exposure_egress == "direct" + assert "serverless-init" not in scenario.components + return ExposureEgress( + interfaces.datadog_direct, + (interfaces.datadog_sidecar,), + EXPECTED_API_KEY, + ) + + +class ExposureEgressContract: + """One exposure contract inherited by each supported topology adapter.""" + + flag_key = "empty-targeting-key-flag" + targeting_key = "exposure-egress-user" + + def setup_exposure_egress(self) -> None: + if not isinstance(context.scenario, FeatureFlaggingAgentlessEndToEndScenario): + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/exposure-egress/config", + make_ufc_fixture(self.flag_key), + ).apply() + + self.responses = [ + weblog.post( + "/ffe", + json={ + "flag": self.flag_key, + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": self.targeting_key, + "attributes": {}, + }, + ) + for _ in range(5) + ] + + def test_exposure_egress(self) -> None: + egress = exposure_egress() + matching_requests = assert_exposure_side_effects_contract( + egress.interface, + self.responses, + flag_key=self.flag_key, + targeting_key=self.targeting_key, + expected_value="on-value", + expected_variant="on", + ) + assert len(matching_requests) == 1 + + if egress.expected_api_key is None: + return + + request = matching_requests[0] + assert request["host"] == "event-platform-intake.datad0g.com" + assert request["response"]["status_code"] == 202 + + headers = {name.lower(): value for name, value in request["request"]["headers"]} + assert headers["dd-api-key"] in {egress.expected_api_key, "--redacted--"} + + for excluded_interface in egress.excluded_interfaces: + assert not any( + exposure_events_from_data(data, {self.flag_key}, self.targeting_key) + for data in excluded_interface.get_data() + ) + + +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_exposures +class Test_FFE_Exposure_Egress_Datadog_Agent(ExposureEgressContract): + pass + + +@scenarios.feature_flagging_and_experimentation_agentless_direct +@features.feature_flags_exposures +class Test_FFE_Exposure_Egress_Agentless_Direct(ExposureEgressContract): + pass + + +@scenarios.feature_flagging_and_experimentation_agentless_serverless +@features.feature_flags_exposures +class Test_FFE_Exposure_Egress_Agentless_Sidecar(ExposureEgressContract): + pass diff --git a/tests/ffe/test_exposures.py b/tests/ffe/test_exposures_datadog_agent.py similarity index 91% rename from tests/ffe/test_exposures.py rename to tests/ffe/test_exposures_datadog_agent.py index 79b523392bc..e2d6b79f01d 100644 --- a/tests/ffe/test_exposures.py +++ b/tests/ffe/test_exposures_datadog_agent.py @@ -1,7 +1,12 @@ -"""Test feature flags exposure events logging in weblog end-to-end scenario.""" +"""Test Feature Flags exposure events through the Datadog Agent.""" import json +from tests.ffe.utils.exposures import ( + EXPOSURES_PATH, + EXPOSURE_WAIT_TIMEOUT_SECONDS, + exposure_events_from_data, +) from tests.ffe.utils.fixtures import make_exposure_ufc_fixture as make_ufc_fixture from utils import ( weblog, @@ -14,41 +19,6 @@ RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" -EXPOSURES_PATH = "/api/v2/exposures" -EXPOSURE_WAIT_TIMEOUT_SECONDS = 30 - - -def exposure_events_from_data( - data: dict, flag_keys: set[str] | None = None, subject_id: str | None = None -) -> list[dict]: - """Return exposure events from one agent payload matching the optional flag/subject filters.""" - if data.get("path") != EXPOSURES_PATH: - return [] - - exposure_data = data.get("request", {}).get("content") - if not isinstance(exposure_data, dict): - return [] - - exposures = exposure_data.get("exposures") - if not isinstance(exposures, list): - return [] - - events = [] - for event in exposures: - if not isinstance(event, dict): - continue - - flag = event.get("flag") - subject = event.get("subject") - event_flag_key = flag.get("key") if isinstance(flag, dict) else None - event_subject_id = subject.get("id") if isinstance(subject, dict) else None - - if flag_keys is not None and event_flag_key not in flag_keys: - continue - if subject_id is not None and event_subject_id != subject_id: - continue - events.append(event) - return events def find_exposure_events(flag_key: str, subject_id: str | None = None) -> list[dict]: @@ -503,57 +473,6 @@ def count_exposure_events(flag_key: str, subject_id: str | None = None) -> int: return len(find_exposure_events(flag_key, subject_id)) -@scenarios.feature_flagging_and_experimentation -@features.feature_flags_exposures -class Test_FFE_Exposure_Caching_Same_Subject: - """Test that exposure caching deduplicates events for the same (subject, allocation, variant). - - When the same subject evaluates the same flag multiple times and gets the same variant, - only one exposure event should be generated due to the exposure cache. - """ - - def setup_ffe_exposure_caching_same_subject(self): - """Set up FFE exposure caching test with multiple evaluations for the same subject.""" - config_id = "ffe-caching-test" - self.flag_key = "same-subject-test-flag" # Unique flag key for this test - rc.tracer_rc_state.reset().set_config(f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key)).apply() - - self.targeting_key = "same-subject-user" - - # Evaluate the same flag multiple times with the same subject - self.responses = [] - for _i in range(5): - r = weblog.post( - "/ffe", - json={ - "flag": self.flag_key, - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": self.targeting_key, - "attributes": {}, - }, - ) - self.responses.append(r) - - def test_ffe_exposure_caching_same_subject(self): - """Test that multiple evaluations for the same subject generate at most one exposure event.""" - # Verify all requests succeeded - for i, r in enumerate(self.responses): - assert r.status_code == 200, f"Request {i + 1} failed: {r.text}" - result = json.loads(r.text) - assert result["value"] == "value-a", f"Request {i + 1}: expected 'value-a', got '{result['value']}'" - - # Count exposure events for this specific subject - exposure_count = wait_for_min_exposure_count(self.flag_key, 1, self.targeting_key) - - # The exposure cache should deduplicate events - we expect exactly 1 exposure - # for the same (subject, allocation, variant) tuple - assert exposure_count == 1, ( - f"Expected exactly 1 exposure event for subject '{self.targeting_key}' due to caching, " - f"but found {exposure_count} events" - ) - - @scenarios.feature_flagging_and_experimentation @features.feature_flags_exposures class Test_FFE_Exposure_Caching_Different_Subjects: diff --git a/tests/ffe/utils/exposures.py b/tests/ffe/utils/exposures.py new file mode 100644 index 00000000000..8076fa02bea --- /dev/null +++ b/tests/ffe/utils/exposures.py @@ -0,0 +1,93 @@ +"""Route-neutral Feature Flags exposure expectations.""" + +from collections.abc import Iterable +import json + +from utils._weblog import HttpResponse +from utils.interfaces._core import ProxyBasedInterfaceValidator + +EXPOSURES_PATH = "/api/v2/exposures" +EXPOSURE_WAIT_TIMEOUT_SECONDS = 30 +EXPOSURE_CACHE_SETTLE_SECONDS = 3 + + +def exposure_events_from_data( + data: dict, flag_keys: set[str] | None = None, subject_id: str | None = None +) -> list[dict]: + """Return matching exposure events from one captured request.""" + if data.get("path") != EXPOSURES_PATH: + return [] + + exposure_data = data.get("request", {}).get("content") + if not isinstance(exposure_data, dict): + return [] + + exposures = exposure_data.get("exposures") + if not isinstance(exposures, list): + return [] + + events = [] + for event in exposures: + if not isinstance(event, dict): + continue + + flag = event.get("flag") + subject = event.get("subject") + event_flag_key = flag.get("key") if isinstance(flag, dict) else None + event_subject_id = subject.get("id") if isinstance(subject, dict) else None + + if flag_keys is not None and event_flag_key not in flag_keys: + continue + if subject_id is not None and event_subject_id != subject_id: + continue + events.append(event) + return events + + +def assert_exposure_side_effects_contract( + interface: ProxyBasedInterfaceValidator, + responses: Iterable[HttpResponse], + *, + flag_key: str, + targeting_key: str, + expected_value: str, + expected_variant: str, + expected_allocation: str = "default-allocation", +) -> list[dict]: + """Assert one exposure contract through Agent, sidecar, or direct intake capture.""" + for index, response in enumerate(responses, start=1): + assert response.status_code == 200, f"Evaluation {index} failed: {response.text}" + value = json.loads(response.text)["value"] + assert value == expected_value, f"Evaluation {index} returned {value!r}" + + def matches_exposure(data: dict) -> bool: + return bool(exposure_events_from_data(data, {flag_key}, targeting_key)) + + assert interface.wait_for( + matches_exposure, + timeout=EXPOSURE_WAIT_TIMEOUT_SECONDS, + ), f"Timed out waiting for exposure event for {flag_key!r} and {targeting_key!r}" + + if not interface.replay: + interface.wait(EXPOSURE_CACHE_SETTLE_SECONDS) + + matching_requests = [data for data in interface.get_data() if matches_exposure(data)] + events = [ + event + for request in matching_requests + for event in exposure_events_from_data(request, {flag_key}, targeting_key) + ] + assert len(events) == 1, f"The exposure cache produced {len(events)} matching events instead of one" + + for request in matching_requests: + context = request["request"]["content"]["context"] + assert context["service"] == "weblog" + assert context["version"] == "1.0.0" + assert context["env"] == "system-tests" + + event = events[0] + assert event["flag"]["key"] == flag_key + assert event["variant"]["key"] == expected_variant + assert event["allocation"]["key"] == expected_allocation + assert event["subject"]["id"] == targeting_key + return matching_requests diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index 11e4470478c..f732e256f98 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -495,6 +495,20 @@ def test_agentless_ffe_test_file(self): "", ) + def test_exposure_egress_test_file(self): + inputs = build_inputs(modified_files=["tests/ffe/test_exposure_egress.py"]) + assert_github_processor( + inputs, + default_libs_with_prod, + default_libs_with_dev, + 3600, + "false", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION," + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT," + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS", + "", + ) + def test_agentless_ffe_mocked_backend_file(self): inputs = build_inputs(modified_files=["utils/mocked_backend/ffe.py"]) assert_github_processor( @@ -503,7 +517,9 @@ def test_agentless_ffe_mocked_backend_file(self): default_libs_with_dev, 3600, "false", - "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS,PARAMETRIC", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS," + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT," + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS,PARAMETRIC", "", ) @@ -515,7 +531,9 @@ def test_end_to_end_scenario_framework_file(self): default_libs_with_dev, 3600, "false", - "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS," + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT," + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS", "end_to_end", ) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index dda1b74ffbf..5f0b8e686c3 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -25,10 +25,23 @@ def test_tracer_release(): assert scenario_groups.end_to_end not in dormant_agentless_scenario.scenario_groups assert scenario_groups.tracer_release not in dormant_agentless_scenario.scenario_groups + agentless_exposure_scenarios = [ + scenarios.feature_flagging_and_experimentation_agentless_direct, + scenarios.feature_flagging_and_experimentation_agentless_serverless, + ] + for exposure_scenario in agentless_exposure_scenarios: + assert exposure_scenario.include_agent is False + assert exposure_scenario.use_proxy is True + assert scenario_groups.ffe in exposure_scenario.scenario_groups + assert scenario_groups.all not in exposure_scenario.scenario_groups + assert scenario_groups.end_to_end not in exposure_scenario.scenario_groups + assert scenario_groups.tracer_release not in exposure_scenario.scenario_groups + not_in_tracer_release_group = [ # list of scenario that will never be part of tracer release scenarios.fuzzer, dormant_agentless_scenario, + *agentless_exposure_scenarios, scenarios.mock_the_test, scenarios.mock_the_test_2, scenarios.test_the_test, diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index f5747101723..1a2f7e1df58 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -1,12 +1,14 @@ """Unit coverage for the mock FFE agentless backend test fixture.""" from pathlib import Path +from typing import Literal from unittest.mock import MagicMock import requests import pytest -from utils import scenarios +from utils import features, scenarios +from utils._context.containers import ServerlessInitContainer from utils._context._scenarios import endtoend as endtoend_scenarios from utils.docker_fixtures._core import HOST_GATEWAY_EXTRA_HOSTS, extra_hosts_for_environment from utils.mocked_backend.ffe import ( @@ -17,6 +19,7 @@ UFC_RESPONSE_TYPE, ) from utils._context._scenarios.endtoend import FeatureFlaggingAgentlessEndToEndScenario +from utils.proxy.ports import ProxyPorts @scenarios.test_the_test @@ -124,6 +127,45 @@ def test_agentless_end_to_end_scenario_starts_backend_before_weblog() -> None: scenario._stop_mock_backend() # noqa: SLF001 - focused lifecycle test +@pytest.mark.parametrize("exposure_egress", ["direct", "sidecar"]) +@scenarios.test_the_test +@features.not_reported +def test_agentless_exposure_scenario_has_no_agent_and_two_capture_routes( + exposure_egress: Literal["direct", "sidecar"], +) -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_EXPOSURES", + doc="test", + exposure_egress=exposure_egress, + ) + + environment = scenario.weblog_infra.library_container.environment + assert scenario.agent_container not in scenario._containers # noqa: SLF001 - focused topology test + assert scenario.proxy_container in scenario._containers # noqa: SLF001 - focused topology test + assert scenario.get_libraries() is None + assert environment["DD_PROXY_HTTPS"] == f"http://proxy:{ProxyPorts.datadog_direct}" + assert environment["HTTPS_PROXY"] == f"http://proxy:{ProxyPorts.datadog_direct}" + assert environment["NODE_EXTRA_CA_CERTS"] == "/usr/local/share/ca-certificates/system-tests-mitmproxy-ca.pem" + assert scenario.weblog_infra.library_container.volumes["./utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem"] == { + "bind": "/usr/local/share/ca-certificates/system-tests-mitmproxy-ca.pem", + "mode": "ro", + } + + if exposure_egress == "direct": + assert "DD_TRACE_AGENT_URL" not in environment + assert not any( + isinstance(container, ServerlessInitContainer) + for container in scenario._containers # noqa: SLF001 - focused topology test + ) + return + + assert environment["DD_TRACE_AGENT_URL"] == "http://ffe-serverless-init:8126" + serverless_init = scenario.serverless_init_container + assert isinstance(serverless_init, ServerlessInitContainer) + assert serverless_init.image.name == "datadog/serverless-init:1.9.13" + assert serverless_init.environment["DD_PROXY_HTTPS"] == f"http://proxy:{ProxyPorts.datadog_sidecar}" + + @scenarios.test_the_test def test_agentless_end_to_end_scenario_closes_backend_when_startup_fails( monkeypatch: pytest.MonkeyPatch, diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index ecf999781c9..2557cce1cba 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -793,6 +793,18 @@ class _Scenarios: "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS" ) + feature_flagging_and_experimentation_agentless_direct = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT", + doc="Validate direct exposure delivery with agentless UFC and no local receiver.", + exposure_egress="direct", + ) + + feature_flagging_and_experimentation_agentless_serverless = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS", + doc="Validate exposure delivery with agentless UFC and serverless-init 1.9.13.", + exposure_egress="sidecar", + ) + remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario( "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE", rc_api_enabled=True, diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index a213b207620..6caab7e7124 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -23,6 +23,7 @@ ProxyContainer, BuddyContainer, TestedContainer, + ServerlessInitContainer, ) from utils._context.weblog_infrastructure import EndToEndWeblogInfra from utils.docker_fixtures._core import extra_hosts_for_environment @@ -628,24 +629,50 @@ def __init__( name: str, *, doc: str = "Validate default agentless UFC delivery and evaluation without a Datadog Agent.", + exposure_egress: Literal["sidecar", "direct"] | None = None, weblog_env: dict[str, str | None] | None = None, ) -> None: + self.exposure_egress = exposure_egress environment: dict[str, str | None] = { - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "1", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", "DD_REMOTE_CONFIGURATION_ENABLED": "false", } environment.update(weblog_env or {}) + other_weblog_containers: tuple[type[TestedContainer], ...] = () + weblog_volumes: dict[str, dict[str, str]] = {} + if exposure_egress is not None: + environment |= { + "DD_SITE": "datad0g.com", + "DD_PROXY_HTTPS": f"http://proxy:{ProxyPorts.datadog_direct}", + "HTTPS_PROXY": f"http://proxy:{ProxyPorts.datadog_direct}", + "NODE_EXTRA_CA_CERTS": "/usr/local/share/ca-certificates/system-tests-mitmproxy-ca.pem", + } + weblog_volumes["./utils/proxy/.mitmproxy/mitmproxy-ca-cert.pem"] = { + "bind": "/usr/local/share/ca-certificates/system-tests-mitmproxy-ca.pem", + "mode": "ro", + } + + if exposure_egress == "sidecar": + environment |= { + "DD_AGENT_HOST": "ffe-serverless-init", + "DD_TRACE_AGENT_PORT": "8126", + "DD_TRACE_AGENT_URL": "http://ffe-serverless-init:8126", + } + other_weblog_containers = (ServerlessInitContainer,) + super().__init__( name, doc=doc, include_agent=False, library_interface_timeout=0, + other_weblog_containers=other_weblog_containers, scenario_groups=[all_scenario_groups.ffe], use_proxy_for_agent=False, - use_proxy_for_weblog=False, + use_proxy_for_weblog=exposure_egress is not None, weblog_env=environment, + weblog_volumes=weblog_volumes, ) def configure(self, config: pytest.Config) -> None: @@ -657,10 +684,54 @@ def configure(self, config: pytest.Config) -> None: self._start_mock_backend() super().configure(config) + if self.exposure_egress is not None: + interfaces.datadog_sidecar.configure(self.host_log_folder, replay=self.replay) + interfaces.datadog_direct.configure(self.host_log_folder, replay=self.replay) except BaseException: self._stop_mock_backend(persist_status=False) raise + @property + def serverless_init_container(self) -> ServerlessInitContainer: + for container in self.weblog_infra.get_containers(): + if isinstance(container, ServerlessInitContainer): + return container + raise ValueError("This scenario has no serverless-init container") + + def _set_containers_dependancies(self) -> None: + super()._set_containers_dependancies() + if self.exposure_egress == "sidecar": + self.serverless_init_container.depends_on.append(self.proxy_container) + + def _start_interfaces_watchdog(self) -> None: + super()._start_interfaces_watchdog() + if self.exposure_egress is not None: + self.start_interfaces_watchdog([interfaces.datadog_sidecar, interfaces.datadog_direct]) + + def _wait_for_app_readiness(self) -> None: + if self.exposure_egress is not None: + return + super()._wait_for_app_readiness() + + def _set_components(self) -> None: + super()._set_components() + if self.exposure_egress == "sidecar": + self.components["serverless-init"] = self.serverless_init_container.serverless_init_version + + def _wait_and_stop_containers(self, *, is_empty_test_run: bool) -> None: + super()._wait_and_stop_containers(is_empty_test_run=is_empty_test_run) + if self.exposure_egress is None: + return + + if self.replay: + interfaces.datadog_sidecar.load_data_from_logs() + interfaces.datadog_direct.load_data_from_logs() + elif self.exposure_egress == "sidecar": + self.serverless_init_container.stop() + + interfaces.datadog_sidecar.check_deserialization_errors() + interfaces.datadog_direct.check_deserialization_errors() + def _start_mock_backend(self) -> None: assert self._mock_backend is None, "mock FFE agentless backend is already running" diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 5f60d546d42..500fd67d2f6 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -838,6 +838,36 @@ def dd_site(self): return os.environ.get("DD_SITE", "datad0g.com") +class ServerlessInitContainer(TestedContainer): + """Run serverless-init as the local trace and Feature Flags relay.""" + + def __init__(self) -> None: + super().__init__( + name="ffe-serverless-init", + image_name="datadog/serverless-init:1.9.13", + environment={ + "DD_API_KEY": _FAKE_DD_API_KEY, + "DD_SITE": "datad0g.com", + "DD_SERVICE": "ffe-system-tests-serverless-init", + "DD_ENV": "system-tests", + "DD_APM_ENABLED": "true", + "DD_APM_NON_LOCAL_TRAFFIC": "true", + "DD_PROXY_HTTPS": f"http://proxy:{ProxyPorts.datadog_sidecar}", + "DD_PROXY_HTTP": f"http://proxy:{ProxyPorts.datadog_sidecar}", + "DD_SERVERLESS_FLUSH_STRATEGY": "periodically,100", + "DD_SKIP_SSL_VALIDATION": "true", + }, + ) + + @property + def serverless_init_version(self) -> Version: + """Read the serverless-init version from the image metadata.""" + version = self.image.labels.get("org.opencontainers.image.version") + if not version: + raise ValueError("The serverless-init image has no OCI version label") + return Version(version) + + class BuddyContainer(TestedContainer): def __init__( self, diff --git a/utils/interfaces/__init__.py b/utils/interfaces/__init__.py index 75da1787ddd..1b5f2f5cab0 100644 --- a/utils/interfaces/__init__.py +++ b/utils/interfaces/__init__.py @@ -26,6 +26,8 @@ backend = _BackendInterfaceValidator(library_interface=library) open_telemetry = OpenTelemetryInterfaceValidator() otel_collector = ProxyBasedInterfaceValidator("otel_collector") +datadog_sidecar = ProxyBasedInterfaceValidator("datadog_sidecar") +datadog_direct = ProxyBasedInterfaceValidator("datadog_direct") postgres = _PostgresStdout() test_agent = _TestAgentInterfaceValidator() @@ -41,6 +43,8 @@ "agent", "agent_stdout", "backend", + "datadog_direct", + "datadog_sidecar", "golang_buddy", "java_buddy", "library", diff --git a/utils/proxy/core.py b/utils/proxy/core.py index d974392293f..7a68e06e0e5 100644 --- a/utils/proxy/core.py +++ b/utils/proxy/core.py @@ -40,6 +40,8 @@ # Used to create the stub TLS server cert (mitmproxy CA is always present at startup). _MITMPROXY_CA_PEM = "/app/utils/proxy/.mitmproxy/mitmproxy-ca.pem" +_MOCKED_BACKEND_PORTS = (ProxyPorts.agent, ProxyPorts.datadog_sidecar, ProxyPorts.datadog_direct) + class _UDPForwarder(asyncio.DatagramProtocol): def __init__(self, target_host: str, target_port: int) -> None: @@ -188,7 +190,7 @@ async def running(self) -> None: def http_connect(self, flow: HTTPFlow) -> None: proxy_port = flow.client_conn.sockname[1] logger.info(f"Flow {flow.id}: CONNECT {flow.request.host}:{flow.request.port} using proxy port {proxy_port}") - if proxy_port == ProxyPorts.agent and self.mocked_backend: + if proxy_port in _MOCKED_BACKEND_PORTS and self.mocked_backend: # Redirect to local stub TLS server so mitmproxy can always complete tunnel setup. # Without this, CONNECT handshake is performed to the backend, and if ever it fails, # request() never fires. @@ -269,7 +271,7 @@ def request(self, flow: HTTPFlow): ) flow.request.scheme = "http" logger.info(f"Flow {flow.id}: reverse proxy to {flow.request.pretty_url}") - elif proxy_port == ProxyPorts.agent and self.mocked_backend: + elif proxy_port in _MOCKED_BACKEND_PORTS and self.mocked_backend: # Since we are faking the backend (generating responses from # scratch), the logic is that the first mock satisfying the # condition wins. Consequently, we check runtime mocks (controlled @@ -328,6 +330,10 @@ def response(self, flow: HTTPFlow): interface = "golang_buddy" elif proxy_port == ProxyPorts.agent: # HTTPS port, as the agent use the proxy with HTTP_PROXY env var interface = "agent" + elif proxy_port == ProxyPorts.datadog_sidecar: + interface = "datadog_sidecar" + elif proxy_port == ProxyPorts.datadog_direct: + interface = "datadog_direct" else: raise ValueError(f"Unknown port provenance for {flow.request}: {proxy_port}") @@ -416,6 +422,8 @@ def start_proxy() -> None: f"regular@{ProxyPorts.golang_buddy}", # golang_buddy f"regular@{ProxyPorts.open_telemetry_weblog}", # Open telemetry weblog f"regular@{ProxyPorts.agent}", # from agent to backend + f"regular@{ProxyPorts.datadog_sidecar}", # Datadog sidecar traffic + f"regular@{ProxyPorts.datadog_direct}", # Datadog direct intake traffic f"regular@{ProxyPorts.otel_collector}", # from otel collector to backend ] diff --git a/utils/proxy/ports.py b/utils/proxy/ports.py index 66f720e031d..676661e0754 100644 --- a/utils/proxy/ports.py +++ b/utils/proxy/ports.py @@ -17,6 +17,12 @@ class ProxyPorts(IntEnum): agent = 8200 + datadog_sidecar = 8201 + """Datadog traffic from the serverless-init sidecar.""" + + datadog_direct = 8202 + """Datadog traffic sent directly by the SDK.""" + python_buddy = 9001 nodejs_buddy = 9002 java_buddy = 9003 diff --git a/utils/scripts/libraries_and_scenarios_rules.yml b/utils/scripts/libraries_and_scenarios_rules.yml index c08fcb2a859..699ba506622 100644 --- a/utils/scripts/libraries_and_scenarios_rules.yml +++ b/utils/scripts/libraries_and_scenarios_rules.yml @@ -224,7 +224,10 @@ patterns: scenario_groups: appsec_rasp_scenario utils/_context/_scenarios/endtoend.py: scenario_groups: end_to_end - scenarios: FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS + scenarios: + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS utils/_context/_scenarios/integration_frameworks.py: scenarios: INTEGRATION_FRAMEWORKS utils/_context/_scenarios/integrations.py: @@ -291,7 +294,11 @@ patterns: scenario_groups: [end_to_end, open_telemetry] utils/mocked_backend/ffe.py: - scenarios: [FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS, PARAMETRIC] + scenarios: + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS + - PARAMETRIC utils/_context/virtual_machine.py: scenario_groups: onboarding @@ -558,6 +565,19 @@ patterns: scenario_groups: null libraries: null + tests/ffe/test_exposure_egress.py: + scenarios: + - FEATURE_FLAGGING_AND_EXPERIMENTATION + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS + + tests/ffe/utils/exposures.py: + # This helper is shared by Agent and agentless exposure tests. + scenarios: + - FEATURE_FLAGGING_AND_EXPERIMENTATION + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT + - FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS + # This helper has no test items of its own, so it's invisible to the scenario map that # drives dynamic selection below. List its consumers' scenarios explicitly. tests/test_telemetry_heartbeat_utils.py: