test(ffe): enable Node.js agentless serverless exposures [nodejs@ffl-2697-nodejs-serverless-evp-routing]

.github/workflows/run-end-to-end.yml

@@ -323,6 +323,12 @@ jobs:
     - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario
       if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"')
       run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS
+    - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT scenario
+      if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT"')
+      run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT
+    - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS scenario
+      if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS"')
+      run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS
     - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario
       if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"')
       run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS

manifests/cpp_httpd.yml

@@ -42,7 +42,8 @@ manifest:
   tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented)
   tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py: missing_feature
-  tests/ffe/test_exposures.py: missing_feature
+  tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py: missing_feature
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_metrics.py: missing_feature
   tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234)

manifests/cpp_nginx.yml

@@ -275,7 +275,8 @@ manifest:
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature
   tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py: missing_feature
-  tests/ffe/test_exposures.py: missing_feature
+  tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py: missing_feature
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_metrics.py: missing_feature
   tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234)

manifests/dotnet.yml

@@ -728,7 +728,10 @@ manifest:
   tests/ffe/test_dynamic_evaluation.py: v3.36.0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184)
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184)
-  tests/ffe/test_exposures.py: v3.36.0
+  tests/ffe/test_exposure_egress.py: v3.36.0
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py: v3.36.0
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_metrics.py: v3.44.0
   tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234)

manifests/golang.yml

@@ -1156,7 +1156,10 @@ manifest:
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance::test_unknown_operator_errors: bug (FFL-2182)
-  tests/ffe/test_exposures.py: v2.6.0-dev  # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0
+  tests/ffe/test_exposure_egress.py: v2.6.0-dev  # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py: v2.6.0-dev  # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0
   tests/ffe/test_flag_eval_evp.py: v2.10.0-dev
   tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation::test_ffe_evp_flagevaluation_degradation: flaky (FFL-2676)
   tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2784)

manifests/java.yml

@@ -3297,11 +3297,17 @@ manifest:
       weblog: [spring-boot]
     - declaration: bug (FFL-2184)
       excluded_weblog: [spring-boot]
-  tests/ffe/test_exposures.py:
+  tests/ffe/test_exposure_egress.py:
     - weblog_declaration:
         "*": irrelevant
         spring-boot: v1.56.0
-  tests/ffe/test_exposures.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1729)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py:
+    - weblog_declaration:
+        "*": irrelevant
+        spring-boot: v1.56.0
+  tests/ffe/test_exposures_datadog_agent.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1729)
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_metrics.py:
     - weblog_declaration:

manifests/nodejs.yml

@@ -1749,11 +1749,15 @@ manifest:
         "*": incomplete_test_app
         express4: *ref_5_77_0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance::test_unknown_operator_errors: bug (FFL-2182)
-  tests/ffe/test_exposures.py:
+  tests/ffe/test_exposure_egress.py:
     - weblog_declaration:
         "*": incomplete_test_app
         express4: *ref_5_77_0
-  tests/ffe/test_exposures.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1730)
+  tests/ffe/test_exposures_datadog_agent.py:
+    - weblog_declaration:
+        "*": incomplete_test_app
+        express4: *ref_5_77_0
+  tests/ffe/test_exposures_datadog_agent.py::Test_FFE_EXP_5_Missing_Targeting_Key: bug (FFL-1730)
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_metrics.py:
     - weblog_declaration:

manifests/php.yml

@@ -800,12 +800,19 @@ manifest:
         "*": v1.21.0-dev
         laravel11x: incomplete_test_app
         symfony7x: incomplete_test_app
-  tests/ffe/test_exposures.py:
+  tests/ffe/test_exposure_egress.py:
     - weblog_declaration:
         "*": v1.21.0-dev
         laravel11x: incomplete_test_app
         symfony7x: incomplete_test_app
-  tests/ffe/test_exposures.py::Test_FFE_Exposure_Events::test_ffe_multiple_remote_config_files: flaky (FFL-2676)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py:
+    - weblog_declaration:
+        "*": v1.21.0-dev
+        laravel11x: incomplete_test_app
+        symfony7x: incomplete_test_app
+  tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Events::test_ffe_multiple_remote_config_files: flaky (FFL-2676)
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: bug (FFL-2676)
   tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation: bug (FFL-2676)

manifests/python.yml

@@ -1382,7 +1382,10 @@ manifest:
   tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2699 tracks Python agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622)
-  tests/ffe/test_exposures.py: v4.2.0-dev
+  tests/ffe/test_exposure_egress.py: v4.2.0-dev
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py: v4.2.0-dev
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_metrics.py: v4.7.0
   tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980)

manifests/ruby.yml

@@ -2002,7 +2002,13 @@ manifest:
     - weblog_declaration:
         "*": irrelevant
         rails72: v2.23.0-dev
-  tests/ffe/test_exposures.py:
+  tests/ffe/test_exposure_egress.py:
+    - weblog_declaration:
+        "*": irrelevant
+        rails72: v2.23.0-dev
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented)
+  tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py:
     - weblog_declaration:
         "*": irrelevant
         rails72: v2.23.0-dev

manifests/rust.yml

@@ -22,7 +22,8 @@ manifest:
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature
   tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py: missing_feature
-  tests/ffe/test_exposures.py: missing_feature
+  tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented)
+  tests/ffe/test_exposures_datadog_agent.py: missing_feature
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
   tests/ffe/test_flag_eval_metrics.py: missing_feature
   tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234)

mirror_images.lock.yaml

@@ -26,6 +26,10 @@ images:
     digest: sha256:0199e251729b25276f97049cbcd39dd1997e98413a07b10a3c7e9e755d505a42
     target: registry.ddbuild.io/system-tests/mirror/datadog/dd-appsec-php-ci:php-8.2-release
     tag: php-8.2-release
+  datadog/serverless-init:1.9.13:
+    digest: sha256:d187a970ce66d6f1ec4ba26e7c38783ad2e1c6e423fc1b1d39e81ba6332317cf
+    target: registry.ddbuild.io/system-tests/mirror/datadog/serverless-init:1.9.13
+    tag: 1.9.13
   datadog/system-tests:apache-mod-7.0-zts.base-v1:
     digest: sha256:bafc072734341cafb86f45673df2de4851a04b951d0717c88f7958cac1815b20
     target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.0-zts.base-v1

mirror_images.yaml

@@ -13,6 +13,7 @@
 - "datadog/agent:latest"
 - "datadog/dd-appsec-php-ci:php-8.0-release"
 - "datadog/dd-appsec-php-ci:php-8.2-release"
+- "datadog/serverless-init:1.9.13"
 - "datadog/system-tests:apache-mod-7.0-zts.base-v1"
 - "datadog/system-tests:apache-mod-7.0.base-v1"
 - "datadog/system-tests:apache-mod-7.1-zts.base-v1"

tests/ffe/README.md

@@ -7,7 +7,8 @@ This directory contains system tests for the Feature Flags & Experimentation (FF
 | File | Description |
 |------|-------------|
 | `test_dynamic_evaluation.py` | Dynamic flag evaluation via Remote Config |
-| `test_exposures.py` | Flag exposure tracking and reporting |
+| `test_exposure_egress.py` | Shared exposure cache and payload contract across Agent, sidecar, and direct egress |
+| `test_exposures_datadog_agent.py` | Flag exposure tracking and reporting through the Datadog Agent |
 | `test_flag_eval_metrics.py` | Evaluation metrics (OTel counter) |
 | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds |
 

tests/ffe/test_exposure_egress.py

@@ -0,0 +1,120 @@
+"""Test one exposure contract through every supported deployment topology."""
+
+from dataclasses import dataclass
+
+from tests.ffe.utils.exposures import assert_exposure_side_effects_contract, exposure_events_from_data
+from tests.ffe.utils.fixtures import make_ufc_fixture
+from utils import context, features, interfaces, remote_config as rc, scenarios, weblog
+from utils._context._scenarios.endtoend import FeatureFlaggingAgentlessEndToEndScenario
+from utils._context.component_version import Version
+from utils.interfaces._core import ProxyBasedInterfaceValidator
+from utils.mocked_backend.ffe import EXPECTED_API_KEY
+
+RC_PATH = "datadog/2/FFE_FLAGS"
+
+
+@dataclass(frozen=True)
+class ExposureEgress:
+    interface: ProxyBasedInterfaceValidator
+    excluded_interfaces: tuple[ProxyBasedInterfaceValidator, ...] = ()
+    expected_api_key: str | None = None
+
+
+def exposure_egress() -> ExposureEgress:
+    """Return the capture interface and route-only expectations for this topology."""
+    scenario = context.scenario
+    if not isinstance(scenario, FeatureFlaggingAgentlessEndToEndScenario):
+        assert scenario.name == "FEATURE_FLAGGING_AND_EXPERIMENTATION"
+        return ExposureEgress(interfaces.agent)
+
+    if scenario.exposure_egress == "sidecar":
+        assert scenario.components["serverless-init"] == Version("1.9.13")
+        expected_api_key = scenario.serverless_init_container.environment["DD_API_KEY"]
+        assert expected_api_key is not None
+        return ExposureEgress(
+            interfaces.datadog_sidecar,
+            (interfaces.datadog_direct,),
+            expected_api_key,
+        )
+
+    assert scenario.exposure_egress == "direct"
+    assert "serverless-init" not in scenario.components
+    return ExposureEgress(
+        interfaces.datadog_direct,
+        (interfaces.datadog_sidecar,),
+        EXPECTED_API_KEY,
+    )
+
+
+class ExposureEgressContract:
+    """One exposure contract inherited by each supported topology adapter."""
+
+    flag_key = "empty-targeting-key-flag"
+    targeting_key = "exposure-egress-user"
+
+    def setup_exposure_egress(self) -> None:
+        if not isinstance(context.scenario, FeatureFlaggingAgentlessEndToEndScenario):
+            rc.tracer_rc_state.reset().set_config(
+                f"{RC_PATH}/exposure-egress/config",
+                make_ufc_fixture(self.flag_key),
+            ).apply()
+
+        self.responses = [
+            weblog.post(
+                "/ffe",
+                json={
+                    "flag": self.flag_key,
+                    "variationType": "STRING",
+                    "defaultValue": "default",
+                    "targetingKey": self.targeting_key,
+                    "attributes": {},
+                },
+            )
+            for _ in range(5)
+        ]
+
+    def test_exposure_egress(self) -> None:
+        egress = exposure_egress()
+        matching_requests = assert_exposure_side_effects_contract(
+            egress.interface,
+            self.responses,
+            flag_key=self.flag_key,
+            targeting_key=self.targeting_key,
+            expected_value="on-value",
+            expected_variant="on",
+        )
+        assert len(matching_requests) == 1
+
+        if egress.expected_api_key is None:
+            return
+
+        request = matching_requests[0]
+        assert request["host"] == "event-platform-intake.datad0g.com"
+        assert request["response"]["status_code"] == 202
+
+        headers = {name.lower(): value for name, value in request["request"]["headers"]}
+        assert headers["dd-api-key"] in {egress.expected_api_key, "--redacted--"}
+
+        for excluded_interface in egress.excluded_interfaces:
+            assert not any(
+                exposure_events_from_data(data, {self.flag_key}, self.targeting_key)
+                for data in excluded_interface.get_data()
+            )
+
+
+@scenarios.feature_flagging_and_experimentation
+@features.feature_flags_exposures
+class Test_FFE_Exposure_Egress_Datadog_Agent(ExposureEgressContract):
+    pass
+
+
+@scenarios.feature_flagging_and_experimentation_agentless_direct
+@features.feature_flags_exposures
+class Test_FFE_Exposure_Egress_Agentless_Direct(ExposureEgressContract):
+    pass
+
+
+@scenarios.feature_flagging_and_experimentation_agentless_serverless
+@features.feature_flags_exposures
+class Test_FFE_Exposure_Egress_Agentless_Sidecar(ExposureEgressContract):
+    pass