Skip to content

Commit 6f35b22

Browse files
lukesteensenlukesteensen
committed
test(smp): benchmark metric tag value allowlists
1 parent 2b7632e commit 6f35b22

51 files changed

Lines changed: 517696 additions & 9 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎test/smp/README.md‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ An experiment picks how much of it to use through `target.command`:
1919

2020
- Most experiments run ADP by itself, in standalone mode, by pointing the command straight at the
2121
ADP binary. The Core Agent never starts.
22-
- The tag filtering experiments run the image's own entrypoint, so the Core Agent comes up and
23-
supervises ADP. They need it: a `metric_tag_filterlist` is Core Agent configuration, and reaches
24-
ADP over the config stream.
22+
- The whole-tag filtering experiments run the image's own entrypoint, so the Core Agent comes up
23+
and supervises ADP. They need it: a `metric_tag_filterlist` is Core Agent configuration, and
24+
reaches ADP over the config stream.
25+
- The tag value allow-list experiments run ADP standalone. `metric_tag_value_allowlist` is an
26+
ADP-only key, so loading it directly avoids measuring the Core Agent in those experiments.
2527

2628
Profiling is SMP's to arrange: the SMP experiment runner loads `ddprof` into the target with `LD_PRELOAD` on the
2729
replicas named by `ddprof_replicas` in `config.yaml`.
@@ -291,12 +293,12 @@ pad differently.
291293

292294
### Tag filtering, as an example
293295

294-
The tag filtering experiments use all of this. `variables:` fixes the corpora and the filterlist
295-
size, the lading generator draws from patterns built with `lading_range`, and
296-
`shared/tagfilter-datadog.yaml.j2` loops over `lading_names` to build the filterlist the Core
297-
Agent streams to ADP. One set of numbers drives both, so the filterlist can't end up naming
298-
metrics the generator never sends — which is a silent failure, since the experiment still runs
299-
green while measuring nothing.
296+
The tag filtering experiments use all of this. `variables:` fixes the corpora and filter sizes,
297+
and the lading generator draws from patterns built with `lading_range`.
298+
`shared/tagfilter-datadog.yaml.j2` loops over `lading_names` to build the whole-tag filterlist the
299+
Core Agent streams to ADP. `shared/tagvaluefilter.yaml.j2` uses the same helpers to build the
300+
ADP-only value allow-list. One set of numbers drives the configuration and traffic in each case,
301+
so a filter can't silently name metrics or values that the generator never sends.
300302

301303
> [!NOTE]
302304
> The rendered `datadog.yaml` files are large — the 10,000-entry variants are ~12 MiB each. That

‎test/smp/regression/adp/experiments.yaml‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,93 @@ templates:
253253
tag_values:
254254
- "value{{ lading_range(0, 10) }}"
255255

256+
# Base template for ADP-only metric tag value allow-list experiments.
257+
#
258+
# Unlike `metric_tag_filterlist`, this configuration does not come from the Core Agent, so these
259+
# experiments run ADP standalone to avoid measuring an unrelated process. Variables drive both
260+
# the generated allow-list and lading's metric/tag corpus.
261+
tagvaluefilter_base:
262+
variables:
263+
metric_prefix: tagvalue.metric.
264+
traffic_metric_first: 10000
265+
traffic_metrics: 500
266+
rule_metric_first: 10000
267+
rule_metrics: 0
268+
rule_mode: none
269+
tag_prefix: tagvalue_tag_
270+
tag_pool: 54
271+
value_prefix: value
272+
value_pool: 12
273+
allowed_values: 10
274+
mismatch_action: remove
275+
replacement: other
276+
277+
target:
278+
command: /opt/datadog-agent/embedded/bin/agent-data-plane --config /etc/agent-data-plane/tagvaluefilter.yaml run
279+
files:
280+
empty.yaml: null
281+
tagvaluefilter.yaml:
282+
source: shared/tagvaluefilter.yaml.j2
283+
environment:
284+
DD_DATA_PLANE_DOGSTATSD_ENABLED: "true"
285+
DD_DOGSTATSD_PORT: "0"
286+
DD_DOGSTATSD_SOCKET: /tmp/adp-dogstatsd-dgram.sock
287+
DD_AGGREGATE_CONTEXT_LIMIT: "80000"
288+
DD_DOGSTATSD_STRING_INTERNER_SIZE_BYTES: "32MiB"
289+
290+
# Value allow-list configuration with no traffic, used to measure compiled configuration memory.
291+
tagvaluefilter_idle:
292+
extends: tagvaluefilter_base
293+
lading:
294+
generator: []
295+
296+
# Value allow-list processing under a fixed 100 MiB/s DogStatsD workload. The corpus contains 500
297+
# metric names, 54 tags per message, and 12 possible values for each tag.
298+
tagvaluefilter_dsd_uds_100mb:
299+
extends: tagvaluefilter_base
300+
lading:
301+
generator:
302+
- unix_datagram:
303+
seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131]
304+
path: /tmp/adp-dogstatsd-dgram.sock
305+
block_cache_method: Fixed
306+
bytes_per_second: "100 MiB"
307+
maximum_prebuild_cache_size_bytes: "500 Mb"
308+
variant:
309+
dogstatsd:
310+
contexts:
311+
inclusive:
312+
min: 78000
313+
max: 78000
314+
tags_per_msg:
315+
inclusive:
316+
min: 54
317+
max: 54
318+
multivalue_count:
319+
inclusive:
320+
min: 2
321+
max: 32
322+
multivalue_pack_probability: 0.08
323+
kind_weights:
324+
metric: 100
325+
event: 0
326+
service_check: 0
327+
# Value allow-lists apply to counters and sketch-backed metrics. Exclude the metric
328+
# types that intentionally bypass the transform so throughput exercises the feature.
329+
metric_weights:
330+
count: 3
331+
gauge: 0
332+
timer: 0
333+
distribution: 1
334+
set: 0
335+
histogram: 0
336+
metric_names:
337+
- "{{ exp_vars.metric_prefix }}{{ lading_range(exp_vars.traffic_metric_first, exp_vars.traffic_metrics) }}"
338+
tag_names:
339+
- "{{ exp_vars.tag_prefix }}{{ lading_range(0, exp_vars.tag_pool) }}"
340+
tag_values:
341+
- "{{ exp_vars.value_prefix }}{{ lading_range(0, exp_vars.value_pool) }}"
342+
256343
# Base template for OTLP experiments
257344
otlp_base:
258345
target:
@@ -697,3 +784,103 @@ experiments:
697784
optimization_goal: cpu
698785
variables:
699786
filterlist_metrics: 10000
787+
788+
# ============================================================================
789+
# Metric tag value allow-list experiments
790+
# ============================================================================
791+
792+
# Idle variants separate the memory used by metric-prefix rules from the memory used by the
793+
# allowed-value sets.
794+
- name: tagvaluefilter_0rules_idle
795+
extends: tagvaluefilter_idle
796+
optimization_goal: memory
797+
798+
- name: tagvaluefilter_1rule_10values_idle
799+
extends: tagvaluefilter_idle
800+
optimization_goal: memory
801+
variables:
802+
rule_mode: broad
803+
rule_metrics: 1
804+
allowed_values: 10
805+
806+
- name: tagvaluefilter_500rules_10values_idle
807+
extends: tagvaluefilter_idle
808+
optimization_goal: memory
809+
variables:
810+
rule_mode: per_metric
811+
rule_metrics: 500
812+
allowed_values: 10
813+
814+
- name: tagvaluefilter_10000rules_10values_idle
815+
extends: tagvaluefilter_idle
816+
optimization_goal: memory
817+
variables:
818+
rule_mode: per_metric
819+
rule_metrics: 10000
820+
allowed_values: 10
821+
822+
- name: tagvaluefilter_1rule_10000values_idle
823+
extends: tagvaluefilter_idle
824+
optimization_goal: memory
825+
variables:
826+
rule_mode: broad
827+
rule_metrics: 1
828+
value_pool: 10000
829+
allowed_values: 10000
830+
831+
# Traffic variants isolate the no-rule baseline, allowed-value lookup, context mutation, prefix
832+
# table scaling, and the full-table miss path.
833+
- name: tagvaluefilter_0rules_100mb
834+
extends: tagvaluefilter_dsd_uds_100mb
835+
optimization_goal: cpu
836+
837+
- name: tagvaluefilter_1rule_allow_100mb
838+
extends: tagvaluefilter_dsd_uds_100mb
839+
optimization_goal: cpu
840+
variables:
841+
rule_mode: broad
842+
rule_metrics: 1
843+
allowed_values: 12
844+
845+
- name: tagvaluefilter_1rule_remove25_100mb
846+
extends: tagvaluefilter_dsd_uds_100mb
847+
optimization_goal: cpu
848+
variables:
849+
rule_mode: broad
850+
rule_metrics: 1
851+
allowed_values: 9
852+
mismatch_action: remove
853+
854+
- name: tagvaluefilter_1rule_replace25_100mb
855+
extends: tagvaluefilter_dsd_uds_100mb
856+
optimization_goal: cpu
857+
variables:
858+
rule_mode: broad
859+
rule_metrics: 1
860+
allowed_values: 9
861+
mismatch_action: replace
862+
863+
- name: tagvaluefilter_500rules_allow_100mb
864+
extends: tagvaluefilter_dsd_uds_100mb
865+
optimization_goal: cpu
866+
variables:
867+
rule_mode: per_metric
868+
rule_metrics: 500
869+
allowed_values: 12
870+
871+
- name: tagvaluefilter_10000rules_allow_100mb
872+
extends: tagvaluefilter_dsd_uds_100mb
873+
optimization_goal: cpu
874+
variables:
875+
rule_mode: per_metric
876+
rule_metrics: 10000
877+
allowed_values: 12
878+
879+
- name: tagvaluefilter_10000rules_miss_100mb
880+
extends: tagvaluefilter_dsd_uds_100mb
881+
optimization_goal: cpu
882+
variables:
883+
rule_mode: per_metric
884+
rule_metric_first: 20000
885+
rule_metrics: 10000
886+
allowed_values: 12

‎test/smp/regression/adp/full/cases/tagvaluefilter_0rules_100mb/agent-data-plane/cert.pem‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDwTCCAqmgAwIBAgIUGNrRiJ81arCE1jTvLRoxxuLVoHAwDQYJKoZIhvcNAQEL
3+
BQAwgYgxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOWTEWMBQGA1UEBwwNTmV3IFlv
4+
cmsgQ2l0eTEWMBQGA1UECgwNRGF0YWRvZywgSW5jLjEoMCYGA1UECwwfU2VsZi1T
5+
aWduZWQgVGVzdGluZyBDZXJ0aWZpY2F0ZTESMBAGA1UEAwwJbG9jYWxob3N0MB4X
6+
DTI1MTExMzE4NDAxMFoXDTM1MTExMTE4NDAxMFowgYgxCzAJBgNVBAYTAlVTMQsw
7+
CQYDVQQIDAJOWTEWMBQGA1UEBwwNTmV3IFlvcmsgQ2l0eTEWMBQGA1UECgwNRGF0
8+
YWRvZywgSW5jLjEoMCYGA1UECwwfU2VsZi1TaWduZWQgVGVzdGluZyBDZXJ0aWZp
9+
Y2F0ZTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
10+
MIIBCgKCAQEAs9oaPxXZ5t2725jEWJ+kLMbH6tFzKIgGEG2wpzJik1AumsSeax40
11+
JvA6W20Gyb73KkbCWM7Hm83++5QesDllpIvK0QNlDsz9VzQtiMCRZQ5Dfuw6stl+
12+
KQaCFvaEXBwvz/kfrIBLDeww9H9VL3YP2JnHIcA4Y5bvdIrSr3q59n7nd9exBtjS
13+
IZxLHxf44/yQMIUcmPESnLNGYLxqIYuHvre+t1CrWduzHwKtsSIP1qy3U2YCRQW6
14+
mxDaZ+aI0BY8vYNX1w4XEeW8NO1HEu25DPXPWGbcSgHVk2VnQKtMIoVjuCGZGbBz
15+
/TFJo69KoW/3CH3WHpT2JVKPV5itxkP68QIDAQABoyEwHzAdBgNVHQ4EFgQUFHdk
16+
73oIdZ7CkilNi96SveiH/OcwDQYJKoZIhvcNAQELBQADggEBAEMd7CiZkIqtE+O4
17+
Nj19xiTflKyw0EBklwlq1NrFb6VpGN3vPcu5X2CNH9p+c+zPYgDSTf+IFBYXO/zE
18+
AzNYjFeUQf6Hsh0fTLqaiaueiAZw1o6QJp7/xFdwFcUFJM72lIlgndkoffEJCLOR
19+
GcjAYD2Qv/oCjQ2B5xeHT9Sk4t85dhiK0aLbeSi+9yqtHis15OW8vqVFJXh1SxMV
20+
zSKN9C4Yw0JKMNqlmePjQIc3d28tm4sUOzD2+qF+mSTSZqvAEuO3loQJPexfsDsF
21+
n929iEzFbOshRpjKp/mZ1jzZewDOIg5Zek1UmbqIHf9MHfpdDYA0zcEnzTQqGFtL
22+
BxX9Nnk=
23+
-----END CERTIFICATE-----
24+
-----BEGIN PRIVATE KEY-----
25+
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCz2ho/Fdnm3bvb
26+
mMRYn6Qsxsfq0XMoiAYQbbCnMmKTUC6axJ5rHjQm8DpbbQbJvvcqRsJYzsebzf77
27+
lB6wOWWki8rRA2UOzP1XNC2IwJFlDkN+7Dqy2X4pBoIW9oRcHC/P+R+sgEsN7DD0
28+
f1Uvdg/YmcchwDhjlu90itKvern2fud317EG2NIhnEsfF/jj/JAwhRyY8RKcs0Zg
29+
vGohi4e+t763UKtZ27MfAq2xIg/WrLdTZgJFBbqbENpn5ojQFjy9g1fXDhcR5bw0
30+
7UcS7bkM9c9YZtxKAdWTZWdAq0wihWO4IZkZsHP9MUmjr0qhb/cIfdYelPYlUo9X
31+
mK3GQ/rxAgMBAAECggEAFPVEe7d7JfkOzB84Oi+YPROI9mcj6UQ4mK9/l2w6qnn8
32+
hFKHN2pUn3j9A+xnjxjuyoFmYlzuS8ysevqevOBjZNJZdxPTMe9XUlMlPztZdhI8
33+
cUzr5i2MExHMFQrzD5zFQZIBS+PDW0L7zXINJjO20wHQf7FboNdU3hrTRFmj1AJx
34+
ExWk+vnY2YrzQLewsbPtK1FX6wqKhYuOSUOjpNJmf+2+aGfR+Bxx2NBF4marJoj3
35+
migN1pThUQAoPXX5qRoX3VpEEZEOMycuSXp+raqcBqSFfv8gz/iM6tLfctRgqe0w
36+
266oxipeLLXLmffYvnXrYXwK4Jl5M6Zd9TVjf6GDBQKBgQD+FSW+YEp6l4Zz5Rwo
37+
Xrb104ILMHsd5V59KyNcd2c3mxMEhodN3sQDEd4OQRmMGIfcivb9Sn49GRBMTu29
38+
f4ax6+F0Su66MOikgdhOYyNWI3TfMbsY/zCyXiGU9MiWVNDzx1nQj8sG7h36dK1l
39+
Yxv6xx/a857kbZYnB53DCQ3dmwKBgQC1NY09e3Qq+hXyuZEJSsHE5LAYd9TeWs+D
40+
FuUmL8FQkGCihQz/fX3+yJar4oXDoIHnhEwGEEKkOoT9EJ4uzW9L0WSy6fopjRs2
41+
lXTnQBbB7414ZYkTM/oanTh7dBVG307oyb6d5+Jl0NPxs5Fomh6xOMWD8hB0+CTw
42+
bAaO/c5YYwKBgATdlM5ze5mjYzC+924SekB0322lbQYiiU+uTswLgU+ASbnxdY/Z
43+
Lzm70tvFBV84bQmdI6OwFIDJBRXhAQ567bJkiPm4IaAxJZNY5TKDFX8lyKwpgKK2
44+
6FDSGqSGl3zBfQreC2tCBapJTwunxlZFsph3zbVcqvNG4fQ3Yh8FAl33AoGADR9l
45+
rRAlp38Y280Ibc3WHnYZMoxrA/c7k9iym4NV0onCFcLg4BesaikIkEYFPdd/0M5J
46+
2x6OVOpP+yua6PTDnI/7ZOGA1kV7tQY5ww1nGIBKlG9178gR0p+UGYychddiFYWW
47+
okTKpmjrEFPaseKHWnosA5QiEPZvZmHMT8qdiNUCgYAweUc7+5slfxKYVLUo34p9
48+
BiED632dV2YA7CYf+iEjXoFjCOH+ZqzI7OGfODpSMpjim3Fxt++UWbJscbNmtCZr
49+
UeNDsJxiC+eksGWqpu0QoKdNPYRBB2QQFcvTT0AVt6PTzKaFVBvF/dnLuODjXp9Y
50+
R5Ivn5V7QPeMY56Uzr2fJQ==
51+
-----END PRIVATE KEY-----

‎test/smp/regression/adp/full/cases/tagvaluefilter_0rules_100mb/agent-data-plane/tagvaluefilter.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
metric_tag_value_allowlist: []

‎test/smp/regression/adp/full/cases/tagvaluefilter_0rules_100mb/experiment.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
optimization_goal: cpu
2+
erratic: false
3+
target:
4+
name: agent-data-plane
5+
command: /opt/datadog-agent/embedded/bin/agent-data-plane --config /etc/agent-data-plane/tagvaluefilter.yaml run
6+
cpu_allotment: 4
7+
memory_allotment: 2GiB
8+
environment:
9+
DD_HOSTNAME: smp-regression
10+
DD_API_KEY: foo00000001
11+
DD_DD_URL: http://127.0.0.1:9091
12+
DD_SERIALIZER_EXPERIMENTAL_USE_V3_API_SERIES_ENDPOINTS: http://127.0.0.1:9091
13+
DD_IPC_CERT_FILE_PATH: /etc/agent-data-plane/cert.pem
14+
DD_LOG_FORMAT_JSON: "true"
15+
DD_DATA_PLANE_STANDALONE_MODE: "true"
16+
DD_DATA_PLANE_DOGSTATSD_ENABLED: "true"
17+
DD_DOGSTATSD_PORT: "0"
18+
DD_DOGSTATSD_SOCKET: /tmp/adp-dogstatsd-dgram.sock
19+
DD_AGGREGATE_CONTEXT_LIMIT: "80000"
20+
DD_DOGSTATSD_STRING_INTERNER_SIZE_BYTES: 32MiB
21+
profiling_environment:
22+
DD_SERVICE: agent-data-plane
23+
DD_PROFILING_NATIVE_PRESET: cpu_live_heap
24+
DD_PROFILING_INLINED_FUNCTIONS: "true"
25+
report_links:
26+
- text: (metrics)
27+
link: https://app.datadoghq.com/dashboard/4br-nxz-khi?fromUser=true&refresh_mode=paused&tpl_var_adp-run-id%5B0%5D={{ job_id
28+
}}&tpl_var_experiment%5B0%5D={{ experiment }}&view=spans&from_ts={{ start_time_ms }}&to_ts={{ end_time_ms }}&live=false
29+
- text: (profiles)
30+
link: https://app.datadoghq.com/profiling/explorer?query=env%3Asingle-machine-performance%20service%3Aagent-data-plane%20job_id%3A{{
31+
job_id }}%20experiment%3A{{ experiment }}&agg_m=count&agg_m_source=base&agg_t=count&fromUser=false&viz=stream&start={{
32+
filter_start }}&end={{ filter_end }}&paused=true
33+
- text: (logs)
34+
link: https://app.datadoghq.com/logs?query=experiment%3A{{ experiment }}%20run_id%3A{{ job_id }}&agg_m=count&agg_m_source=base&agg_q=%40span.url&agg_q_source=base&agg_t=count&fromUser=true&index=single-machine-performance-target-logs&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=time%2Cdesc&top_n=100&top_o=top&viz=stream&x_missing=true&from_ts={{
35+
filter_start }}&to_ts={{ filter_end }}&live=false

‎test/smp/regression/adp/full/cases/tagvaluefilter_0rules_100mb/lading/lading.yaml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
blackhole:
2+
- http:
3+
binding_addr: 127.0.0.1:9091
4+
target_metrics:
5+
- prometheus:
6+
uri: http://127.0.0.1:5100/metrics
7+
generator:
8+
- unix_datagram:
9+
seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107,
10+
109, 113, 127, 131]
11+
path: /tmp/adp-dogstatsd-dgram.sock
12+
block_cache_method: Fixed
13+
bytes_per_second: 100 MiB
14+
maximum_prebuild_cache_size_bytes: 500 Mb
15+
variant:
16+
dogstatsd:
17+
contexts:
18+
inclusive:
19+
min: 78000
20+
max: 78000
21+
tags_per_msg:
22+
inclusive:
23+
min: 54
24+
max: 54
25+
multivalue_count:
26+
inclusive:
27+
min: 2
28+
max: 32
29+
multivalue_pack_probability: 0.08
30+
kind_weights:
31+
metric: 100
32+
event: 0
33+
service_check: 0
34+
metric_weights:
35+
count: 3
36+
gauge: 0
37+
timer: 0
38+
distribution: 1
39+
set: 0
40+
histogram: 0
41+
metric_names:
42+
- tagvalue.metric.{{10000-10499}}
43+
tag_names:
44+
- tagvalue_tag_{{0-53}}
45+
tag_values:
46+
- value{{0-11}}

0 commit comments

Comments
 (0)