Skip to content

Commit f152503

Browse files
committed
chore(config): migrate DogStatsD source and mapper to typed config
Build the DogStatsD source and metric mapper from the resolved `SalukiConfiguration` model (`domains.dogstatsd`) instead of the raw `GenericConfiguration` map. - Source: `from_configuration` takes `&domains::dogstatsd::Domain` plus a caller-derived default capture path (from the unmodeled `run_path`), and strips all source serde from the component struct. - Mapper: `from_configuration` takes `&domains::dogstatsd::Mapper`; the interner byte budget parses as `ByteSize` in `SalukiOnly`. - Hoist the Saluki-only DogStatsD listener/context/mapper defaults into hand-written model `Default` impls; retire the component smoke tests. Progresses #1788
1 parent 166c295 commit f152503

10 files changed

Lines changed: 509 additions & 616 deletions

File tree

bin/agent-data-plane/src/cli/run.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use saluki_components::{
2828
},
2929
forwarders::{ClusterAgentForwarderConfiguration, DatadogForwarderConfiguration, OtlpForwarderConfiguration},
3030
relays::otlp::OtlpRelayConfiguration,
31-
sources::{ChecksIPCConfiguration, DogStatsDConfiguration, OtlpConfiguration},
31+
sources::{ChecksIPCConfiguration, DogStatsDConfiguration, OtlpConfiguration, DOGSTATSD_CAPTURE_DIR},
3232
transforms::{
3333
AggregateConfiguration, ApmStatsTransformConfiguration, AutoscalingFailoverGatewayConfiguration,
3434
ChainedConfiguration, DogStatsDMapperConfiguration, HostEnrichmentConfiguration,
@@ -723,17 +723,24 @@ async fn add_dsd_pipeline_to_blueprint(
723723
// │ (destination) │ │ (Datadog Platform) │
724724
// └─────────────────────┘ └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘
725725

726-
let dsd_config = DogStatsDConfiguration::from_configuration(config)
726+
let saluki = config_system.config();
727+
// `run_path` is deliberately not modeled, so read it from the raw config to derive the default
728+
// DogStatsD capture directory (used only when `dogstatsd_capture_path` is unset).
729+
let default_dsd_capture_path = config
730+
.try_get_typed::<PathBuf>("run_path")
731+
.ok()
732+
.flatten()
733+
.map(|run_path| run_path.join(DOGSTATSD_CAPTURE_DIR));
734+
let dsd_config = DogStatsDConfiguration::from_configuration(&saluki.domains.dogstatsd, default_dsd_capture_path)
727735
.error_context("Failed to configure DogStatsD source.")?
728736
.with_workload_provider(env_provider.workload().clone())
729737
.with_capture_entity_resolver(env_provider.workload().clone());
730738
let dsd_prefix_filter_configuration = DogStatsDPrefixFilterConfiguration::from_configuration(config)?;
731-
let dsd_mapper_config = DogStatsDMapperConfiguration::from_configuration(config)?;
739+
let dsd_mapper_config = DogStatsDMapperConfiguration::from_configuration(&saluki.domains.dogstatsd.mapper)?;
732740
let dsd_enrich_config =
733741
ChainedConfiguration::default().with_transform_builder("dogstatsd_mapper", dsd_mapper_config);
734742
let dsd_tag_filterlist_config = TagFilterlistConfiguration::from_configuration(config)
735743
.error_context("Failed to configure metric tag filterlist transform.")?;
736-
let saluki = config_system.config();
737744
let dsd_agg_config = AggregateConfiguration::from_configuration(
738745
&saluki.domains.dogstatsd.aggregation,
739746
&saluki.shared.metrics_encoding.histogram,

docs/agent-data-plane/configuration/dogstatsd.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,7 @@ The following settings are specific to ADP and have no equivalent in the core ag
431431
| `dogstatsd_buffer_count` | Baseline receive buffers | 128 |
432432
| `dogstatsd_cached_contexts_limit` | Max cached metric contexts | |
433433
| `dogstatsd_cached_tagsets_limit` | Max cached tagsets | |
434-
| `dogstatsd_mapper_string_interner_size` | Mapper string interner capacity | |
434+
| `dogstatsd_mapper_string_interner_size` | Mapper string interner byte budget | |
435435
| `dogstatsd_minimum_sample_rate` | Floor for metric sample rates | |
436436
| `dogstatsd_permissive_decoding` | Relaxes decoder strictness | true |
437437
| `dogstatsd_string_interner_size_bytes` | Explicit byte budget for context interner | |

lib/agent-data-plane-config-system/src/saluki_only.rs

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,10 @@ pub struct SalukiOnly {
134134
pub dogstatsd_allow_context_heap_allocs: Option<bool>,
135135
/// Floor for metric sample rates (`dogstatsd_minimum_sample_rate`).
136136
pub dogstatsd_minimum_sample_rate: Option<f64>,
137-
/// Mapper string interner entry count (`dogstatsd_mapper_string_interner_size`).
138-
pub dogstatsd_mapper_string_interner_size: Option<u64>,
137+
/// Mapper string interner byte budget (`dogstatsd_mapper_string_interner_size`), given as a
138+
/// bare integer number of bytes or a byte-size string such as `64KiB`. `ByteSize` accepts both
139+
/// forms.
140+
pub dogstatsd_mapper_string_interner_size: Option<ByteSize>,
139141

140142
// ── aggregation keys (all top-level) ──────────────────────────────────────
141143
/// Aggregation window size, in seconds (`aggregate_window_duration_seconds`).
@@ -454,7 +456,7 @@ impl SalukiOnly {
454456
dsd.contexts.minimum_sample_rate = v;
455457
}
456458
if let Some(v) = self.dogstatsd_mapper_string_interner_size {
457-
dsd.mapper.string_interner_size = v;
459+
dsd.mapper.string_interner_size = v.as_u64();
458460
}
459461
if let Some(v) = self.aggregate_window_duration_seconds {
460462
dsd.aggregation.window_duration_seconds = v;
@@ -691,6 +693,24 @@ mod tests {
691693
assert_eq!(config.domains.checks.ipc_endpoint.0, "localhost:5006");
692694
}
693695

696+
/// `dogstatsd_mapper_string_interner_size` is a byte size the source may express as a bare
697+
/// integer (bytes) or a suffixed string. Both must deserialize to the same byte count.
698+
#[test]
699+
fn mapper_string_interner_size_accepts_a_bare_integer_or_a_string() {
700+
for (value, expected) in [
701+
(json!({ "dogstatsd_mapper_string_interner_size": 2048 }), 2048),
702+
(
703+
json!({ "dogstatsd_mapper_string_interner_size": "64KiB" }),
704+
ByteSize::kib(64).as_u64(),
705+
),
706+
] {
707+
let saluki_only: SalukiOnly = serde_json::from_value(value).expect("interner size deserializes");
708+
let mut config = SalukiConfiguration::default();
709+
saluki_only.seed(&mut config);
710+
assert_eq!(config.domains.dogstatsd.mapper.string_interner_size, expected);
711+
}
712+
}
713+
694714
#[test]
695715
fn ottl_filter_config_rejects_unknown_fields_and_values() {
696716
for value in [
@@ -730,7 +750,17 @@ mod tests {
730750
let mut config = SalukiConfiguration::default();
731751
saluki_only.seed(&mut config);
732752

733-
let agg = &config.domains.dogstatsd.aggregation;
753+
let dsd = &config.domains.dogstatsd;
754+
assert_eq!(dsd.listeners.buffer_count, 128);
755+
assert_eq!(dsd.listeners.buffer_count_max, 256);
756+
assert!(dsd.listeners.permissive_decoding);
757+
assert_eq!(dsd.contexts.cached_contexts_limit, 500_000);
758+
assert_eq!(dsd.contexts.cached_tagsets_limit, 500_000);
759+
assert!(dsd.contexts.allow_context_heap_allocs);
760+
assert_eq!(dsd.contexts.minimum_sample_rate, 0.000000003845);
761+
assert_eq!(dsd.mapper.string_interner_size, 65_536);
762+
763+
let agg = &dsd.aggregation;
734764
assert_eq!(agg.window_duration_seconds, 10);
735765
assert_eq!(agg.context_limit, 1_000_000);
736766
assert_eq!(agg.flush_interval, Duration::from_secs(15));

lib/agent-data-plane-config/src/domains/dogstatsd.rs

Lines changed: 82 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ pub struct Domain {
4646
}
4747

4848
/// Source listeners and packet-decoding options.
49-
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
49+
#[derive(Clone, Debug, PartialEq, Serialize)]
5050
pub struct Listeners {
5151
/// UDP port DogStatsD listens on.
5252
pub port: u16,
@@ -114,6 +114,39 @@ pub struct Listeners {
114114
pub forward_port: u16,
115115
}
116116

117+
impl Default for Listeners {
118+
fn default() -> Self {
119+
Self {
120+
// Saluki-schema-only knobs whose defaults are not published by the Datadog Agent schema,
121+
// so they are seeded only when set; absent that, these values stand and must match what
122+
// the DogStatsD source expects.
123+
buffer_count: 128,
124+
buffer_count_max: 256,
125+
permissive_decoding: true,
126+
// Datadog-schema and unset-by-default knobs: the witness driver overwrites the former,
127+
// and the latter are genuinely zero/empty/None when unset.
128+
port: 0,
129+
tcp_port: 0,
130+
socket: None,
131+
stream_socket: None,
132+
pipe_name: None,
133+
windows_pipe_security_descriptor: String::new(),
134+
non_local_traffic: false,
135+
bind_host: None,
136+
so_rcvbuf: 0,
137+
buffer_size: 0,
138+
autoscale_udp_listeners: false,
139+
provider_kind: String::new(),
140+
capture_path: PathBuf::new(),
141+
capture_depth: 0,
142+
eol_required: Vec::new(),
143+
stream_log_too_big: false,
144+
forward_host: None,
145+
forward_port: 0,
146+
}
147+
}
148+
}
149+
117150
/// Origin detection and tag cardinality.
118151
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
119152
pub struct OriginDetection {
@@ -153,8 +186,15 @@ pub struct Telemetry {
153186
pub origin_breakdown: bool,
154187
}
155188

189+
/// The default floor for DogStatsD metric sample rates. (not in Datadog Agent config schema)
190+
///
191+
/// Roughly 260M samples; sample rates below this are clamped to bound tracked-sample memory growth.
192+
const fn default_minimum_sample_rate() -> f64 {
193+
0.000000003845
194+
}
195+
156196
/// Context cache sizing and sample-rate floor.
157-
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
197+
#[derive(Clone, Debug, PartialEq, Serialize)]
158198
pub struct Contexts {
159199
/// Maximum number of metric contexts held in the cache. (not in Datadog Agent config schema)
160200
pub cached_contexts_limit: usize,
@@ -178,6 +218,24 @@ pub struct Contexts {
178218
pub minimum_sample_rate: f64,
179219
}
180220

221+
impl Default for Contexts {
222+
fn default() -> Self {
223+
Self {
224+
// Saluki-schema-only knobs: not published by the Datadog Agent schema, so they are
225+
// seeded only when set; absent that, these values stand and must match what the
226+
// DogStatsD source expects.
227+
cached_contexts_limit: 500_000,
228+
cached_tagsets_limit: 500_000,
229+
allow_context_heap_allocs: true,
230+
minimum_sample_rate: default_minimum_sample_rate(),
231+
// Datadog-schema knob: overwritten by the witness driver.
232+
string_interner_size: 0,
233+
// Unset by default; `None` selects the entry-count-derived interner size.
234+
string_interner_size_bytes: None,
235+
}
236+
}
237+
}
238+
181239
/// Metric aggregation window and flush behavior.
182240
#[derive(Clone, Debug, PartialEq, Serialize)]
183241
pub struct Aggregation {
@@ -232,19 +290,39 @@ impl Default for Aggregation {
232290
}
233291
}
234292

293+
/// The default byte budget for the mapper's string interner. (not in Datadog Agent config schema)
294+
///
295+
/// 64 KiB, matching the DogStatsD mapper's prior default.
296+
const fn default_mapper_string_interner_size() -> u64 {
297+
65_536
298+
}
299+
235300
/// DogStatsD metric mapper.
236-
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
301+
#[derive(Clone, Debug, PartialEq, Serialize)]
237302
pub struct Mapper {
238303
/// Mapper profiles that rewrite matching metric names and tags.
239304
pub profiles: Vec<MapperProfile>,
240305

241306
/// Number of mapper match results cached.
242307
pub cache_size: usize,
243308

244-
/// Number of entries the mapper's string interner holds. (not in Datadog Agent config schema)
309+
/// Byte budget for the mapper's string interner. (not in Datadog Agent config schema)
245310
pub string_interner_size: u64,
246311
}
247312

313+
impl Default for Mapper {
314+
fn default() -> Self {
315+
Self {
316+
// Datadog-schema knobs: overwritten by the witness driver.
317+
profiles: Vec::new(),
318+
cache_size: 0,
319+
// Saluki-schema-only knob: seeded only when set; absent that, this value stands and
320+
// must be non-zero, as the mapper rejects a zero-sized interner.
321+
string_interner_size: default_mapper_string_interner_size(),
322+
}
323+
}
324+
}
325+
248326
/// One mapper profile: a name, a metric prefix, and the mappings under it.
249327
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
250328
pub struct MapperProfile {

lib/datadog-agent/config-overlay-model/src/saluki_keys.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -520,10 +520,10 @@ pub static SALUKI_KEYS: &[SalukiKey] = &[
520520
// ── dogstatsd_mapper.rs ──────────────────────────────────────────────────
521521
SalukiKey {
522522
yaml_path: "dogstatsd_mapper_string_interner_size",
523-
description: "Mapper string interner capacity",
523+
description: "Mapper string interner byte budget",
524524
default: "",
525525
documentation: None,
526-
value_type: "ValueType::Integer",
526+
value_type: "ValueType::String",
527527
schema_default: None,
528528
env_vars: &[],
529529
env_var_override: None,

lib/datadog-agent/config-testing/src/config_registry/dogstatsd_mapper.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ static DOGSTATSD_MAPPER_STRING_INTERNER_SIZE_SCHEMA: SchemaEntry = SchemaEntry {
88
schema: Schema::Saluki,
99
yaml_path: "dogstatsd_mapper_string_interner_size",
1010
env_vars: &[],
11-
value_type: ValueType::Integer,
11+
value_type: ValueType::String,
1212
default: None,
1313
};
1414

0 commit comments

Comments
 (0)