Skip to content

Commit ef6d87a

Browse files
authored
fix(metrics): dedup metrics v3 tags (#1817)
## Summary <!-- Please provide a brief summary about what this PR does. This should help the reviewers give feedback faster and with higher quality. --> Fixes logical mismatches found when testing the v3 pipeline in validation mode where v2 tag hashes differed from v3 the v3 tag hash. ## Change Type - [x] Bug fix - [ ] New feature - [ ] Non-functional (chore, refactoring, docs) - [ ] Performance ## How did you test this PR? <!-- Please how you tested these changes here --> ## References <!-- Please list any issues closed by this PR. --> <!-- - Closes: <issue link> --> <!-- Any other issues or PRs relevant to this PR? Feel free to list them here. -->
1 parent 53976e1 commit ef6d87a

1 file changed

Lines changed: 14 additions & 7 deletions

File tree

  • lib/saluki-components/src/encoders/datadog/metrics

lib/saluki-components/src/encoders/datadog/metrics/mod.rs

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use protobuf::{rt::WireType, CodedOutputStream};
88
use resource_accounting::{MemoryBounds, MemoryBoundsBuilder};
99
use saluki_common::{
1010
buf::{ChunkedBytesBuffer, FrozenChunkedBytesBuffer},
11+
iter::ReusableDeduplicator,
1112
task::HandleExt as _,
1213
};
1314
use saluki_config::GenericConfiguration;
@@ -1252,9 +1253,10 @@ async fn flush_payload(
12521253
// Encodes a batch of metrics to V3 columnar format.
12531254
fn encode_v3_metrics_batch(metrics: &[Metric], additional_tags: &SharedTagSet) -> Result<Vec<u8>, GenericError> {
12541255
let mut writer = v3::V3Writer::new();
1256+
let mut tags_deduplicator = ReusableDeduplicator::new();
12551257

12561258
for metric in metrics {
1257-
write_metric_to_v3(&mut writer, metric, additional_tags);
1259+
write_metric_to_v3(&mut writer, metric, additional_tags, &mut tags_deduplicator);
12581260
}
12591261

12601262
let mut output = Vec::new();
@@ -1266,7 +1268,10 @@ fn encode_v3_metrics_batch(metrics: &[Metric], additional_tags: &SharedTagSet) -
12661268
}
12671269

12681270
/// Writes a single metric to the V3 writer.
1269-
fn write_metric_to_v3(writer: &mut v3::V3Writer, metric: &Metric, additional_tags: &SharedTagSet) {
1271+
fn write_metric_to_v3(
1272+
writer: &mut v3::V3Writer, metric: &Metric, additional_tags: &SharedTagSet,
1273+
tags_deduplicator: &mut ReusableDeduplicator<Tag>,
1274+
) {
12701275
let metric_type = match metric.values() {
12711276
MetricValues::Counter(..) => v3::V3MetricType::Count,
12721277
MetricValues::Rate(..) => v3::V3MetricType::Rate,
@@ -1278,12 +1283,14 @@ fn write_metric_to_v3(writer: &mut v3::V3Writer, metric: &Metric, additional_tag
12781283
let mut builder = writer.write(metric_type, metric.context().name());
12791284

12801285
// Tags - chain instrumented + additional + origin tags
1281-
let all_tags = metric
1286+
let chained_tags = metric
12821287
.context()
12831288
.tags()
12841289
.into_iter()
12851290
.chain(additional_tags)
1286-
.chain(metric.context().origin_tags())
1291+
.chain(metric.context().origin_tags());
1292+
let all_tags = tags_deduplicator
1293+
.deduplicated(chained_tags)
12871294
.filter(|t| is_sketch || !is_v3_series_resource_tag(t) && !is_v3_series_device_tag(t))
12881295
.map(|t| t.as_str());
12891296
builder.set_tags(all_tags);
@@ -1295,13 +1302,13 @@ fn write_metric_to_v3(writer: &mut v3::V3Writer, metric: &Metric, additional_tag
12951302
}
12961303
if !is_sketch {
12971304
let mut device_resource = None;
1298-
for tag in metric
1305+
let chained_tags = metric
12991306
.context()
13001307
.origin_tags()
13011308
.into_iter()
13021309
.chain(metric.context().tags())
1303-
.chain(additional_tags)
1304-
{
1310+
.chain(additional_tags);
1311+
for tag in tags_deduplicator.deduplicated(chained_tags) {
13051312
if is_v3_series_device_tag(tag) {
13061313
device_resource = tag.value().filter(|device| !device.is_empty());
13071314
} else if is_v3_series_resource_tag(tag) {

0 commit comments

Comments
 (0)