fix(libdd-trace-utils): apply SpanLink flags masking when v0.5 json encoding (#2314)

# What does this PR do?

A tracer flags a `SpanLink` with a 32-bit flags value. Bit 31 is a sentinel: it marks that the tracer explicitly set the sampling decision, as opposed to leaving `flags` at its default of zero. The remaining bits carry the sampling decision itself (bit 0: kept or dropped).

# Motivation

v0.4 and v0.5 disagree on whether the wire value includes this sentinel.

- In v0.4's native msgpack encoding, dd-trace-py adds the sentinel bit before it writes flags. A kept link becomes `0x8000_0001`; a link the tracer explicitly dropped becomes `0x8000_0000`.
- v0.5 has no native span-link field, so it encodes links as a JSON array under `meta["_dd.span_links"]`. dd-trace-py's v0.5 JSON encoder never adds the sentinel. A kept link's flags in this JSON is plain 1; the sentinel bit never appears there.

libdatadog's v0.5 encoder builds this JSON from the same `SpanLink` struct that v0.4 uses, so flags may already carry the sentinel bit. Before this fix, the v0.5 serializer wrote that raw value straight into the JSON, so a kept link produced `"flags": 2147483649` instead of `1` — a value no v0.5 producer would ever emit, and one that downstream consumers checking against small integers would not recognize as "kept".

This PR masks bit 31 off the value the v0.5 serializer writes into flags, while it still decides whether to emit the flags key at all from the raw, unmasked value. Deciding presence from the masked value would break the case of a link the tracer explicitly dropped: raw `0x8000_0000` masks to `0`, indistinguishable from "flags never set" if presence were decided post-mask. With this fix, an explicitly dropped link still emits `"flags": 0`; a link that never set flags emits no flags key at all.

# Additional Notes

Anything else we should know when reviewing?

# How to test the change?

- Added `span_link_flags_sentinel_bit_masked_test`: covers the unset, kept, and explicitly-dropped states of the sentinel bit.
- `cargo test -p libdd-trace-utils`, `cargo fmt --check`, and `cargo clippy -D warnings` pass.


Co-authored-by: bob.weinand <bob.weinand@datadoghq.com>

Author: brettlangdon

libdd-trace-utils/src/agentless_encoder/mod.rs

@@ -27,7 +27,7 @@
 //! TODO: span normalization (service/name/resource/type truncation + defaults)
 
 use crate::span::v04::{AttributeAnyValue, AttributeArrayValue, Span, SpanEvent, SpanLink};
-use crate::span::TraceData;
+use crate::span::{TraceData, SPAN_LINK_FLAGS_SET_SENTINEL};
 use crate::tracer_metadata::TracerMetadata;
 use serde::{
     ser::{SerializeMap, SerializeSeq},
@@ -341,9 +341,14 @@ fn encode_span_link<T: TraceData, S: Serializer>(
             }),
         )?;
     }
-    // `flags == 0` means no sampling decision is available; omit the field.
+    // When `flags` is 0, no sampling decision exists, so omit the field. Before emission,
+    // mask off the internal "explicitly set" sentinel (bit 31), because this JSON field uses
+    // the same `_dd.span_links` key that the v0.5 encoder produces and must match its output.
     if link.flags != 0 {
-        map.serialize_entry("flags", &(link.flags as u64))?;
+        map.serialize_entry(
+            "flags",
+            &((link.flags & !SPAN_LINK_FLAGS_SET_SENTINEL) as u64),
+        )?;
     }
     let tracestate: &str = link.tracestate.borrow();
     if !tracestate.is_empty() {

libdd-trace-utils/src/agentless_encoder/tests.rs

@@ -178,6 +178,51 @@ fn span_links_serialised_into_meta_as_json_string() {
     assert_eq!(link_obj["tracestate"], "dd=s:1");
 }
 
+#[cfg_attr(miri, ignore)] // serde_json/rmp_serde overhead is prohibitively slow under Miri
+#[test]
+fn span_link_flags_sentinel_bit_masked() {
+    // The internal "explicitly set" sentinel (bit 31) must never appear in the
+    // `_dd.span_links` JSON, which downstream consumers treat as the real W3C trace-flags value.
+    // Covers both sentinel states: kept (0x8000_0001) and explicitly dropped (0x8000_0000).
+    fn encoded_flags(flags: u32) -> serde_json::Value {
+        let link = SpanLink::<BytesData> {
+            trace_id: 0x11,
+            span_id: 0x22,
+            flags,
+            ..Default::default()
+        };
+        let span: Span<BytesData> = Span {
+            service: bs("svc"),
+            name: bs("op"),
+            trace_id: 1,
+            span_id: 1,
+            parent_id: 0,
+            start: 0,
+            duration: 1,
+            span_links: vec![link],
+            ..Default::default()
+        };
+        let v = json_from_bytes(&encode_payload(&[vec![span]], &base_metadata()).unwrap());
+        let s = &v["traces"][0]["spans"][0];
+        let raw = s["meta"]["_dd.span_links"]
+            .as_str()
+            .expect("meta[_dd.span_links] must be a string");
+        let links: serde_json::Value = serde_json::from_str(raw).expect("must be valid JSON");
+        links[0]["flags"].clone()
+    }
+
+    assert_eq!(
+        encoded_flags(0x8000_0001),
+        1,
+        "the sentinel bit must not leak into the _dd.span_links JSON"
+    );
+    assert_eq!(
+        encoded_flags(0x8000_0000),
+        0,
+        "an explicit drop decision must still emit flags: 0, not omit the field"
+    );
+}
+
 #[cfg_attr(miri, ignore)] // serde_json/rmp_serde overhead is prohibitively slow under Miri
 #[test]
 fn span_events_serialised_into_meta_as_json_string() {

libdd-trace-utils/src/json_log_encoder/mod.rs

@@ -443,6 +443,43 @@ mod tests {
         assert_eq!(span_json["span_events"][0]["time_unix_nano"], 123);
     }
 
+    #[test]
+    fn span_link_flags_sentinel_bit_masked() {
+        // The internal "explicitly set" sentinel (bit 31) must never appear in the emitted
+        // JSON log, which downstream consumers treat as the real W3C trace-flags value.
+        // Covers both sentinel states: kept (0x8000_0001) and explicitly dropped (0x8000_0000).
+        use crate::span::v04::SpanLink;
+
+        fn encoded_flags(flags: u32) -> Value {
+            let span = SpanSlice {
+                span_id: 1,
+                span_links: vec![SpanLink {
+                    trace_id: 7,
+                    span_id: 8,
+                    flags,
+                    ..Default::default()
+                }],
+                ..Default::default()
+            };
+            let mut out = Vec::new();
+            encode_traces(&[vec![span]], &mut out, MAX).unwrap();
+            let emitted = lines(&out);
+            let parsed: Value = serde_json::from_str(&emitted[0]).unwrap();
+            parsed["traces"][0][0]["span_links"][0]["flags"].clone()
+        }
+
+        assert_eq!(
+            encoded_flags(0x8000_0001),
+            1,
+            "the sentinel bit must not leak into the JSON log"
+        );
+        assert_eq!(
+            encoded_flags(0x8000_0000),
+            0,
+            "an explicit drop decision must still emit flags: 0, not omit the field"
+        );
+    }
+
     #[test]
     fn empty_inner_trace_writes_nothing() {
         // One trace containing zero spans: nothing is emitted and no spans counted.

libdd-trace-utils/src/json_log_encoder/span.rs

@@ -16,7 +16,7 @@
 //! `T: Serialize` bound — keeping the public exporter API free of that bound.
 
 use crate::span::v04::{Span, SpanEvent, SpanLink};
-use crate::span::TraceData;
+use crate::span::{TraceData, SPAN_LINK_FLAGS_SET_SENTINEL};
 use serde::ser::{SerializeSeq, SerializeStruct};
 use serde::{Serialize, Serializer};
 use std::borrow::Borrow;
@@ -102,7 +102,9 @@ impl<T: TraceData> Serialize for LogSpanLink<'_, T> {
             state.serialize_field("tracestate", &link.tracestate)?;
         }
         if has_flags {
-            state.serialize_field("flags", &link.flags)?;
+            // Mask off the internal "explicitly set" sentinel (bit 31): this JSON log field
+            // is consumer-facing and must not expose the internal wire encoding.
+            state.serialize_field("flags", &(link.flags & !SPAN_LINK_FLAGS_SET_SENTINEL))?;
         }
         state.end()
     }

libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs

@@ -281,3 +281,56 @@ pub fn encode_span<W: RmpWrite, T: TraceData>(
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::super::to_vec_from_v04;
+    use crate::span::v04::SpanLink;
+    use crate::span::BytesData;
+
+    #[test]
+    fn span_link_flags_are_not_masked_on_the_native_v04_wire() {
+        // The v0.4 native msgpack wire format is the only place where the sentinel bit (bit 31)
+        // stays in the raw value. The Datadog Agent and other tracers already decode this exact
+        // bit pattern from v0.4 span links.
+        let link = SpanLink::<BytesData> {
+            trace_id: 1,
+            span_id: 2,
+            flags: 0x8000_0001,
+            ..Default::default()
+        };
+        let span = crate::span::v04::Span::<BytesData> {
+            span_id: 1,
+            span_links: vec![link],
+            ..Default::default()
+        };
+        let bytes = to_vec_from_v04(&[vec![span]]);
+        let value = rmpv::decode::read_value(&mut &bytes[..]).expect("decode failed");
+        let traces = value.as_array().expect("top-level must be an array");
+        let trace = traces[0].as_array().expect("trace must be an array");
+        let encoded_span = &trace[0];
+        let span_links = encoded_span
+            .as_map()
+            .expect("span must be a map")
+            .iter()
+            .find(|(k, _)| k.as_str() == Some("span_links"))
+            .map(|(_, v)| v)
+            .expect("span_links must be present")
+            .as_array()
+            .expect("span_links must be an array");
+        let flags = span_links[0]
+            .as_map()
+            .expect("span link must be a map")
+            .iter()
+            .find(|(k, _)| k.as_str() == Some("flags"))
+            .map(|(_, v)| v)
+            .expect("flags must be present")
+            .as_u64()
+            .expect("flags must be a u64");
+
+        assert_eq!(
+            flags, 0x8000_0001,
+            "v0.4's native wire format must carry flags raw, including the sentinel bit"
+        );
+    }
+}

libdd-trace-utils/src/otlp_encoder/mapper.rs

@@ -11,7 +11,7 @@
 
 use super::OtlpResourceInfo;
 use crate::span::v04::{Span, SpanEvent, SpanLink};
-use crate::span::TraceData;
+use crate::span::{TraceData, SPAN_LINK_FLAGS_SET_SENTINEL};
 use std::borrow::Borrow;
 
 use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq;
@@ -469,8 +469,9 @@ fn map_span_link<T: TraceData>(link: &SpanLink<T>) -> ProtoLink {
             .collect(),
         dropped_attributes_count: 0,
         // W3C trace flags of the linked context (sampled bit, etc.); carry them through so OTLP
-        // consumers see the same link metadata the tracer recorded.
-        flags: link.flags,
+        // consumers see the same link metadata the tracer recorded. Bit 31 is an internal
+        // "explicitly set" sentinel that must not leak into OTLP's flags field.
+        flags: link.flags & !SPAN_LINK_FLAGS_SET_SENTINEL,
     }
 }
 
@@ -1005,6 +1006,42 @@ mod tests {
         );
     }
 
+    #[test]
+    fn span_link_flags_sentinel_bit_masked() {
+        // The internal "explicitly set" sentinel (bit 31) must never appear in OTLP's
+        // Link.flags, which downstream consumers treat as the real W3C trace-flags value.
+        // Covers both sentinel states: kept (0x8000_0001) and explicitly dropped (0x8000_0000).
+        fn mapped_flags(flags: u32) -> u32 {
+            let mut span: Span<BytesData> = Span {
+                trace_id: 1,
+                span_id: 2,
+                name: libdd_tinybytes::BytesString::from_static("s"),
+                start: 0,
+                duration: 1,
+                ..Default::default()
+            };
+            span.span_links.push(SpanLink {
+                trace_id: 0x11,
+                span_id: 0x22,
+                flags,
+                ..Default::default()
+            });
+            let req = map_traces_to_otlp(vec![vec![span]], &OtlpResourceInfo::default(), false);
+            req.resource_spans[0].scope_spans[0].spans[0].links[0].flags
+        }
+
+        assert_eq!(
+            mapped_flags(0x8000_0001),
+            1,
+            "OTLP Link.flags must not carry the internal sentinel bit"
+        );
+        assert_eq!(
+            mapped_flags(0x8000_0000),
+            0,
+            "an explicit drop decision must still map to flags: 0"
+        );
+    }
+
     #[test]
     fn test_otel_trace_semantics_enabled() {
         // With OTel-semantics on, the DD-promoted attributes (service.name/operation.name/

libdd-trace-utils/src/span/mod.rs

@@ -19,6 +19,20 @@ use std::marker::PhantomData;
 use std::ptr::NonNull;
 use std::{fmt, ptr};
 
+/// A `SpanLink`'s `flags` field reserves bit 31 to mean "a value was explicitly set", separate
+/// from the sampling decision carried in the low bits. The sentinel bit distinguishes
+/// `flags == 0` (never set) from an explicit decision of `0`, for example a dropped context.
+/// Without the sentinel, both cases look identical on the wire.
+///
+/// Every non-JSON wire format keeps this bit raw, except OTLP. The native v0.4 msgpack format
+/// (`msgpack_encoder::v04::span_v04`), the v1 msgpack format, and the native protobuf format
+/// (`libdd_trace_protobuf::pb::SpanLink`) all keep the sentinel raw in `flags`. Tracers already
+/// send the bit set in these formats. JSON formats and OTLP protobuf must mask this bit before
+/// they emit `flags`, because those consumers treat `flags` as the real W3C trace-flags value.
+/// The JSON formats are the v0.5 `_dd.span_links` dictionary, agentless JSON, and structured
+/// JSON logging.
+pub(crate) const SPAN_LINK_FLAGS_SET_SENTINEL: u32 = 1 << 31;
+
 /// Trait representing the requirements for a type to be used as a Span "string" type.
 /// Note: Borrow<str> is not required by the derived traits, but allows to access HashMap elements
 /// from a static str and check if the string is empty.

libdd-trace-utils/src/span/v05/mod.rs

@@ -4,7 +4,7 @@
 pub mod dict;
 
 use crate::span::v04::{AttributeAnyValue, AttributeArrayValue, SpanEvent, SpanLink};
-use crate::span::{SharedDictBytes, SpanText, TraceData};
+use crate::span::{SharedDictBytes, SpanText, TraceData, SPAN_LINK_FLAGS_SET_SENTINEL};
 use anyhow::Result;
 use indexmap::map::RawEntryApiV1;
 use libdd_tinybytes::BytesString;
@@ -43,7 +43,7 @@ pub struct Span {
 ///   low 64 bits).
 /// - `span_id` is the 64-bit id hex-encoded as 16 lowercase chars.
 /// - `tracestate` and `attributes` are only emitted when non-empty.
-/// - `flags` is only emitted when not zero.
+/// - `flags` is only emitted when not zero, with [`SPAN_LINK_FLAGS_SET_SENTINEL`] masked off.
 struct SpanLinksSerializerV05<'a, T: TraceData>(&'a [SpanLink<T>]);
 struct SpanLinkSerializerV05<'a, T: TraceData>(&'a SpanLink<T>);
 
@@ -81,7 +81,7 @@ impl<'a, T: TraceData> Serialize for SpanLinkSerializerV05<'a, T> {
             )?;
         }
         if has_flags {
-            map.serialize_entry("flags", &link.flags)?;
+            map.serialize_entry("flags", &(link.flags & !SPAN_LINK_FLAGS_SET_SENTINEL))?;
         }
         map.end()
     }
@@ -465,8 +465,8 @@ mod tests {
         assert!(meta_json(&dict, &v05_span, "meta_struct").is_none());
     }
 
-    /// A link with no tracestate and no attributes serializes only hex `trace_id`/`span_id`;
-    /// `flags` is dropped.
+    /// A link with no tracestate and no attributes serializes only hex `trace_id`/`span_id`,
+    /// plus `flags` since it is non-zero.
     #[test]
     fn span_link_minimal_serialization_test() {
         let links = vec![SpanLink::<BytesData> {
@@ -484,6 +484,38 @@ mod tests {
         );
     }
 
+    /// Covers all three [`SPAN_LINK_FLAGS_SET_SENTINEL`] states: unset, kept, and explicitly
+    /// dropped.
+    #[test]
+    fn span_link_flags_sentinel_bit_masked_test() {
+        let kept = vec![SpanLink::<BytesData> {
+            span_id: 1,
+            flags: 0x8000_0001,
+            ..Default::default()
+        }];
+        let json = serde_json::to_string(&SpanLinksSerializerV05::<BytesData>(&kept)).unwrap();
+        let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
+        assert_eq!(parsed[0]["flags"], serde_json::json!(1));
+
+        let dropped = vec![SpanLink::<BytesData> {
+            span_id: 2,
+            flags: 0x8000_0000,
+            ..Default::default()
+        }];
+        let json = serde_json::to_string(&SpanLinksSerializerV05::<BytesData>(&dropped)).unwrap();
+        let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
+        assert_eq!(parsed[0]["flags"], serde_json::json!(0));
+
+        let unset = vec![SpanLink::<BytesData> {
+            span_id: 3,
+            flags: 0,
+            ..Default::default()
+        }];
+        let json = serde_json::to_string(&SpanLinksSerializerV05::<BytesData>(&unset)).unwrap();
+        let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
+        assert!(parsed[0].get("flags").is_none());
+    }
+
     /// Multiple links serialize as an ordered JSON array preserving input order.
     #[test]
     fn span_links_multiple_serialization_test() {