Skip to content

APM: slab-allocate span attributes in v0.4/v0.5 trace conversion #53739

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gh-worker-dd-mergequeue-cf854d merged 3 commits into from
Jul 20, 2026
Merged

APM: slab-allocate span attributes in v0.4/v0.5 trace conversion #53739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2a90f08
APM: slab-allocate span attributes in v0.4/v0.5 trace conversion
ajgajg1134 Jul 16, 2026
bf07a6c
not default yet
ajgajg1134 Jul 16, 2026
3d2d40e
APM: bound slab pre-allocations in v0.4/v0.5 trace conversion
ajgajg1134 Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 69 additions & 47 deletions pkg/proto/pbgo/trace/idx/span.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1180,25 +1180,32 @@ func (s *InternalSpan) UnmarshalMsgConverted(bts []byte, convertedFields *SpanCo
if s.span.Attributes == nil && numMetaFields > 0 {
s.span.Attributes = make(map[uint32]*AnyValue, numMetaFields)
}
for numMetaFields > 0 {
var metaVal uint32
numMetaFields--
var metaKey uint32
metaKey, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
if numMetaFields > 0 {
if err = checkSlabCount(numMetaFields, bts); err != nil {
err = msgp.WrapError(err, "Meta")
return
}
metaVal, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
err = msgp.WrapError(err, "Meta", metaKey)
return
}
s.handlePromotedMetaFields(metaKey, metaVal, convertedFields)
s.span.Attributes[metaKey] = &AnyValue{
Value: &AnyValue_StringValueRef{
StringValueRef: metaVal,
},
// Slab-allocate the AnyValue containers and their string-ref oneof
// wrappers for every meta entry in two allocations, rather than two per
// entry. The map holds pointers into these backing arrays.
values := make([]AnyValue, numMetaFields)
refs := make([]AnyValue_StringValueRef, numMetaFields)
for i := uint32(0); i < numMetaFields; i++ {
var metaKey, metaVal uint32
metaKey, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
err = msgp.WrapError(err, "Meta")
return
}
metaVal, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
err = msgp.WrapError(err, "Meta", metaKey)
return
}
s.handlePromotedMetaFields(metaKey, metaVal, convertedFields)
refs[i].StringValueRef = metaVal
values[i].Value = &refs[i]
s.span.Attributes[metaKey] = &values[i]
}
}
case "metrics":
Expand All @@ -1215,25 +1222,32 @@ func (s *InternalSpan) UnmarshalMsgConverted(bts []byte, convertedFields *SpanCo
if s.span.Attributes == nil && numMetricsFields > 0 {
s.span.Attributes = make(map[uint32]*AnyValue, numMetricsFields)
}
for numMetricsFields > 0 {
var value float64
numMetricsFields--
var key uint32
key, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
if numMetricsFields > 0 {
if err = checkSlabCount(numMetricsFields, bts); err != nil {
err = msgp.WrapError(err, "Metrics")
return
}
value, bts, err = parseFloat64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Metrics", key)
return
}
s.handlePromotedMetricsFields(key, value, convertedFields)
s.span.Attributes[key] = &AnyValue{
Value: &AnyValue_DoubleValue{
DoubleValue: value,
},
// Slab-allocate the AnyValue containers and their double oneof wrappers
// for every metric in two allocations, rather than two per metric.
values := make([]AnyValue, numMetricsFields)
doubles := make([]AnyValue_DoubleValue, numMetricsFields)
for i := uint32(0); i < numMetricsFields; i++ {
var value float64
var key uint32
key, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
err = msgp.WrapError(err, "Metrics")
return
}
value, bts, err = parseFloat64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Metrics", key)
return
}
s.handlePromotedMetricsFields(key, value, convertedFields)
doubles[i].DoubleValue = value
values[i].Value = &doubles[i]
s.span.Attributes[key] = &values[i]
}
}
case "type":
Expand All @@ -1257,24 +1271,32 @@ func (s *InternalSpan) UnmarshalMsgConverted(bts []byte, convertedFields *SpanCo
if s.span.Attributes == nil && numMetaStructFields > 0 {
s.span.Attributes = make(map[uint32]*AnyValue, numMetaStructFields)
}
for numMetaStructFields > 0 {
var value []byte
numMetaStructFields--
var key uint32
key, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
if numMetaStructFields > 0 {
if err = checkSlabCount(numMetaStructFields, bts); err != nil {
err = msgp.WrapError(err, "MetaStruct")
return
}
value, bts, err = msgp.ReadBytesBytes(bts, value)
if err != nil {
err = msgp.WrapError(err, "MetaStruct", key)
return
}
s.span.Attributes[key] = &AnyValue{
Value: &AnyValue_BytesValue{
BytesValue: value,
},
// Slab-allocate the AnyValue containers and their bytes oneof wrappers
// for every meta_struct entry in two allocations, rather than two per
// entry.
values := make([]AnyValue, numMetaStructFields)
byteVals := make([]AnyValue_BytesValue, numMetaStructFields)
for i := uint32(0); i < numMetaStructFields; i++ {
var value []byte
var key uint32
key, bts, err = parseStringBytesRef(s.Strings, bts)
if err != nil {
err = msgp.WrapError(err, "MetaStruct")
return
}
value, bts, err = msgp.ReadBytesBytes(bts, value)
if err != nil {
err = msgp.WrapError(err, "MetaStruct", key)
return
}
byteVals[i].BytesValue = value
values[i].Value = &byteVals[i]
s.span.Attributes[key] = &values[i]
}
}
case "span_links":
Expand Down
99 changes: 67 additions & 32 deletions pkg/proto/pbgo/trace/idx/span_v05.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ func (tp *InternalTracerPayload) UnmarshalMsgDictionary(bts []byte) error {
tp.Chunks = make([]*InternalTraceChunk, sz)
}
chunkConvertedFields := ChunkConvertedFields{}
convertedTagSet := false
for i := range tp.Chunks {
sz, bts, err = safeReadHeaderBytes(bts, msgp.ReadArrayHeaderBytes)
if err != nil {
Expand All @@ -101,6 +102,13 @@ func (tp *InternalTracerPayload) UnmarshalMsgDictionary(bts []byte) error {
if bts, err = tp.Chunks[i].Spans[j].UnmarshalMsgDictionaryConverted(bts, convertedFields, dictSize, newZeroRef); err != nil {
return err
}
if !convertedTagSet {
// _dd.convertedv1 marks that this payload was converted from the v0.5
// wire format. It is a debugging aid, so we only tag the first span of
// the payload rather than paying the allocation on every span.
tp.Chunks[i].Spans[j].SetStringAttribute("_dd.convertedv1", "v05")
convertedTagSet = true
}
rootSampling.ReconcileSamplingPriorityAfterChunkSpan(convertedFields, tp.Chunks[i].Spans[j].ParentID())
}
tp.Chunks[i].ApplyPromotedFields(convertedFields, &chunkConvertedFields)
Expand Down Expand Up @@ -205,22 +213,29 @@ func (s *InternalSpan) UnmarshalMsgDictionaryConverted(bts []byte, convertedFiel
if s.span.Attributes == nil && sz > 0 {
s.span.Attributes = make(map[uint32]*AnyValue, sz)
}
for sz > 0 {
sz--
var key, val uint32
key, bts, err = readV05StringRef(dictSize, newZeroRef, bts)
if err != nil {
return bts, err
}
val, bts, err = readV05StringRef(dictSize, newZeroRef, bts)
if err != nil {
if sz > 0 {
if err = checkSlabCount(sz, bts); err != nil {
return bts, err
}
s.handlePromotedMetaFields(key, val, convertedFields)
s.span.Attributes[key] = &AnyValue{
Value: &AnyValue_StringValueRef{
StringValueRef: val,
},
// Slab-allocate the AnyValue containers and their string-ref oneof wrappers
// for every meta entry in two allocations, rather than two per entry. The map
// holds pointers into these backing arrays, which live as long as the span.
values := make([]AnyValue, sz)
refs := make([]AnyValue_StringValueRef, sz)
for i := uint32(0); i < sz; i++ {
var key, val uint32
key, bts, err = readV05StringRef(dictSize, newZeroRef, bts)
if err != nil {
return bts, err
}
val, bts, err = readV05StringRef(dictSize, newZeroRef, bts)
if err != nil {
return bts, err
}
s.handlePromotedMetaFields(key, val, convertedFields)
refs[i].StringValueRef = val
values[i].Value = &refs[i]
s.span.Attributes[key] = &values[i]
}
}
// Metrics (10)
Expand All @@ -231,33 +246,38 @@ func (s *InternalSpan) UnmarshalMsgDictionaryConverted(bts []byte, convertedFiel
if s.span.Attributes == nil && sz > 0 {
s.span.Attributes = make(map[uint32]*AnyValue, sz)
}
for sz > 0 {
sz--
var (
key uint32
val float64
)
key, bts, err = readV05StringRef(dictSize, newZeroRef, bts)
if err != nil {
return bts, err
}
val, bts, err = parseFloat64Bytes(bts)
if err != nil {
if sz > 0 {
if err = checkSlabCount(sz, bts); err != nil {
return bts, err
}
s.handlePromotedMetricsFields(key, val, convertedFields)
s.span.Attributes[key] = &AnyValue{
Value: &AnyValue_DoubleValue{
DoubleValue: val,
},
// Slab-allocate the AnyValue containers and their double oneof wrappers for
// every metric in two allocations, rather than two per metric.
values := make([]AnyValue, sz)
doubles := make([]AnyValue_DoubleValue, sz)
Comment thread
ajgajg1134 marked this conversation as resolved.
for i := uint32(0); i < sz; i++ {
var (
key uint32
val float64
)
key, bts, err = readV05StringRef(dictSize, newZeroRef, bts)
if err != nil {
return bts, err
}
val, bts, err = parseFloat64Bytes(bts)
if err != nil {
return bts, err
}
s.handlePromotedMetricsFields(key, val, convertedFields)
doubles[i].DoubleValue = val
values[i].Value = &doubles[i]
s.span.Attributes[key] = &values[i]
}
}
// Type (11)
s.span.TypeRef, bts, err = readV05StringRef(dictSize, newZeroRef, bts)
if err != nil {
return bts, err
}
s.SetStringAttribute("_dd.convertedv1", "v05")
return bts, nil
}

Expand All @@ -274,3 +294,18 @@ func safeReadHeaderBytes(b []byte, read func([]byte) (uint32, []byte, error)) (u
}
return sz, bts, err
}

// minBytesPerSlabEntry is a conservative lower bound on the wire size of one meta/metrics/meta_struct
// entry: every entry reads at least two msgpack values (e.g. two 1-byte nils).
const minBytesPerSlabEntry = 2

// checkSlabCount guards a slab pre-allocation (make([]AnyValue, n) and its oneof-wrapper sibling)
// against a claimed entry count that couldn't possibly be backed by the remaining bytes. Without this,
// a tiny malicious payload could set a map header to millions of entries and force a large allocation
// before decoding ever reaches the missing bytes and fails naturally.
func checkSlabCount(n uint32, remaining []byte) error {
if uint64(n)*minBytesPerSlabEntry > uint64(len(remaining)) {
return fmt.Errorf("not enough data for %d entries", n)
}
return nil
}
Loading
Loading