Skip to content

Add support for DogStatsD v1.1 - value packing #762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tiedotguy merged 3 commits into from
Jul 28, 2025
Merged

Add support for DogStatsD v1.1 - value packing #762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a51297b
Add support for DogStatsD v1.1 - value packing
Jul 3, 2025
2f855c8
Add more tests.
Jul 4, 2025
e65f218
Change wording in comment, handle empty StringValue
Jul 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
41.0.0
------
- Add support for [DogStatsD protocol v1.1](https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#dogstatsd-protocol-v11) - value packing.

40.1.0
------
- Add support for configuration of receiver's buffer size - `receive-buffer-size`
Expand Down
11 changes: 11 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,17 @@ A simple way to test your installation or send metrics from a script is to use

echo 'abc.def.g:10|c' | nc -w1 -u localhost 8125

Since 41.0.0, server supports value packing from [DogStatsD protocol v1.1](https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#dogstatsd-protocol-v11)
It means it's possible to send multiple values within single line, for example when you do pre-consolidation on client side.
It support all metric types except `SET`

The format of value-packing is:

<bucket name>:<value1>:<value2>,<value3>|<type>\n

You can send as much values as you would like, however remember to not exceed single packet size limits (usually size of MTU)
as it will cause packets fragmentation and can lead to worse performance or lost packets.

Monitoring
----------
Many metrics for the internal processes are emitted. See METRICS.md for details. Go expvar is also
Expand Down
2 changes: 1 addition & 1 deletion internal/fixtures/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func SortCompare(ms []*gostatsd.Metric) func(i, j int) bool {
if ms[i].Type == gostatsd.SET {
return ms[i].StringValue < ms[j].StringValue
} else {
return ms[i].Value < ms[j].Value
return ms[i].Values[0] < ms[j].Values[0]
Copy link
Collaborator

@tiedotguy tiedotguy Jul 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note for other reviewers: this code path is used in tests, to provide stable sorting, and making it easier to compare lists. It doesn't need to be perfect in production, just good enough in tests.

}
}
return len(ms[i].Tags) < len(ms[j].Tags)
Expand Down
33 changes: 27 additions & 6 deletions internal/lexer/lexer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"math"
"strconv"
"strings"

"github.com/atlassian/gostatsd"
"github.com/atlassian/gostatsd/internal/pool"
Expand Down Expand Up @@ -101,14 +102,34 @@ func (l *Lexer) Run(input []byte, namespace string) (*gostatsd.Metric, *gostatsd
if l.m != nil {
l.m.Rate = l.sampling
if l.m.Type != gostatsd.SET {
v, err := strconv.ParseFloat(l.m.StringValue, 64)
if err != nil {
return nil, nil, err
// Count number of values by checking colons to preallocate array
var values []float64
if l.m.StringValue == "" {
values = make([]float64, 0, 0)
} else {
count := 1
for i := 0; i < len(l.m.StringValue); i++ {
if l.m.StringValue[i] == ':' {
count++
}
}
values = make([]float64, 0, count)
}
if math.IsNaN(v) {
return nil, nil, errNaN
for _, stringValue := range strings.Split(l.m.StringValue, ":") {
if stringValue == "" {
// SKip the value, it could be something like a.packing:1:2:|ms|#|:|c:xyz
continue
}
v, err := strconv.ParseFloat(stringValue, 64)
if err != nil {
return nil, nil, err
}
if math.IsNaN(v) {
return nil, nil, errNaN
}
values = append(values, v)
}
l.m.Value = v
l.m.Values = values
l.m.StringValue = ""
}
l.m.Tags = l.tags
Expand Down
104 changes: 58 additions & 46 deletions internal/lexer/lexer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,54 +14,66 @@ import (
func TestMetricsLexer(t *testing.T) {
t.Parallel()
tests := map[string]gostatsd.Metric{
"foo.bar.baz:2|c": {Name: "foo.bar.baz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "abc.def.g", Value: 3, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "def.g", Value: 10, Type: gostatsd.TIMER, Rate: 1.0},
"def.h:10|h": {Name: "def.h", Value: 10, Type: gostatsd.TIMER, Rate: 1.0},
"def.i:10|h|#foo": {Name: "def.i", Value: 10, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"smp.rte:5|c|@0.1": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"smp.rte:5|c|#foo:bar,baz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"foo.bar.baz:2|c": {Name: "foo.bar.baz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "abc.def.g", Values: []float64{3}, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "def.g", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0},
"def.h:10|h": {Name: "def.h", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0},
"def.i:10|h|#foo": {Name: "def.i", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"smp.rte:5|c|@0.1": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"smp.rte:5|c|#foo:bar,baz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"uniq.usr:joe|s": {Name: "uniq.usr", StringValue: "joe", Type: gostatsd.SET, Rate: 1.0},
"fooBarBaz:2|c": {Name: "fooBarBaz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|#Foo:Bar,baz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar", "baz"}},
"smp.gge:1|g|#Foo:Bar": {Name: "smp.gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar"}},
"smp.gge:1|g|#fo_o:ba-r": {Name: "smp.gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"fo_o:ba-r"}},
"smp gge:1|g": {Name: "smp_gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"smp/gge:1|g": {Name: "smp-gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"smp,gge$:1|g": {Name: "smpgge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"fooBarBaz:2|c": {Name: "fooBarBaz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|#Foo:Bar,baz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar", "baz"}},
"smp.gge:1|g|#Foo:Bar": {Name: "smp.gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar"}},
"smp.gge:1|g|#fo_o:ba-r": {Name: "smp.gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"fo_o:ba-r"}},
"smp gge:1|g": {Name: "smp_gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"smp/gge:1|g": {Name: "smp-gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"smp,gge$:1|g": {Name: "smpgge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"un1qu3:john|s": {Name: "un1qu3", StringValue: "john", Type: gostatsd.SET, Rate: 1.0},
"un1qu3:john|s|#some:42": {Name: "un1qu3", StringValue: "john", Type: gostatsd.SET, Rate: 1.0, Tags: gostatsd.Tags{"some:42"}},
"da-sh:1|s": {Name: "da-sh", StringValue: "1", Type: gostatsd.SET, Rate: 1.0},
"under_score:1|s": {Name: "under_score", StringValue: "1", Type: gostatsd.SET, Rate: 1.0},
"a:1|g|#f,,": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#,,f": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#f,,z": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"a:1|g|#": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,,": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"foo.bar.baz:2|c|c:xyz": {Name: "foo.bar.baz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|@0.1|c:xyz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz|c:xyz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"def.i:10|h|#foo|c:xyz": {Name: "def.i", Value: 10, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"c.after.tags:1|g|#f,,|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#,,f|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#f,,z|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"c.after.tags:1|g|#|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c::,#@": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"field.order.rev.all:1|g|c:xyz|#foo:bar|@0.1": {Name: "field.order.rev.all", Value: 1, Type: gostatsd.GAUGE, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar"}},
"field.order.rev.notags:1|g|c:xyz|@0.1": {Name: "field.order.rev.notags", Value: 1, Type: gostatsd.GAUGE, Rate: 0.1},
"new.last.prefix:1|g|#,,|c:xyz|x:": {Name: "new.last.prefix", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.empty:1|g|#,|c:xyz|": {Name: "new.last.empty", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.colon:1|g|#|c:xyz|:": {Name: "new.last.colon", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.prefix:1|g|x:#,,|c:xyz|": {Name: "new.first.prefix", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.empty:1|g||#,|c:xyz": {Name: "new.first.empty", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.colon:1|g|:|#|c:xyz": {Name: "new.first.colon", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.prefix:1|g|#,,|x:|c:xyz": {Name: "new.mid.prefix", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.empty:1|g|#,||c:xyz": {Name: "new.mid.empty", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.colon:1|g|#|:|c:xyz": {Name: "new.mid.colon", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#f,,": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#,,f": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#f,,z": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"a:1|g|#": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,,": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"foo.bar.baz:2|c|c:xyz": {Name: "foo.bar.baz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|@0.1|c:xyz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz|c:xyz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"def.i:10|h|#foo|c:xyz": {Name: "def.i", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"c.after.tags:1|g|#f,,|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#,,f|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#f,,z|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"c.after.tags:1|g|#|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c::,#@": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"field.order.rev.all:1|g|c:xyz|#foo:bar|@0.1": {Name: "field.order.rev.all", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar"}},
"field.order.rev.notags:1|g|c:xyz|@0.1": {Name: "field.order.rev.notags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 0.1},
"new.last.prefix:1|g|#,,|c:xyz|x:": {Name: "new.last.prefix", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.empty:1|g|#,|c:xyz|": {Name: "new.last.empty", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.colon:1|g|#|c:xyz|:": {Name: "new.last.colon", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.prefix:1|g|x:#,,|c:xyz|": {Name: "new.first.prefix", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.empty:1|g||#,|c:xyz": {Name: "new.first.empty", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.colon:1|g|:|#|c:xyz": {Name: "new.first.colon", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.prefix:1|g|#,,|x:|c:xyz": {Name: "new.mid.prefix", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.empty:1|g|#,||c:xyz": {Name: "new.mid.empty", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.colon:1|g|#|:|c:xyz": {Name: "new.mid.colon", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
// Value packing tests
"a.packing:1:2|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:1:2:3|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2, 3}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:1:2:|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:1:2|c|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.COUNTER, Rate: 1.0},
"a.packing:1:2:3|c|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2, 3}, Type: gostatsd.COUNTER, Rate: 1.0},
"a.packing:1:2:|c|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.COUNTER, Rate: 1.0},
"a.packing:1:2|g|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.GAUGE, Rate: 1.0},
"a.packing:1:2:3|g|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2, 3}, Type: gostatsd.GAUGE, Rate: 1.0},
"a.packing:1:2:|g|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.GAUGE, Rate: 1.0},
"a.packing:::|g|#|:|c:xyz": {Name: "a.packing", Values: []float64{}, Type: gostatsd.GAUGE, Rate: 1.0},
}

compareMetric(t, tests, "")
Expand All @@ -85,9 +97,9 @@ func TestInvalidMetricsLexer(t *testing.T) {
}

tests := map[string]gostatsd.Metric{
"foo.bar.baz:2|c": {Name: "stats.foo.bar.baz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "stats.abc.def.g", Value: 3, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "stats.def.g", Value: 10, Type: gostatsd.TIMER, Rate: 1.0},
"foo.bar.baz:2|c": {Name: "stats.foo.bar.baz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "stats.abc.def.g", Values: []float64{3}, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "stats.def.g", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0},
"uniq.usr:joe|s": {Name: "stats.uniq.usr", StringValue: "joe", Type: gostatsd.SET, Rate: 1.0},
}

Expand Down
6 changes: 3 additions & 3 deletions metric_consolidator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ func TestConsolidation(t *testing.T) {
m1 := &Metric{
Name: "foo",
Type: COUNTER,
Value: 1,
Values: []float64{1},
Rate: 1,
Timestamp: 10,
}
m2 := &Metric{
Name: "foo",
Type: COUNTER,
Value: 3,
Values: []float64{3},
Rate: 0.1,
Timestamp: 20,
}
Expand Down Expand Up @@ -76,7 +76,7 @@ func randomMetric(seed, variations int) *Metric {
if m.Type == SET {
m.StringValue = fmt.Sprintf("%d", seed)
} else {
m.Value = float64(seed)
m.Values = []float64{float64(seed)}
m.Rate = 1
}
m.Timestamp = 10
Expand Down
Loading
Loading