Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
41.0.0
------
- Add support for [DogStatsD protocol v1.1](https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#dogstatsd-protocol-v11) - value packing.

40.1.0
------
- Add support for configuration of receiver's buffer size - `receive-buffer-size`
Expand Down
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,17 @@ A simple way to test your installation or send metrics from a script is to use

echo 'abc.def.g:10|c' | nc -w1 -u localhost 8125

Since 41.0.0, server supports value packing from [DogStatsD protocol v1.1](https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#dogstatsd-protocol-v11)
It means it's possible to send multiple values within single line, for example when you do pre-consolidation on client side.
It support all metric types except `SET`

The format of value-packing is:

<bucket name>:<value1>:<value2>,<value3>|<type>\n

You can send as much values as you would like, however remember to not exceed single packet size limits (usually size of MTU)
as it will cause packets fragmentation and can lead to worse performance or lost packets.

Monitoring
----------
Many metrics for the internal processes are emitted. See METRICS.md for details. Go expvar is also
Expand Down
2 changes: 1 addition & 1 deletion internal/fixtures/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func SortCompare(ms []*gostatsd.Metric) func(i, j int) bool {
if ms[i].Type == gostatsd.SET {
return ms[i].StringValue < ms[j].StringValue
} else {
return ms[i].Value < ms[j].Value
return ms[i].Values[0] < ms[j].Values[0]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note for other reviewers: this code path is used in tests, to provide stable sorting, and making it easier to compare lists. It doesn't need to be perfect in production, just good enough in tests.

}
}
return len(ms[i].Tags) < len(ms[j].Tags)
Expand Down
29 changes: 23 additions & 6 deletions internal/lexer/lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"math"
"strconv"
"strings"

"github.com/atlassian/gostatsd"
"github.com/atlassian/gostatsd/internal/pool"
Expand Down Expand Up @@ -101,14 +102,30 @@ func (l *Lexer) Run(input []byte, namespace string) (*gostatsd.Metric, *gostatsd
if l.m != nil {
l.m.Rate = l.sampling
if l.m.Type != gostatsd.SET {
v, err := strconv.ParseFloat(l.m.StringValue, 64)
if err != nil {
return nil, nil, err
// Count number of colons to preallocate array
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it count number of colons? why it starts from 1?
based on the code below I guess it's actually the count of values split by colons?

Copy link
Collaborator

@tiedotguy tiedotguy Jul 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We start at count = 1. There's always 1 more value than there is colons.

edit: unless there's empty values, but this is pre-alocation, so we're just trying to avoid re-allocations. We can go a bit over.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've changed the comment and logic a bit to make it clearer.

count := 1
for i := 0; i < len(l.m.StringValue); i++ {
if l.m.StringValue[i] == ':' {
count++
}
}
if math.IsNaN(v) {
return nil, nil, errNaN
values := make([]float64, 0, count)

for _, stringValue := range strings.Split(l.m.StringValue, ":") {
if stringValue == "" {
// SKip the value, it could be something like a.packing:1:2:|ms|#|:|c:xyz
continue
}
v, err := strconv.ParseFloat(stringValue, 64)
if err != nil {
return nil, nil, err
}
if math.IsNaN(v) {
return nil, nil, errNaN
}
values = append(values, v)
}
l.m.Value = v
l.m.Values = values
l.m.StringValue = ""
}
l.m.Tags = l.tags
Expand Down
103 changes: 57 additions & 46 deletions internal/lexer/lexer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,54 +14,65 @@ import (
func TestMetricsLexer(t *testing.T) {
t.Parallel()
tests := map[string]gostatsd.Metric{
"foo.bar.baz:2|c": {Name: "foo.bar.baz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "abc.def.g", Value: 3, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "def.g", Value: 10, Type: gostatsd.TIMER, Rate: 1.0},
"def.h:10|h": {Name: "def.h", Value: 10, Type: gostatsd.TIMER, Rate: 1.0},
"def.i:10|h|#foo": {Name: "def.i", Value: 10, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"smp.rte:5|c|@0.1": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"smp.rte:5|c|#foo:bar,baz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"foo.bar.baz:2|c": {Name: "foo.bar.baz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "abc.def.g", Values: []float64{3}, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "def.g", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0},
"def.h:10|h": {Name: "def.h", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0},
"def.i:10|h|#foo": {Name: "def.i", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"smp.rte:5|c|@0.1": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"smp.rte:5|c|#foo:bar,baz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"uniq.usr:joe|s": {Name: "uniq.usr", StringValue: "joe", Type: gostatsd.SET, Rate: 1.0},
"fooBarBaz:2|c": {Name: "fooBarBaz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|#Foo:Bar,baz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar", "baz"}},
"smp.gge:1|g|#Foo:Bar": {Name: "smp.gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar"}},
"smp.gge:1|g|#fo_o:ba-r": {Name: "smp.gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"fo_o:ba-r"}},
"smp gge:1|g": {Name: "smp_gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"smp/gge:1|g": {Name: "smp-gge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"smp,gge$:1|g": {Name: "smpgge", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"fooBarBaz:2|c": {Name: "fooBarBaz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|#Foo:Bar,baz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar", "baz"}},
"smp.gge:1|g|#Foo:Bar": {Name: "smp.gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"Foo:Bar"}},
"smp.gge:1|g|#fo_o:ba-r": {Name: "smp.gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"fo_o:ba-r"}},
"smp gge:1|g": {Name: "smp_gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"smp/gge:1|g": {Name: "smp-gge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"smp,gge$:1|g": {Name: "smpgge", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"un1qu3:john|s": {Name: "un1qu3", StringValue: "john", Type: gostatsd.SET, Rate: 1.0},
"un1qu3:john|s|#some:42": {Name: "un1qu3", StringValue: "john", Type: gostatsd.SET, Rate: 1.0, Tags: gostatsd.Tags{"some:42"}},
"da-sh:1|s": {Name: "da-sh", StringValue: "1", Type: gostatsd.SET, Rate: 1.0},
"under_score:1|s": {Name: "under_score", StringValue: "1", Type: gostatsd.SET, Rate: 1.0},
"a:1|g|#f,,": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#,,f": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#f,,z": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"a:1|g|#": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,,": {Name: "a", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"foo.bar.baz:2|c|c:xyz": {Name: "foo.bar.baz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|@0.1|c:xyz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz|c:xyz": {Name: "smp.rte", Value: 5, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"def.i:10|h|#foo|c:xyz": {Name: "def.i", Value: 10, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"c.after.tags:1|g|#f,,|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#,,f|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#f,,z|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"c.after.tags:1|g|#|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c:xyz": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c::,#@": {Name: "c.after.tags", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"field.order.rev.all:1|g|c:xyz|#foo:bar|@0.1": {Name: "field.order.rev.all", Value: 1, Type: gostatsd.GAUGE, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar"}},
"field.order.rev.notags:1|g|c:xyz|@0.1": {Name: "field.order.rev.notags", Value: 1, Type: gostatsd.GAUGE, Rate: 0.1},
"new.last.prefix:1|g|#,,|c:xyz|x:": {Name: "new.last.prefix", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.empty:1|g|#,|c:xyz|": {Name: "new.last.empty", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.colon:1|g|#|c:xyz|:": {Name: "new.last.colon", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.prefix:1|g|x:#,,|c:xyz|": {Name: "new.first.prefix", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.empty:1|g||#,|c:xyz": {Name: "new.first.empty", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.colon:1|g|:|#|c:xyz": {Name: "new.first.colon", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.prefix:1|g|#,,|x:|c:xyz": {Name: "new.mid.prefix", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.empty:1|g|#,||c:xyz": {Name: "new.mid.empty", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.colon:1|g|#|:|c:xyz": {Name: "new.mid.colon", Value: 1, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#f,,": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#,,f": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"a:1|g|#f,,z": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"a:1|g|#": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"a:1|g|#,,": {Name: "a", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"foo.bar.baz:2|c|c:xyz": {Name: "foo.bar.baz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"smp.rte:5|c|@0.1|c:xyz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1},
"smp.rte:5|c|@0.1|#foo:bar,baz|c:xyz": {Name: "smp.rte", Values: []float64{5}, Type: gostatsd.COUNTER, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar", "baz"}},
"def.i:10|h|#foo|c:xyz": {Name: "def.i", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0, Tags: gostatsd.Tags{"foo"}},
"c.after.tags:1|g|#f,,|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#,,f|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f"}},
"c.after.tags:1|g|#f,,z|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0, Tags: gostatsd.Tags{"f", "z"}},
"c.after.tags:1|g|#|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c:xyz": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"c.after.tags:1|g|#,,|c::,#@": {Name: "c.after.tags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"field.order.rev.all:1|g|c:xyz|#foo:bar|@0.1": {Name: "field.order.rev.all", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 0.1, Tags: gostatsd.Tags{"foo:bar"}},
"field.order.rev.notags:1|g|c:xyz|@0.1": {Name: "field.order.rev.notags", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 0.1},
"new.last.prefix:1|g|#,,|c:xyz|x:": {Name: "new.last.prefix", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.empty:1|g|#,|c:xyz|": {Name: "new.last.empty", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.last.colon:1|g|#|c:xyz|:": {Name: "new.last.colon", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.prefix:1|g|x:#,,|c:xyz|": {Name: "new.first.prefix", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.empty:1|g||#,|c:xyz": {Name: "new.first.empty", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.first.colon:1|g|:|#|c:xyz": {Name: "new.first.colon", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.prefix:1|g|#,,|x:|c:xyz": {Name: "new.mid.prefix", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.empty:1|g|#,||c:xyz": {Name: "new.mid.empty", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
"new.mid.colon:1|g|#|:|c:xyz": {Name: "new.mid.colon", Values: []float64{1}, Type: gostatsd.GAUGE, Rate: 1.0},
// Value packing tests
"a.packing:1:2|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:1:2:3|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2, 3}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:1:2:|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:|ms|#|:|c:xyz": {Name: "a.packing", Values: []float64{}, Type: gostatsd.TIMER, Rate: 1.0},
"a.packing:1:2|c|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.COUNTER, Rate: 1.0},
"a.packing:1:2:3|c|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2, 3}, Type: gostatsd.COUNTER, Rate: 1.0},
"a.packing:1:2:|c|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.COUNTER, Rate: 1.0},
"a.packing:1:2|g|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.GAUGE, Rate: 1.0},
"a.packing:1:2:3|g|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2, 3}, Type: gostatsd.GAUGE, Rate: 1.0},
"a.packing:1:2:|g|#|:|c:xyz": {Name: "a.packing", Values: []float64{1, 2}, Type: gostatsd.GAUGE, Rate: 1.0},
}

compareMetric(t, tests, "")
Expand All @@ -85,9 +96,9 @@ func TestInvalidMetricsLexer(t *testing.T) {
}

tests := map[string]gostatsd.Metric{
"foo.bar.baz:2|c": {Name: "stats.foo.bar.baz", Value: 2, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "stats.abc.def.g", Value: 3, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "stats.def.g", Value: 10, Type: gostatsd.TIMER, Rate: 1.0},
"foo.bar.baz:2|c": {Name: "stats.foo.bar.baz", Values: []float64{2}, Type: gostatsd.COUNTER, Rate: 1.0},
"abc.def.g:3|g": {Name: "stats.abc.def.g", Values: []float64{3}, Type: gostatsd.GAUGE, Rate: 1.0},
"def.g:10|ms": {Name: "stats.def.g", Values: []float64{10}, Type: gostatsd.TIMER, Rate: 1.0},
"uniq.usr:joe|s": {Name: "stats.uniq.usr", StringValue: "joe", Type: gostatsd.SET, Rate: 1.0},
}

Expand Down
6 changes: 3 additions & 3 deletions metric_consolidator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ func TestConsolidation(t *testing.T) {
m1 := &Metric{
Name: "foo",
Type: COUNTER,
Value: 1,
Values: []float64{1},
Rate: 1,
Timestamp: 10,
}
m2 := &Metric{
Name: "foo",
Type: COUNTER,
Value: 3,
Values: []float64{3},
Rate: 0.1,
Timestamp: 20,
}
Expand Down Expand Up @@ -76,7 +76,7 @@ func randomMetric(seed, variations int) *Metric {
if m.Type == SET {
m.StringValue = fmt.Sprintf("%d", seed)
} else {
m.Value = float64(seed)
m.Values = []float64{float64(seed)}
m.Rate = 1
}
m.Timestamp = 10
Expand Down
Loading
Loading