Skip to content

Commit bb429a9

Browse files
authored
Adds a SendFlows() message to the sender for synchronous usage (#35)
* Remove warning * Remove redundant cast * Add helper to send flows synchronously * Pull out sample rate logic into function, remove metrics for sampling * Re-use logic for serialization * Add test, fix metrics bug, decouple from agg workers * Remove metrics from test * Add comment around messagePrefix * Avoid modifying backing URL when starting a sender * Assert/require that the backing URL is unchanged when a sender is setup * Add docs on device ID override in SendFlows * Track SendFlows to ensure a "Stop()" call waits for any outstanding calls to finish * Use more explicit metric for flows sent * Add notes on downsampling/rate limiting for SendFlows
1 parent 2dd1dff commit bb429a9

9 files changed

Lines changed: 253 additions & 65 deletions

File tree

agg/agg.go

Lines changed: 9 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ import (
44
"sync"
55
"time"
66

7-
"github.com/kentik/libkflow/chf"
7+
capnp "zombiezen.com/go/capnproto2"
8+
89
"github.com/kentik/libkflow/flow"
910
"github.com/kentik/libkflow/metrics"
10-
capnp "zombiezen.com/go/capnproto2"
1111
)
1212

1313
type Agg struct {
@@ -106,7 +106,7 @@ func (a *Agg) aggregate() {
106106
}
107107

108108
func (a *Agg) dispatch() {
109-
msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
109+
_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
110110
if err != nil {
111111
a.error(err)
112112
return
@@ -122,49 +122,19 @@ func (a *Agg) dispatch() {
122122
return
123123
}
124124

125-
root, err := chf.NewRootPackedCHF(seg)
126-
if err != nil {
127-
a.error(err)
128-
return
129-
}
125+
// adjust the sample rate for the provided flows
126+
flow.NormalizeSampleRate(flows, resampleRateAdj)
130127

131-
msgs, err := root.NewMsgs(int32(len(flows)))
128+
// serialize the data using the provided segment (backed by msg)
129+
message, err := flow.ToCapnProtoMessage(flows, seg)
132130
if err != nil {
133131
a.error(err)
134132
return
135133
}
136134

137-
var sampleRate uint32
138-
var adjustedSR uint32
139-
140-
for i, f := range flows {
141-
sampleRate = f.SampleRate
142-
adjustedSR = sampleRate * 100
143-
144-
if resampleRateAdj > 1.0 {
145-
adjustedSR = uint32(float32(adjustedSR) * resampleRateAdj)
146-
}
147-
148-
f.SampleAdj = true
149-
f.SampleRate = adjustedSR
150-
151-
var list chf.Custom_List
152-
if n := int32(len(f.Customs)); n > 0 {
153-
if list, err = chf.NewCustom_List(seg, n); err != nil {
154-
a.error(err)
155-
return
156-
}
157-
}
158-
159-
f.FillCHF(msgs.At(i), list)
160-
}
161-
162-
root.SetMsgs(msgs)
163-
a.output <- msg
135+
a.output <- message
164136

165-
a.metrics.OrigSampleRate.Update(int64(sampleRate))
166-
a.metrics.NewSampleRate.Update(int64(adjustedSR))
167-
a.metrics.TotalFlowsOut.Mark(int64(count))
137+
a.metrics.TotalFlowsOut.Mark(int64(len(flows)))
168138
}
169139

170140
func (a *Agg) error(err error) {

agg/agg_test.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@ import (
77

88
capnp "zombiezen.com/go/capnproto2"
99

10+
"github.com/stretchr/testify/assert"
11+
1012
"github.com/kentik/libkflow/chf"
1113
"github.com/kentik/libkflow/flow"
1214
"github.com/kentik/libkflow/metrics"
13-
"github.com/stretchr/testify/assert"
1415
)
1516

1617
func TestAggSimple(t *testing.T) {
@@ -68,8 +69,6 @@ func setup(t *testing.T, interval time.Duration, fps int) (*testState, *assert.A
6869
metrics := &metrics.Metrics{
6970
TotalFlowsIn: metrics.NewMeter(),
7071
TotalFlowsOut: metrics.NewMeter(),
71-
OrigSampleRate: metrics.NewHistogram(metrics.NewUniformSample(100)),
72-
NewSampleRate: metrics.NewHistogram(metrics.NewUniformSample(100)),
7372
RateLimitDrops: metrics.NewMeter(),
7473
BytesSent: metrics.NewMeter(),
7574
}

api/client.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,14 +376,16 @@ func (c *Client) UpdateInterfacesDirectly(dev *Device, updates map[string]Interf
376376
return nil
377377
}
378378

379+
// SendFlow sends the provided buffer containing a gzipped, cap'n proto packed, representation of flows to the provided
380+
// url.
379381
func (c *Client) SendFlow(url string, buf *bytes.Buffer) error {
380382
r, err := c.do("POST", url, "application/binary", buf, true)
381383
if err != nil {
382384
return err
383385
}
384386

385387
defer r.Body.Close()
386-
io.Copy(io.Discard, r.Body)
388+
_, _ = io.Copy(io.Discard, r.Body)
387389

388390
if r.StatusCode != 200 {
389391
return fmt.Errorf("api: HTTP status code %d", r.StatusCode)

flow/flow.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ func (f *Flow) FillCHF(kflow chf.CHF, list chf.Custom_List) {
262262

263263
for i, c := range f.Customs {
264264
kc := list.At(i)
265-
kc.SetId(uint32(c.ID))
265+
kc.SetId(c.ID)
266266

267267
switch c.Type {
268268
case Str:

flow/sampling.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package flow
2+
3+
// NormalizeSampleRate adjusts the sample rate in place on the provided [flow.Flow] slice based on a provided
4+
// adjustment factor if it is > 1.0. The adjustment factor is multiplied by the original sample rate and 100 to get
5+
// the new sample rate, as it is expected that a [flow.Flow] with a sample rate that does not account for this change.
6+
func NormalizeSampleRate(flows []Flow, resampleRateAdj float32) {
7+
for i := range flows {
8+
sampleRate := flows[i].SampleRate
9+
adjustedSR := sampleRate * 100
10+
11+
if resampleRateAdj > 1.0 {
12+
adjustedSR = uint32(float32(adjustedSR) * resampleRateAdj)
13+
}
14+
15+
flows[i].SampleAdj = true
16+
flows[i].SampleRate = adjustedSR
17+
}
18+
}

flow/serialization.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package flow
2+
3+
import (
4+
"fmt"
5+
6+
capnp "zombiezen.com/go/capnproto2"
7+
8+
"github.com/kentik/libkflow/chf"
9+
)
10+
11+
// ToCapnProtoMessage converts a slice of Flow objects into a Cap'n Proto message.
12+
func ToCapnProtoMessage(flows []Flow, segment *capnp.Segment) (*capnp.Message, error) {
13+
packedCHF, err := chf.NewRootPackedCHF(segment)
14+
if err != nil {
15+
return nil, fmt.Errorf("failed to create root packed CHF: %w", err)
16+
}
17+
18+
chfList, err := packedCHF.NewMsgs(int32(len(flows)))
19+
if err != nil {
20+
return nil, fmt.Errorf("failed to create messages list: %w", err)
21+
}
22+
23+
for i, f := range flows {
24+
var list chf.Custom_List
25+
if n := int32(len(f.Customs)); n > 0 {
26+
list, err = chf.NewCustom_List(segment, n)
27+
if err != nil {
28+
return nil, fmt.Errorf("failed to create custom list: %w", err)
29+
}
30+
}
31+
f.FillCHF(chfList.At(i), list)
32+
}
33+
34+
err = packedCHF.SetMsgs(chfList)
35+
if err != nil {
36+
return nil, fmt.Errorf("failed to set messages: %w", err)
37+
}
38+
return segment.Message(), nil
39+
}

metrics/metrics.go

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ type Metrics struct {
2020
reg metrics.Registry
2121
TotalFlowsIn metrics.Meter
2222
TotalFlowsOut metrics.Meter
23-
OrigSampleRate metrics.Histogram
24-
NewSampleRate metrics.Histogram
2523
RateLimitDrops metrics.Meter
2624
BytesSent metrics.Meter
2725
}
@@ -35,18 +33,12 @@ func New(companyID int, deviceID int, program, version string) *Metrics {
3533

3634
// NewWithRegistry returns a new Metrics but allows a specific registry to be used rather than creating a new one
3735
func NewWithRegistry(reg metrics.Registry, companyID int, deviceID int, program, version string) *Metrics {
38-
sample := func() metrics.Sample {
39-
return metrics.NewExpDecaySample(MetricsSampleSize, MetricsSampleAlpha)
40-
}
41-
4236
suffix := fmt.Sprintf("^ver=%s^ft=%s^dt=%s^level=%s^cid=%s^did=%s", program+"-"+version, program, "libkflow", "primary", strconv.Itoa(companyID), strconv.Itoa(deviceID))
4337

4438
return &Metrics{
4539
reg: reg,
4640
TotalFlowsIn: metrics.GetOrRegisterMeter("client_Total"+suffix, reg),
4741
TotalFlowsOut: metrics.GetOrRegisterMeter("client_DownsampleFPS"+suffix, reg),
48-
OrigSampleRate: metrics.GetOrRegisterHistogram("client_OrigSampleRate"+suffix, reg, sample()),
49-
NewSampleRate: metrics.GetOrRegisterHistogram("client_NewSampleRate"+suffix, reg, sample()),
5042
RateLimitDrops: metrics.GetOrRegisterMeter("client_RateLimitDrops"+suffix, reg),
5143
BytesSent: metrics.GetOrRegisterMeter("client_BytesSent"+suffix, reg),
5244
}

send.go

Lines changed: 109 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,26 @@ import (
44
"bytes"
55
"compress/gzip"
66
"context"
7+
"fmt"
78
"net/url"
89
"os"
910
"sync"
1011
"time"
1112

13+
"github.com/tinylib/msgp/msgp"
14+
capnp "zombiezen.com/go/capnproto2"
15+
1216
"github.com/kentik/libkflow/agg"
1317
"github.com/kentik/libkflow/api"
1418
"github.com/kentik/libkflow/flow"
1519
"github.com/kentik/libkflow/log"
1620
"github.com/kentik/libkflow/metrics"
17-
"github.com/tinylib/msgp/msgp"
18-
capnp "zombiezen.com/go/capnproto2"
1921
)
2022

23+
// messagePrefix is an 80-byte prefix for the message header when sending kflow to the Kentik API. This is a deprecated
24+
// header, but the bytes must remain for backwards compatibility with the Kentik API.
25+
var messagePrefix = [80]byte{}
26+
2127
// A Sender aggregates and transmits flow information to Kentik.
2228
type Sender struct {
2329
agg *agg.Agg
@@ -56,6 +62,80 @@ func (s *Sender) Send(flow *flow.Flow) {
5662
s.agg.Add(flow)
5763
}
5864

65+
// SendFlows sends the flows to the Kentik API, returning the number of bytes sent as the payload. The device ID on
66+
// the flows is set to the device ID of the sender, regardless of what it was previously set to. This is to ensure all
67+
// data matches the expectations of the downstream URL/API.
68+
//
69+
// This will directly send the slice of flows without any additional downsampling or rate limiting. This does not
70+
// contribute to the underlying Send call.
71+
func (s *Sender) SendFlows(flows []flow.Flow) (int64, error) {
72+
s.workers.Add(1)
73+
defer s.workers.Done()
74+
75+
if s.Device == nil {
76+
return 0, fmt.Errorf("device not initialized")
77+
}
78+
if len(flows) == 0 {
79+
return 0, nil
80+
}
81+
decoratedURL, err := s.createURLString()
82+
if err != nil {
83+
return 0, fmt.Errorf("failed to create URL string: %w", err)
84+
}
85+
86+
if s.Metrics != nil {
87+
s.Metrics.TotalFlowsIn.Mark(int64(len(flows)))
88+
}
89+
90+
// ensure all flows have the device ID set; otherwise it may not be properly queried
91+
for i := range flows {
92+
flows[i].DeviceId = uint32(s.Device.ID)
93+
}
94+
95+
// ensure the sample rate is matching the kentik api expectations
96+
flow.NormalizeSampleRate(flows, 0)
97+
98+
// serialize the data
99+
_, segment, err := capnp.NewMessage(capnp.SingleSegment(nil))
100+
if err != nil {
101+
return 0, fmt.Errorf("failed to create capn proto segment: %w", err)
102+
}
103+
message, err := flow.ToCapnProtoMessage(flows, segment)
104+
if err != nil {
105+
return 0, fmt.Errorf("failed to convert flows to capn proto: %w", err)
106+
}
107+
108+
// write the data with additional gzip compression
109+
buf := &bytes.Buffer{}
110+
z := gzip.NewWriter(buf)
111+
_, err = z.Write(messagePrefix[:])
112+
if err != nil {
113+
return 0, fmt.Errorf("failed to write empty message header: %w", err)
114+
}
115+
err = capnp.NewPackedEncoder(z).Encode(message)
116+
if err != nil {
117+
return 0, fmt.Errorf("failed to encode packed capn proto message: %w", err)
118+
}
119+
err = z.Close()
120+
if err != nil {
121+
return 0, fmt.Errorf("failed to close gzip writer: %w", err)
122+
}
123+
124+
// send the compressed and packed message to the Kentik API
125+
payloadLength := int64(len(buf.Bytes()))
126+
err = s.client.SendFlow(decoratedURL, buf)
127+
if err != nil {
128+
return 0, err
129+
}
130+
131+
if s.Metrics != nil {
132+
s.Metrics.TotalFlowsOut.Mark(int64(len(flows)))
133+
s.Metrics.BytesSent.Mark(payloadLength)
134+
}
135+
136+
return payloadLength, nil
137+
}
138+
59139
// Stop requests a graceful shutdown of the Sender.
60140
func (s *Sender) Stop(wait time.Duration) bool {
61141
s.agg.Stop()
@@ -107,18 +187,18 @@ func (s *Sender) SendEncodedDNS(data []byte) {
107187
}
108188

109189
func (s *Sender) start(agg *agg.Agg, client *api.Client, device *api.Device, n int) error {
110-
q := s.url.Query()
111-
q.Set("sid", "0")
112-
q.Set("sender_id", device.ClientID())
113-
114190
s.agg = agg
115-
s.url.RawQuery = q.Encode()
116191
s.Device = device
117192
s.client = client
118193
s.workers.Add(n)
119194

195+
decoratedURL, err := s.createURLString()
196+
if err != nil {
197+
return fmt.Errorf("failed to create URL string: %w", err)
198+
}
199+
120200
for i := 0; i < n; i++ {
121-
go s.dispatch()
201+
go s.dispatch(decoratedURL)
122202
}
123203
go s.monitor()
124204
go s.update()
@@ -128,16 +208,16 @@ func (s *Sender) start(agg *agg.Agg, client *api.Client, device *api.Device, n i
128208
return nil
129209
}
130210

131-
func (s *Sender) dispatch() {
211+
// dispatch runs a loop to send aggregated flow from the [agg.Agg] to the Kentik API.
212+
func (s *Sender) dispatch(url string) {
132213
buf := &bytes.Buffer{}
133214
cid := [80]byte{}
134-
url := s.url.String()
135215
z := gzip.NewWriter(buf)
136216

137217
for msg := range s.agg.Output() {
138218
log.Debugf("dispatching aggregated flow")
139219
z.Reset(buf)
140-
z.Write(cid[:])
220+
_, _ = z.Write(cid[:])
141221

142222
err := capnp.NewPackedEncoder(z).Encode(msg)
143223
if err != nil {
@@ -247,3 +327,21 @@ func (s *Sender) error(err error) {
247327
default:
248328
}
249329
}
330+
331+
// createURLString creates the full URL to use when sending data to the Kentik API.
332+
func (s *Sender) createURLString() (string, error) {
333+
if s.Device == nil {
334+
return "", fmt.Errorf("device not initialized")
335+
}
336+
if s.url == nil {
337+
return "", fmt.Errorf("url not initialized")
338+
}
339+
340+
// Create a new URL to avoid modifying the original, backed by the config
341+
u := *s.url
342+
q := u.Query()
343+
q.Set("sid", "0")
344+
q.Set("sender_id", s.Device.ClientID())
345+
u.RawQuery = q.Encode()
346+
return u.String(), nil
347+
}

0 commit comments

Comments
 (0)