You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
fix(network/tracer): fix data race on conntrackerTelemetry.lastRegisters
Collect() did a non-atomic read-modify-write on the package-level
conntrackerTelemetry.lastRegisters field. When two Prometheus Gather
ticks overlapped, both goroutines raced on this field.
Move lastRegisters to the ebpfConntracker instance and use a CAS loop
(CompareAndSwap) so the read-modify-write is atomic and the stored
counter only moves forward. Making the baseline per-instance also fixes
a regression on module restart: a new conntracker gets a fresh
telemetry map, so the package-level baseline (which retained the old
map's counter) would have suppressed all deltas until the new counter
exceeded the old total.
Found via a race-detector-enabled build in staging (see #54333).
Copy file name to clipboardExpand all lines: pkg/network/tracer/ebpf_conntracker.go
+27-4Lines changed: 27 additions & 4 deletions
Original file line number
Diff line number
Diff line change
@@ -13,6 +13,7 @@ import (
13
13
"fmt"
14
14
"io"
15
15
"net/netip"
16
+
"sync/atomic"
16
17
"time"
17
18
18
19
manager "github.com/DataDog/ebpf-manager"
@@ -62,14 +63,12 @@ var conntrackerTelemetry = struct {
62
63
getsTotal telemetryComp.Counter
63
64
unregistersTotal telemetryComp.Counter
64
65
registersTotal*prometheus.Desc
65
-
lastRegistersuint64
66
66
}{
67
67
telemetryimpl.GetCompatComponent().NewHistogram(ebpfConntrackerModuleName, "gets_duration_nanoseconds", []string{}, "Histogram measuring the time spent retrieving connection tuples from the EBPF map", defaultBuckets),
68
68
telemetryimpl.GetCompatComponent().NewHistogram(ebpfConntrackerModuleName, "unregisters_duration_nanoseconds", []string{}, "Histogram measuring the time spent deleting connection tuples from the EBPF map", defaultBuckets),
69
69
telemetryimpl.GetCompatComponent().NewCounter(ebpfConntrackerModuleName, "gets_total", []string{}, "Counter measuring the total number of attempts to get connection tuples from the EBPF map"),
70
70
telemetryimpl.GetCompatComponent().NewCounter(ebpfConntrackerModuleName, "unregisters_total", []string{}, "Counter measuring the total number of attempts to delete connection tuples from the EBPF map"),
71
71
prometheus.NewDesc(ebpfConntrackerModuleName+"__registers_total", "Counter measuring the total number of attempts to update/create connection tuples in the EBPF map", nil, nil),
72
-
0,
73
72
}
74
73
75
74
typeebpfConntrackerstruct {
@@ -83,6 +82,13 @@ type ebpfConntracker struct {
83
82
stopchanstruct{}
84
83
85
84
isPrebuiltbool
85
+
86
+
// lastRegisters is the baseline for the registers_total delta reported via
87
+
// Collect. It is per-instance so that a module restart (which builds a new
88
+
// ebpfConntracker with a fresh telemetry map) starts from zero instead of
89
+
// inheriting the previous map's counter, which would suppress all deltas
0 commit comments