Merge pull request #527 from dqminh/regexp-cache

add a negative cache to regexp decoder

Author: bobrik

cmd/ebpf_exporter/main.go

@@ -41,6 +41,7 @@ func main() {
 	metricsPath := kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
 	capabilities := kingpin.Flag("capabilities.keep", "Comma separated list of capabilities to keep (cap_syslog, cap_bpf, etc.), 'all' or 'none'").Default("all").String()
 	btfPath := kingpin.Flag("btf.path", "Optional BTF file path.").Default("").String()
+	skipCacheSize := kingpin.Flag("config.skip-cache-size", "Size of the LRU skip cache").Int()
 	kingpin.Version(version.Print("ebpf_exporter"))
 	kingpin.HelpFlag.Short('h')
 	kingpin.Parse()
@@ -92,7 +93,7 @@ func main() {
 
 	notify("creating exporter...")
 
-	e, err := exporter.New(configs, tracing.NewProvider(processor), *btfPath)
+	e, err := exporter.New(configs, *skipCacheSize, tracing.NewProvider(processor), *btfPath)
 	if err != nil {
 		log.Fatalf("Error creating exporter: %s", err)
 	}

decoder/decoder.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/cloudflare/ebpf_exporter/v2/config"
 	"github.com/cloudflare/ebpf_exporter/v2/kallsyms"
+	lru "github.com/hashicorp/golang-lru/v2"
 )
 
 // ErrSkipLabelSet instructs exporter to skip label set
@@ -21,13 +22,14 @@ type Decoder interface {
 
 // Set is a set of Decoders that may be applied to produce a label
 type Set struct {
-	mu       sync.Mutex
-	decoders map[string]Decoder
-	cache    map[string]map[string][]string
+	mu        sync.Mutex
+	decoders  map[string]Decoder
+	cache     map[string]map[string][]string
+	skipCache *lru.Cache[string, struct{}]
 }
 
 // NewSet creates a Set with all known decoders
-func NewSet() (*Set, error) {
+func NewSet(skipCacheSize int) (*Set, error) {
 	cgroup, err := NewCgroupDecoder()
 	if err != nil {
 		return nil, fmt.Errorf("error creating cgroup decoder: %w", err)
@@ -38,7 +40,7 @@ func NewSet() (*Set, error) {
 		return nil, fmt.Errorf("error creating ksym decoder: %w", err)
 	}
 
-	return &Set{
+	s := &Set{
 		decoders: map[string]Decoder{
 			"cgroup":       cgroup,
 			"dname":        &Dname{},
@@ -60,7 +62,16 @@ func NewSet() (*Set, error) {
 			"uint":         &UInt{},
 		},
 		cache: map[string]map[string][]string{},
-	}, nil
+	}
+
+	if skipCacheSize > 0 {
+		skipCache, err := lru.New[string, struct{}](skipCacheSize)
+		if err != nil {
+			return nil, err
+		}
+		s.skipCache = skipCache
+	}
+	return s, nil
 }
 
 // decode transforms input byte field into a string according to configuration
@@ -75,6 +86,9 @@ func (s *Set) decode(in []byte, label config.Label) ([]byte, error) {
 		decoded, err := s.decoders[decoder.Name].Decode(result, decoder)
 		if err != nil {
 			if errors.Is(err, ErrSkipLabelSet) {
+				if s.skipCache != nil {
+					s.skipCache.Add(string(in), struct{}{})
+				}
 				return decoded, err
 			}
 
@@ -106,6 +120,14 @@ func (s *Set) DecodeLabelsForMetrics(in []byte, name string, labels []config.Lab
 		return cached, nil
 	}
 
+	// Also check the skip cache if the input would have return ErrSkipLabelSet
+	// and return the error early.
+	if s.skipCache != nil {
+		if _, ok := s.skipCache.Get(string(in)); ok {
+			return nil, ErrSkipLabelSet
+		}
+	}
+
 	values, err := s.decodeLabels(in, labels)
 	if err != nil {
 		return nil, err

decoder/decoder_test.go

@@ -1,6 +1,7 @@
 package decoder
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 	"testing"
@@ -148,7 +149,7 @@ func TestDecodeLabels(t *testing.T) {
 	}
 
 	for i, c := range cases {
-		s, err := NewSet()
+		s, err := NewSet(0)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -178,6 +179,118 @@ func TestDecodeLabels(t *testing.T) {
 	}
 }
 
+func TestDecodeSkipLabels(t *testing.T) {
+	cases := []struct {
+		in          []byte
+		skipCacheIn string
+		labels      []config.Label
+		out         []string
+		err         bool
+	}{
+		{
+			in:          append([]byte{0x8, 0x0, 0x0, 0x0}, zeroPaddedString("bananas", 32)...),
+			skipCacheIn: "",
+			labels: []config.Label{
+				{
+					Name: "number",
+					Size: 4,
+					Decoders: []config.Decoder{
+						{
+							Name: "uint",
+						},
+					},
+				},
+				{
+					Name: "fruit",
+					Size: 32,
+					Decoders: []config.Decoder{
+						{
+							Name: "string",
+						},
+						{
+							Name: "regexp",
+							Regexps: []string{
+								"^bananas$",
+								"$is-banana-even-fruit$",
+							},
+						},
+					},
+				},
+			},
+			out: []string{"8", "bananas"},
+			err: false,
+		},
+		{
+			in:          append([]byte{0x8, 0x0, 0x0, 0x0}, zeroPaddedString("bananas", 32)...),
+			skipCacheIn: string(zeroPaddedString("bananas", 32)),
+			labels: []config.Label{
+				{
+					Name: "number",
+					Size: 4,
+					Decoders: []config.Decoder{
+						{
+							Name: "uint",
+						},
+					},
+				},
+				{
+					Name: "fruit",
+					Size: 32,
+					Decoders: []config.Decoder{
+						{
+							Name: "string",
+						},
+						{
+							Name: "regexp",
+							Regexps: []string{
+								"^tomato$",
+							},
+						},
+					},
+				},
+			},
+			out: []string{"8", "bananas"},
+			err: true, // this label should be skipped, only tomatoes allowed
+		},
+	}
+
+	for i, c := range cases {
+		s, err := NewSet(100)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		out, err := s.DecodeLabelsForMetrics(c.in, fmt.Sprintf("test:%d", i), c.labels)
+		if c.err {
+			if err == nil {
+				t.Errorf("Expected error for input %#v and labels %#v, but did not receive it", c.in, c.labels)
+			}
+
+			if errors.Is(err, ErrSkipLabelSet) {
+				if !s.skipCache.Contains(c.skipCacheIn) {
+					t.Errorf("Expected skipCache to have input %#v", c.skipCacheIn)
+				}
+			}
+
+			continue
+		}
+
+		if err != nil {
+			t.Errorf("Error decoding %#v with labels set to %#v: %s", c.in, c.labels, err)
+		}
+
+		if len(c.out) != len(out) {
+			t.Errorf("Expected %d outputs (%v), received %d (%v)", len(c.out), c.out, len(out), out)
+		}
+
+		for i := 0; i < len(c.out) && i < len(out); i++ {
+			if c.out[i] != out[i] {
+				t.Errorf("Output label %d for input %#v is wrong: expected %s, but received %s", i, c.in, c.out[i], out[i])
+			}
+		}
+	}
+}
+
 func TestDecoderSetConcurrency(t *testing.T) {
 	in := append([]byte{0x8, 0x0, 0x0, 0x0}, zeroPaddedString("bananas", 32)...)
 
@@ -209,7 +322,7 @@ func TestDecoderSetConcurrency(t *testing.T) {
 		},
 	}
 
-	s, err := NewSet()
+	s, err := NewSet(0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -274,7 +387,7 @@ func TestDecoderSetCache(t *testing.T) {
 		},
 	}
 
-	s, err := NewSet()
+	s, err := NewSet(0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -345,7 +458,7 @@ func BenchmarkCache(b *testing.B) {
 		},
 	}
 
-	s, err := NewSet()
+	s, err := NewSet(0)
 	if err != nil {
 		b.Fatal(err)
 	}

exporter/exporter.go

@@ -56,7 +56,7 @@ type Exporter struct {
 }
 
 // New creates a new exporter with the provided config
-func New(configs []config.Config, tracingProvider tracing.Provider, btfPath string) (*Exporter, error) {
+func New(configs []config.Config, skipCacheSize int, tracingProvider tracing.Provider, btfPath string) (*Exporter, error) {
 	enabledConfigsDesc := prometheus.NewDesc(
 		prometheus.BuildFQName(prometheusNamespace, "", "enabled_configs"),
 		"The set of enabled configs",
@@ -101,7 +101,7 @@ func New(configs []config.Config, tracingProvider tracing.Provider, btfPath stri
 		decoderErrorCount.WithLabelValues(config.Name).Add(0.0)
 	}
 
-	decoders, err := decoder.NewSet()
+	decoders, err := decoder.NewSet(skipCacheSize)
 	if err != nil {
 		return nil, fmt.Errorf("error creating decoder set: %w", err)
 	}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/aquasecurity/libbpfgo v0.9.1-libbpf-1.5.1
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/iovisor/gobpf v0.2.0
 	github.com/jaypipes/pcidb v1.0.1
 	github.com/mdlayher/sdnotify v1.0.0

go.sum

@@ -32,6 +32,8 @@ github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrR
 github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/iovisor/gobpf v0.2.0 h1:34xkQxft+35GagXBk3n23eqhm0v7q0ejeVirb8sqEOQ=
 github.com/iovisor/gobpf v0.2.0/go.mod h1:WSY9Jj5RhdgC3ci1QaacvbFdQ8cbrEjrpiZbLHLt2s4=
 github.com/jaypipes/pcidb v1.0.1 h1:WB2zh27T3nwg8AE8ei81sNRb9yWBii3JGNJtT7K9Oic=

tracing/extract_test.go

@@ -151,7 +151,7 @@ func TestExtractFilled(t *testing.T) {
 		},
 	}
 
-	decoders, err := decoder.NewSet()
+	decoders, err := decoder.NewSet(0)
 	if err != nil {
 		t.Fatalf("Error creating decoders set: %v", err)
 	}