Skip to content

Commit d0529b9

Browse files
authored
Merge pull request #44 from bobrik/program-tags
Add ebpf_exporter_ebpf_programs with program tags
2 parents cf6b8fc + 597ad89 commit d0529b9

File tree

8 files changed

+149
-61
lines changed

8 files changed

+149
-61
lines changed

README.md

+35
Original file line numberDiff line numberDiff line change
@@ -772,6 +772,41 @@ name: <decoder name>
772772
# ... decoder specific configuration
773773
```
774774
775+
## Built-in metrics
776+
777+
### `ebpf_exporter_enabled_programs`
778+
779+
This gauge reports a timeseries for every loaded logical program:
780+
781+
```
782+
# HELP ebpf_exporter_enabled_programs The set of enabled programs
783+
# TYPE ebpf_exporter_enabled_programs gauge
784+
ebpf_exporter_enabled_programs{name="xfs_reclaim"} 1
785+
```
786+
787+
### `ebpf_exporter_ebpf_programs`
788+
789+
This gauge reports information available for every ebpf program:
790+
791+
```
792+
# HELP ebpf_exporter_ebpf_programs Info about ebpf programs
793+
# TYPE ebpf_exporter_ebpf_programs gauge
794+
ebpf_exporter_ebpf_programs{function="xfs_fs_free_cached_objects_end",program="xfs_reclaim",tag="d5e845dc27b372e4"} 1
795+
ebpf_exporter_ebpf_programs{function="xfs_fs_free_cached_objects_start",program="xfs_reclaim",tag="c2439d02dd0ba000"} 1
796+
ebpf_exporter_ebpf_programs{function="xfs_fs_nr_cached_objects_end",program="xfs_reclaim",tag="598375893f34ef39"} 1
797+
ebpf_exporter_ebpf_programs{function="xfs_fs_nr_cached_objects_start",program="xfs_reclaim",tag="cf30348184f983dd"} 1
798+
```
799+
800+
Here `tag` can be used for tracing and performance analysis with two conditions:
801+
802+
* `net.core.bpf_jit_kallsyms=1` sysctl is set
803+
* `--kallsyms=/proc/kallsyms` is passed to `perf record`
804+
805+
Newer kernels allow `--kallsyms` to `perf top` as well,
806+
in the future it may not be required at all:
807+
808+
* https://www.spinics.net/lists/linux-perf-users/msg07216.html
809+
775810
## License
776811
777812
MIT

benchmark/README.md

+5
Original file line numberDiff line numberDiff line change
@@ -80,3 +80,8 @@ Tracing 1 functions for "sys_getpid"... Hit Ctrl-C to end.
8080
These measurements already include some overhead from `funclatency` itself.
8181

8282
You can see frequency of calls in the output as well.
83+
84+
## Performance analysis with perf
85+
86+
See [`ebpf_exporter_ebpf_programs`](../README.md#ebpf_exporter_ebpf_programs)
87+
for more information on runtime analysis of performance impact.

exporter/attach.go

+62-58
Original file line numberDiff line numberDiff line change
@@ -6,91 +6,95 @@ import (
66
"github.com/iovisor/gobpf/bcc"
77
)
88

9-
// attach attaches functions to tracing points in provided module
10-
func attach(module *bcc.Module, kprobes, kretprobes, tracepoints, rawTracepoints map[string]string) error {
11-
if err := attachKprobes(module, kprobes); err != nil {
12-
return fmt.Errorf("failed to attach kprobes: %s", err)
13-
}
9+
// attacher attaches some sort of tracepoints or probes
10+
type attacher func(*bcc.Module, map[string]string) (map[string]uint64, error)
1411

15-
if err := attachKretprobes(module, kretprobes); err != nil {
16-
return fmt.Errorf("failed to attach kretprobes: %s", err)
17-
}
12+
// mergeTags runs attacher and merges produced tags
13+
func mergedTags(dst map[string]uint64, attach attacher, module *bcc.Module, attachments map[string]string) error {
14+
src, err := attach(module, attachments)
1815

19-
if err := attachTracepoints(module, tracepoints); err != nil {
20-
return fmt.Errorf("failed to attach tracepoints: %s", err)
16+
if err != nil {
17+
return err
2118
}
2219

23-
if err := attachRawTracepoints(module, rawTracepoints); err != nil {
24-
return fmt.Errorf("failed to attach raw tracepoints: %s", err)
20+
for name, tag := range src {
21+
dst[name] = tag
2522
}
2623

2724
return nil
2825
}
2926

30-
// attachKprobes attaches functions to their kprobles in provided module
31-
func attachKprobes(module *bcc.Module, kprobes map[string]string) error {
32-
for kprobeName, targetName := range kprobes {
33-
target, err := module.LoadKprobe(targetName)
34-
if err != nil {
35-
return fmt.Errorf("failed to load target %q: %s", targetName, err)
36-
}
27+
// attach attaches functions to tracing points in provided module
28+
func attach(module *bcc.Module, kprobes, kretprobes, tracepoints, rawTracepoints map[string]string) (map[string]uint64, error) {
29+
tags := map[string]uint64{}
3730

38-
err = module.AttachKprobe(kprobeName, target)
39-
if err != nil {
40-
return fmt.Errorf("failed to attach kprobe %q to %q: %s", kprobeName, targetName, err)
41-
}
31+
if err := mergedTags(tags, attachKprobes, module, kprobes); err != nil {
32+
return nil, fmt.Errorf("failed to attach kprobes: %s", err)
4233
}
4334

44-
return nil
45-
}
35+
if err := mergedTags(tags, attachKretprobes, module, kretprobes); err != nil {
36+
return nil, fmt.Errorf("failed to attach kretprobes: %s", err)
37+
}
4638

47-
// attachKretprobes attaches functions to their kretprobles in provided module
48-
func attachKretprobes(module *bcc.Module, kretprobes map[string]string) error {
49-
for kretprobeName, targetName := range kretprobes {
50-
target, err := module.LoadKprobe(targetName)
51-
if err != nil {
52-
return fmt.Errorf("failed to load target %q: %s", targetName, err)
53-
}
39+
if err := mergedTags(tags, attachTracepoints, module, tracepoints); err != nil {
40+
return nil, fmt.Errorf("failed to attach tracepoints: %s", err)
41+
}
5442

55-
err = module.AttachKretprobe(kretprobeName, target)
56-
if err != nil {
57-
return fmt.Errorf("failed to attach kretprobe %q to %q: %s", kretprobeName, targetName, err)
58-
}
43+
if err := mergedTags(tags, attachRawTracepoints, module, rawTracepoints); err != nil {
44+
return nil, fmt.Errorf("failed to attach raw tracepoints: %s", err)
5945
}
6046

61-
return nil
47+
return tags, nil
6248
}
6349

64-
// attachTracepoints attaches functions to their tracepoints in provided module
65-
func attachTracepoints(module *bcc.Module, tracepoints map[string]string) error {
66-
for tracepointName, targetName := range tracepoints {
67-
target, err := module.LoadTracepoint(targetName)
68-
if err != nil {
69-
return fmt.Errorf("failed to load target %q: %s", targetName, err)
70-
}
50+
// probeLoader attaches some sort of probe
51+
type probeLoader func(string) (int, error)
7152

72-
err = module.AttachTracepoint(tracepointName, target)
53+
// probeAttacher attaches loaded some sort of probe to some sort of tracepoint
54+
type probeAttacher func(string, int) error
55+
56+
// attachSomething attaches some kind of probes and returns program tags
57+
func attachSomething(module *bcc.Module, loader probeLoader, attacher probeAttacher, probes map[string]string) (map[string]uint64, error) {
58+
tags := map[string]uint64{}
59+
60+
for probe, targetName := range probes {
61+
target, err := loader(targetName)
7362
if err != nil {
74-
return fmt.Errorf("failed to attach tracepoint %q to %q: %s", tracepointName, targetName, err)
63+
return nil, fmt.Errorf("failed to load probe %q: %s", targetName, err)
7564
}
76-
}
77-
78-
return nil
79-
}
8065

81-
// attachRawTracepoints attaches functions to their tracepoints in provided module
82-
func attachRawTracepoints(module *bcc.Module, tracepoints map[string]string) error {
83-
for tracepointName, targetName := range tracepoints {
84-
target, err := module.LoadRawTracepoint(targetName)
66+
tag, err := module.GetProgramTag(target)
8567
if err != nil {
86-
return fmt.Errorf("failed to load target %q: %s", targetName, err)
68+
return nil, fmt.Errorf("failed to get program tag for %q (fd=%d): %s", targetName, target, err)
8769
}
8870

89-
err = module.AttachRawTracepoint(tracepointName, target)
71+
tags[targetName] = tag
72+
73+
err = attacher(probe, target)
9074
if err != nil {
91-
return fmt.Errorf("failed to attach raw tracepoint %q to %q: %s", tracepointName, targetName, err)
75+
return nil, fmt.Errorf("failed to attach probe %q to %q: %s", probe, targetName, err)
9276
}
9377
}
9478

95-
return nil
79+
return tags, nil
80+
}
81+
82+
// attachKprobes attaches functions to their kprobles in provided module
83+
func attachKprobes(module *bcc.Module, kprobes map[string]string) (map[string]uint64, error) {
84+
return attachSomething(module, module.LoadKprobe, module.AttachKprobe, kprobes)
85+
}
86+
87+
// attachKretprobes attaches functions to their kretprobles in provided module
88+
func attachKretprobes(module *bcc.Module, kretprobes map[string]string) (map[string]uint64, error) {
89+
return attachSomething(module, module.LoadKprobe, module.AttachKretprobe, kretprobes)
90+
}
91+
92+
// attachTracepoints attaches functions to their tracepoints in provided module
93+
func attachTracepoints(module *bcc.Module, tracepoints map[string]string) (map[string]uint64, error) {
94+
return attachSomething(module, module.LoadTracepoint, module.AttachTracepoint, tracepoints)
95+
}
96+
97+
// attachRawTracepoints attaches functions to their tracepoints in provided module
98+
func attachRawTracepoints(module *bcc.Module, tracepoints map[string]string) (map[string]uint64, error) {
99+
return attachSomething(module, module.LoadRawTracepoint, module.AttachRawTracepoint, tracepoints)
96100
}

exporter/exporter.go

+23-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ type Exporter struct {
2121
modules map[string]*bcc.Module
2222
ksyms map[uint64]string
2323
enabledProgramsDesc *prometheus.Desc
24+
programInfoDesc *prometheus.Desc
25+
programTags map[string]map[string]uint64
2426
descs map[string]map[string]*prometheus.Desc
2527
decoders *decoder.Set
2628
}
@@ -34,11 +36,20 @@ func New(config config.Config) *Exporter {
3436
nil,
3537
)
3638

39+
programInfoDesc := prometheus.NewDesc(
40+
prometheus.BuildFQName(prometheusNamespace, "", "ebpf_programs"),
41+
"Info about ebpf programs",
42+
[]string{"program", "function", "tag"},
43+
nil,
44+
)
45+
3746
return &Exporter{
3847
config: config,
3948
modules: map[string]*bcc.Module{},
4049
ksyms: map[uint64]string{},
4150
enabledProgramsDesc: enabledProgramsDesc,
51+
programInfoDesc: programInfoDesc,
52+
programTags: map[string]map[string]uint64{},
4253
descs: map[string]map[string]*prometheus.Desc{},
4354
decoders: decoder.NewSet(),
4455
}
@@ -56,10 +67,14 @@ func (e *Exporter) Attach() error {
5667
return fmt.Errorf("error compiling module for program %q", program.Name)
5768
}
5869

59-
if err := attach(module, program.Kprobes, program.Kretprobes, program.Tracepoints, program.RawTracepoints); err != nil {
70+
tags, err := attach(module, program.Kprobes, program.Kretprobes, program.Tracepoints, program.RawTracepoints)
71+
72+
if err != nil {
6073
return fmt.Errorf("failed to attach to program %q: %s", program.Name, err)
6174
}
6275

76+
e.programTags[program.Name] = tags
77+
6378
for _, perfEventConfig := range program.PerfEvents {
6479
target, err := module.LoadPerfEvent(perfEventConfig.Target)
6580
if err != nil {
@@ -96,6 +111,7 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
96111
}
97112

98113
ch <- e.enabledProgramsDesc
114+
ch <- e.programInfoDesc
99115

100116
for _, program := range e.config.Programs {
101117
if _, ok := e.descs[program.Name]; !ok {
@@ -118,6 +134,12 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
118134
ch <- prometheus.MustNewConstMetric(e.enabledProgramsDesc, prometheus.GaugeValue, 1, program.Name)
119135
}
120136

137+
for program, tags := range e.programTags {
138+
for function, tag := range tags {
139+
ch <- prometheus.MustNewConstMetric(e.programInfoDesc, prometheus.GaugeValue, 1, program, function, fmt.Sprintf("%x", tag))
140+
}
141+
}
142+
121143
e.collectCounters(ch)
122144
e.collectHistograms(ch)
123145
}

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
77
github.com/davecgh/go-spew v1.1.1 // indirect
88
github.com/golang/protobuf v1.2.0 // indirect
9-
github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31
9+
github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280
1010
github.com/kr/pretty v0.1.0 // indirect
1111
github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect
1212
github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
1010
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
1111
github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31 h1:1aM9orF3Ap9PACtNbi+PLEFpyowpy0Xz9U/mLh28m+E=
1212
github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31/go.mod h1:+5U5qu5UOu8YJ5oHVLvWKH7/Dr5QNHU7mZ2RfPEeXg8=
13+
github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280 h1:/s/gVPSoThcIBrunQtZNgTZJzSPWuI2ASMDdiRQio5g=
14+
github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280/go.mod h1:+5U5qu5UOu8YJ5oHVLvWKH7/Dr5QNHU7mZ2RfPEeXg8=
1315
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
1416
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
1517
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

vendor/github.com/iovisor/gobpf/bcc/module.go

+20
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vendor/modules.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ github.com/alecthomas/units
77
github.com/beorn7/perks/quantile
88
# github.com/golang/protobuf v1.2.0
99
github.com/golang/protobuf/proto
10-
# github.com/iovisor/gobpf v0.0.0-20181115102816-c5918016dd31
10+
# github.com/iovisor/gobpf v0.0.0-20190101180936-b0976764b280
1111
github.com/iovisor/gobpf/bcc
1212
github.com/iovisor/gobpf/pkg/ksym
1313
github.com/iovisor/gobpf/pkg/cpuonline

0 commit comments

Comments
 (0)