Skip to content

Commit e97a6da

Browse files
AsphalttAsphaltt
committed
pwru: Support --output-skb-metadata
It's really useful to output specified metadatas of skb instead of outputing the whole skb btf data. However, it's not easy to `--output-skb-metadata 'skb->mark'` because the metadata is provided by users. Here're the steps to achieve it: 1. Parse the text to C AST. 2. Convert the C AST to bpf insns with `skb` btf type info. 3. Inject the `set_skb_metadata` stub with the generated insns. 4. Output the metadata with the provided field's btf type info. e.g. ```bash $ sudo ./pwru --filter-func 'icmp_rcv' --output-skb-metadata 'skb->hash' --output-skb-metadata 'skb->sw_hash' --output-skb-metadata 'skb->l4_hash' --output-skb-metadata 'skb->mark' host 1.1.1.1 2025/03/19 16:27:18 Attaching kprobes (via kprobe-multi)... 1 / 1 [----------------------------------------------------------------------------------------------------------------------------------------] 100.00% ? p/s 2025/03/19 16:27:18 Attached (ignored 0) 2025/03/19 16:27:18 Listening for events.. SKB CPU PROCESS NETNS MARK/x IFACE PROTO MTU LEN TUPLE FUNC 0xffff991f3ec49600 6 <empty>:0 4026531840 0 ens33:2 0x0800 65536 64 1.1.1.1:0->192.168.241.133:0(icmp) icmp_rcv hash=0 sw_hash=0 l4_hash=0 mark=0 ^C2025/03/19 16:32:56 Received signal, exiting program.. 2025/03/19 16:32:56 Detaching kprobes... 1 / 1 [----------------------------------------------------------------------------------------------------------------------------------------] 100.00% ? p/s ``` Signed-off-by: Leon Hwang <[email protected]>
1 parent 2d1076e commit e97a6da

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+9651
-4
lines changed

‎bpf/kprobe_pwru.c

+14-1
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ struct event_t {
8989
u64 param_second;
9090
u64 param_third;
9191
u32 cpu_id;
92+
u64 skb_metadata[4]; /* output up-to-4 metadata */
9293
} __attribute__((packed));
9394

9495
#define MAX_QUEUE_ENTRIES 10000
@@ -146,7 +147,7 @@ struct config {
146147
u8 output_stack: 1;
147148
u8 output_caller: 1;
148149
u8 output_cb: 1;
149-
u8 output_unused: 1;
150+
u8 output_skb_metadata: 1;
150151
u8 is_set: 1;
151152
u8 track_skb: 1;
152153
u8 track_skb_by_stackid: 1;
@@ -271,6 +272,12 @@ filter(struct sk_buff *skb) {
271272
return filter_pcap(skb) && filter_meta(skb);
272273
}
273274

275+
static __noinline void
276+
set_skb_metadata(struct sk_buff *skb, u64 *metadata) {
277+
/* This func will be rewrote by Go. */
278+
metadata[0] = (u64)(void *) skb;
279+
}
280+
274281
static __always_inline void
275282
set_meta(struct sk_buff *skb, struct skb_meta *meta) {
276283
meta->netns = get_netns(skb);
@@ -452,6 +459,12 @@ set_output(void *ctx, struct sk_buff *skb, struct event_t *event) {
452459
struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)&skb->cb;
453460
bpf_probe_read_kernel(&event->meta.cb, sizeof(event->meta.cb), (void *)&cb->data);
454461
}
462+
463+
if (cfg->output_skb_metadata) {
464+
u64 data[4];
465+
set_skb_metadata(skb, data);
466+
__builtin_memcpy(event->skb_metadata, data, sizeof(data));
467+
}
455468
}
456469

457470
static __noinline bool

‎go.mod

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
module github.com/cilium/pwru
22

3-
go 1.23.1
3+
go 1.23.4
4+
5+
toolchain go1.24.0
46

57
require (
8+
github.com/Asphaltt/mybtf v0.0.0-20250315135407-f9d09086616b
69
github.com/cheggaaa/pb/v3 v3.1.6
710
github.com/cilium/ebpf v0.17.3
811
github.com/cloudflare/cbpfc v0.0.0-20221017140110-11acb56438a2
912
github.com/jsimonetti/rtnetlink v1.4.2
13+
github.com/leonhwangprojects/bice v0.1.1
1014
github.com/spf13/pflag v1.0.6
1115
github.com/tklauser/ps v0.0.3
1216
github.com/vishvananda/netns v0.0.5
@@ -28,4 +32,5 @@ require (
2832
github.com/mdlayher/socket v0.4.1 // indirect
2933
github.com/pkg/errors v0.9.1 // indirect
3034
github.com/rivo/uniseg v0.2.0 // indirect
35+
rsc.io/c2go v0.0.0-20170620140410-520c22818a08 // indirect
3136
)

‎go.sum

+6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/Asphaltt/mybtf v0.0.0-20250315135407-f9d09086616b h1:LRS0ckDlNnX2Bux8k0HqLVf/2PqvvykoB8HSxXf7XjA=
2+
github.com/Asphaltt/mybtf v0.0.0-20250315135407-f9d09086616b/go.mod h1:ZxswciFofS8TXGsc2j5CWCY8O3XwKEysCnU7hoI1fnI=
13
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
24
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
35
github.com/cheggaaa/pb/v3 v3.1.6 h1:h0x+vd7EiUohAJ29DJtJy+SNAc55t/elW3jCD086EXk=
@@ -30,6 +32,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
3032
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
3133
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
3234
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
35+
github.com/leonhwangprojects/bice v0.1.1 h1:gHJebJcWS8xRxJtiv3ko+HI34PRtlBpY5QlwFqgLLYg=
36+
github.com/leonhwangprojects/bice v0.1.1/go.mod h1:6Lsun8A3o0pPfjuCfpUR71WBuOLO64oQhXWdHdlW4SE=
3337
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
3438
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
3539
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -73,3 +77,5 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
7377
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
7478
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
7579
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
80+
rsc.io/c2go v0.0.0-20170620140410-520c22818a08 h1:AAIN5uzUq20OU2cNoPTVljUm7JDaD0Z/+Xl/uMBHJgU=
81+
rsc.io/c2go v0.0.0-20170620140410-520c22818a08/go.mod h1:hE73EMCiJ3lIDPgEDEK781LZrVhFDxT+I+ziQRoLVNE=

‎internal/pwru/config.go

+4
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ const (
2323
OutputStackMask
2424
OutputCallerMask
2525
OutputCbMask
26+
OutputSkbMetadataMask
2627
)
2728

2829
const (
@@ -72,6 +73,9 @@ func GetConfig(flags *Flags) (cfg FilterCfg, err error) {
7273
if flags.OutputCaller {
7374
cfg.OutputFlags |= OutputCallerMask
7475
}
76+
if len(flags.OutputSkbMetadata) > 0 {
77+
cfg.OutputFlags |= OutputSkbMetadataMask
78+
}
7579
if flags.FilterTraceTc || flags.OutputSkbCB {
7680
cfg.OutputFlags |= OutputCbMask
7781
}

‎internal/pwru/output.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ type output struct {
4444
printShinfoMap *ebpf.Map
4545
printStackMap *ebpf.Map
4646
addr2name Addr2Name
47+
skbMetadata []*SkbMetadata
4748
writer *os.File
4849
kprobeMulti bool
4950
kfreeReasons map[uint64]string
@@ -89,7 +90,7 @@ func centerAlignString(s string, width int) string {
8990
return fmt.Sprintf("%s%s%s", strings.Repeat(" ", leftPadding), s, strings.Repeat(" ", rightPadding))
9091
}
9192

92-
func NewOutput(flags *Flags, printSkbMap, printShinfoMap, printStackMap *ebpf.Map, addr2Name Addr2Name, kprobeMulti bool, btfSpec *btf.Spec) (*output, error) {
93+
func NewOutput(flags *Flags, printSkbMap, printShinfoMap, printStackMap *ebpf.Map, addr2Name Addr2Name, mds []*SkbMetadata, kprobeMulti bool, btfSpec *btf.Spec) (*output, error) {
9394
writer := os.Stdout
9495

9596
if flags.OutputFile != "" {
@@ -120,6 +121,7 @@ func NewOutput(flags *Flags, printSkbMap, printShinfoMap, printStackMap *ebpf.Ma
120121
printShinfoMap: printShinfoMap,
121122
printStackMap: printStackMap,
122123
addr2name: addr2Name,
124+
skbMetadata: mds,
123125
writer: writer,
124126
kprobeMulti: kprobeMulti,
125127
kfreeReasons: reasons,
@@ -450,6 +452,10 @@ func (o *output) Print(event *Event) {
450452
fmt.Fprintf(o.writer, " %s", outFuncName)
451453
}
452454

455+
if len(o.skbMetadata) != 0 {
456+
outputSkbMetadata(o.writer, o.skbMetadata, event.SkbMetadata[:])
457+
}
458+
453459
if o.flags.OutputStack && event.PrintStackId > 0 {
454460
fmt.Fprintf(o.writer, "%s", getStackData(event, o))
455461
}

‎internal/pwru/skb_metadata.go

+191
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
/* Copyright Leon Hwang */
3+
/* Copyright Authors of Cilium */
4+
5+
package pwru
6+
7+
import (
8+
"encoding/binary"
9+
"errors"
10+
"fmt"
11+
"io"
12+
"strings"
13+
14+
"github.com/Asphaltt/mybtf"
15+
"github.com/cilium/ebpf"
16+
"github.com/cilium/ebpf/asm"
17+
"github.com/cilium/ebpf/btf"
18+
"github.com/leonhwangprojects/bice"
19+
)
20+
21+
const labelSetSkbMetadataExit = "__set_skb_metadata_exit"
22+
23+
const setSkbMetadataStub = "set_skb_metadata"
24+
25+
const maxSkbMetadata = 4
26+
27+
type SkbMetadata struct {
28+
expr string
29+
last string
30+
t btf.Type // type of last field
31+
insn asm.Instructions
32+
}
33+
34+
func isChar(c byte) bool {
35+
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
36+
}
37+
38+
func isDigit(c byte) bool {
39+
return c >= '0' && c <= '9'
40+
}
41+
42+
func isValidChar(c byte) bool {
43+
return isChar(c) || c == '_' || isDigit(c)
44+
}
45+
46+
func newSkbMetadata(expr string, spec *btf.Spec) (*SkbMetadata, error) {
47+
var md SkbMetadata
48+
md.expr = expr
49+
50+
if !strings.HasPrefix(expr, "skb") {
51+
return nil, errors.New("--output-skb-metadata must starts with skb")
52+
}
53+
54+
for i := len(expr) - 1; i >= 0; i-- {
55+
if !isValidChar(expr[i]) {
56+
md.last = expr[i+1:]
57+
break
58+
}
59+
}
60+
61+
types, err := spec.AnyTypesByName("sk_buff")
62+
if err != nil {
63+
return nil, err
64+
}
65+
66+
skb := types[0]
67+
ptr := &btf.Pointer{Target: skb}
68+
69+
res, err := bice.Access(bice.AccessOptions{
70+
Expr: expr,
71+
Type: ptr,
72+
Src: asm.R1,
73+
Dst: asm.R3,
74+
LabelExit: labelSetSkbMetadataExit,
75+
})
76+
if err != nil {
77+
return nil, fmt.Errorf("failed to parse expr %s: %w", expr, err)
78+
}
79+
80+
size, err := btf.Sizeof(res.LastField)
81+
if err != nil {
82+
return nil, err
83+
}
84+
if size > 8 {
85+
return nil, fmt.Errorf("skb metadata field '%s' (type %v) is too large, max 8 bytes", md.last, res.LastField)
86+
}
87+
88+
md.t = res.LastField
89+
md.insn = res.Insns
90+
return &md, nil
91+
}
92+
93+
func ParseSkbMetadataExprs(exprs []string, spec *btf.Spec) ([]*SkbMetadata, error) {
94+
var md []*SkbMetadata
95+
96+
for _, e := range exprs {
97+
if e == "" {
98+
continue
99+
}
100+
101+
m, err := newSkbMetadata(e, spec)
102+
if err != nil {
103+
return nil, fmt.Errorf("failed to parse skb metadata expr %s: %w", e, err)
104+
}
105+
106+
md = append(md, m)
107+
}
108+
109+
if len(md) > maxSkbMetadata {
110+
return nil, fmt.Errorf("too many skb metadata exprs, max %d", maxSkbMetadata)
111+
}
112+
113+
return md, nil
114+
}
115+
116+
func InjectSetSkbMetadata(prog *ebpf.ProgramSpec, md []*SkbMetadata) error {
117+
if len(md) == 0 {
118+
return nil
119+
}
120+
121+
var insns asm.Instructions
122+
if len(md) > 1 {
123+
insns = append(insns,
124+
asm.Mov.Reg(asm.R7, asm.R1), // r7 = skb
125+
)
126+
}
127+
insns = append(insns,
128+
asm.Mov.Reg(asm.R6, asm.R2), // r6 = metadata
129+
)
130+
131+
insns = append(insns, md[0].insn...)
132+
insns = append(insns, asm.StoreMem(asm.R6, 0, asm.R3, asm.DWord)) // metadata[0] = skb->field
133+
134+
offset := 0
135+
for _, m := range md[1:] {
136+
offset += 8
137+
insns = append(insns,
138+
asm.Mov.Reg(asm.R1, asm.R7), // r1 = skb
139+
)
140+
insns = append(insns, m.insn...)
141+
insns = append(insns, asm.StoreMem(asm.R6, int16(offset), asm.R3, asm.DWord)) // metadata[i] = skb->field
142+
}
143+
144+
insns = append(insns,
145+
asm.Return().WithSymbol(labelSetSkbMetadataExit),
146+
)
147+
148+
startIdx := -1
149+
for i, ins := range prog.Instructions {
150+
if ins.Symbol() == setSkbMetadataStub {
151+
startIdx = i
152+
break
153+
}
154+
}
155+
if startIdx == -1 {
156+
return errors.New("failed to find set_skb_metadata stub")
157+
}
158+
159+
retIdx := -1
160+
retOpCode := asm.Return().OpCode
161+
for i := startIdx + 1; i < len(prog.Instructions); i++ {
162+
if prog.Instructions[i].OpCode == retOpCode {
163+
retIdx = i
164+
break
165+
}
166+
}
167+
if retIdx == -1 {
168+
return errors.New("failed to find ret instruction")
169+
}
170+
171+
// replace the stub's instructions
172+
insns[0] = insns[0].WithMetadata(prog.Instructions[startIdx].Metadata)
173+
prog.Instructions = append(prog.Instructions[:startIdx],
174+
append(insns, prog.Instructions[retIdx+1:]...)...)
175+
176+
return nil
177+
}
178+
179+
func outputSkbMetadata(w io.Writer, md []*SkbMetadata, data []uint64) {
180+
for i, m := range md {
181+
var b [8]byte
182+
binary.NativeEndian.PutUint64(b[:], data[i])
183+
184+
s, err := mybtf.DumpData(m.t, b[:])
185+
if err != nil {
186+
fmt.Fprintf(w, " %s=..ERR..", m.last)
187+
} else {
188+
fmt.Fprintf(w, " %s=%s", m.last, s)
189+
}
190+
}
191+
}

‎internal/pwru/types.go

+4
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ type Flags struct {
5353
OutputJson bool
5454
OutputTCPFlags bool
5555

56+
OutputSkbMetadata []string
57+
5658
KMods []string
5759
AllKMods bool
5860

@@ -88,6 +90,7 @@ func (f *Flags) SetFlags() {
8890
flag.Uint64Var(&f.OutputLimitLines, "output-limit-lines", 0, "exit the program after the number of events has been received/printed")
8991
flag.BoolVar(&f.OutputSkbCB, "output-skb-cb", false, "print skb->cb")
9092
flag.BoolVar(&f.OutputTCPFlags, "output-tcp-flags", false, "print TCP flags")
93+
flag.StringSliceVar(&f.OutputSkbMetadata, "output-skb-metadata", nil, "print skb metadata (e.g., \"skb->mark\", \"skb->hash\")")
9194

9295
flag.StringVar(&f.OutputFile, "output-file", "", "write traces to file")
9396

@@ -182,6 +185,7 @@ type Event struct {
182185
ParamSecond uint64
183186
ParamThird uint64
184187
CPU uint32
188+
SkbMetadata [4]uint64
185189
}
186190

187191
type markFlagValue struct {

‎main.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,12 @@ func main() {
133133
}
134134
}
135135

136+
// --output-skb-metadata
137+
skbMds, err := pwru.ParseSkbMetadataExprs(flags.OutputSkbMetadata, btfSpec)
138+
if err != nil {
139+
log.Fatalf("Failed to parse skb metadata exprs: %s", err)
140+
}
141+
136142
for name, program := range bpfSpec.Programs {
137143
// Skip the skb-tracking ones that should not inject pcap-filter.
138144
switch name {
@@ -153,6 +159,9 @@ func main() {
153159
if err = libpcap.InjectFilters(program, flags.FilterPcap); err != nil {
154160
log.Fatalf("Failed to inject filter ebpf for %s: %v", name, err)
155161
}
162+
if err := pwru.InjectSetSkbMetadata(program, skbMds); err != nil {
163+
log.Fatalf("Failed to inject skb metadata: %v", err)
164+
}
156165
}
157166

158167
skbBtfID, err := pwru.GetStructBtfID(btfSpec, "sk_buff")
@@ -284,7 +293,7 @@ func main() {
284293
printSkbMap := coll.Maps["print_skb_map"]
285294
printShinfoMap := coll.Maps["print_shinfo_map"]
286295
printStackMap := coll.Maps["print_stack_map"]
287-
output, err := pwru.NewOutput(&flags, printSkbMap, printShinfoMap, printStackMap, addr2name, useKprobeMulti, btfSpec)
296+
output, err := pwru.NewOutput(&flags, printSkbMap, printShinfoMap, printStackMap, addr2name, skbMds, useKprobeMulti, btfSpec)
288297
if err != nil {
289298
log.Fatalf("Failed to create outputer: %s", err)
290299
}

0 commit comments

Comments
 (0)