[WP] Add metric for dropped packets

Author: theop-dd

pkg/security/ebpf/c/include/maps.h

@@ -98,6 +98,7 @@ BPF_LRU_MAP(dns_responses_sent_to_userspace, u16, struct dns_responses_sent_to_u
 BPF_LRU_MAP(capabilities_usage, struct capabilities_usage_key_t, struct capabilities_usage_entry_t, 1) // max entries will be overridden at runtime
 BPF_LRU_MAP(sock_cookie_pid, u64, u32, 1); // max entries will be overridden at runtime
 BPF_LRU_MAP(memfd_tracking, struct memfd_key_t, u32, 1024)
+BPF_LRU_MAP(dropped_packets, u64, u64, 512)
 
 BPF_LRU_MAP_FLAGS(tasks_in_coredump, u64, u8, 64, BPF_F_NO_COMMON_LRU)
 BPF_LRU_MAP_FLAGS(syscalls, u64, struct syscall_cache_t, 1, BPF_F_NO_COMMON_LRU) // max entries will be overridden at runtime

pkg/security/ebpf/probes/rawpacket/bpffilter.go

@@ -37,6 +37,9 @@ const (
 	PolicyDrop
 )
 
+// MaxDropActionFilters is the maximum number of network drop action filters tracked in kernel.
+const MaxDropActionFilters = 512
+
 // ToTCAct converts a policy to a TCAct
 func (p Policy) ToTCAct() TCAct {
 	switch p {

pkg/security/ebpf/probes/rawpacket/pcap.go

@@ -11,6 +11,7 @@ package rawpacket
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/asm"
@@ -39,6 +40,9 @@ const (
 
 	// payload size
 	structRawPacketEventDataSize = 256
+
+	dropStatsKeyStackOffset = int16(-8)
+	dropStatsValStackOffset = int16(-16)
 )
 
 // ProgOpts defines options
@@ -60,6 +64,7 @@ type ProgOpts struct {
 	ctxSaveReg            asm.Register
 	tailCallMapFd         int
 	hasGetCurrentCgroupId bool
+	dropStatsMapFd        int
 }
 
 // DefaultProgOpts default options
@@ -97,6 +102,52 @@ func (opts *ProgOpts) WithGetCurrentCgroupID(hasGetCurrentCgroupId bool) *ProgOp
 	return opts
 }
 
+// WithDropStatsMapFd sets the map fd used to count dropped packets per filter index.
+func (opts *ProgOpts) WithDropStatsMapFd(fd int) *ProgOpts {
+	opts.dropStatsMapFd = fd
+	return opts
+}
+
+func dropStatsIncrementInsts(filterIndex int, dropStatsMapFd int, nextLabel string) asm.Instructions {
+	incLabel := fmt.Sprintf("inc_drop_stat_%d", filterIndex)
+	initLabel := fmt.Sprintf("init_drop_stat_%d", filterIndex)
+
+	return asm.Instructions{
+		asm.Mov.Reg(asm.R1, asm.RFP).WithSymbol(incLabel),
+		asm.Add.Imm(asm.R1, int32(dropStatsKeyStackOffset)),
+		asm.Mov.Imm(asm.R2, int32(filterIndex)),
+		asm.StoreMem(asm.R1, 0, asm.R2, asm.DWord),
+
+		asm.LoadMapPtr(asm.R1, dropStatsMapFd),
+		asm.Mov.Reg(asm.R2, asm.RFP),
+		asm.Add.Imm(asm.R2, int32(dropStatsKeyStackOffset)),
+		asm.FnMapLookupElem.Call(),
+		asm.JEq.Imm(asm.R0, 0, initLabel),
+
+		asm.Mov.Reg(asm.R5, asm.R0),
+		asm.LoadMem(asm.R6, asm.R5, 0, asm.DWord),
+		asm.Add.Imm(asm.R6, 1),
+		asm.StoreMem(asm.R5, 0, asm.R6, asm.DWord),
+
+		asm.Ja.Label(nextLabel),
+
+		asm.Mov.Reg(asm.R3, asm.RFP).WithSymbol(initLabel),
+		asm.Add.Imm(asm.R3, int32(dropStatsValStackOffset)),
+		asm.Mov.Imm(asm.R4, 1),
+		asm.StoreMem(asm.R3, 0, asm.R4, asm.DWord),
+
+		asm.LoadMapPtr(asm.R1, dropStatsMapFd),
+		asm.Mov.Reg(asm.R2, asm.RFP),
+		asm.Add.Imm(asm.R2, int32(dropStatsKeyStackOffset)),
+		asm.Mov.Reg(asm.R3, asm.RFP),
+		asm.Add.Imm(asm.R3, int32(dropStatsValStackOffset)),
+		asm.Mov.Imm(asm.R4, 0),
+		asm.FnMapUpdateElem.Call(),
+
+		asm.Ja.Label(nextLabel),
+	}
+}
+
 // FilterToInsts compile a bpf filter expression
 func FilterToInsts(index int, filter Filter, opts ProgOpts) (asm.Instructions, error) {
 	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, 256, filter.BPFFilter)
@@ -131,25 +182,42 @@ func FilterToInsts(index int, filter Filter, opts ProgOpts) (asm.Instructions, e
 		)
 		resultLabel = ""
 	}
+	useDropStats := opts.dropStatsMapFd != 0
 
+	// Initialize labels
 	mismatchLabel := fmt.Sprintf("mismatch_%d_", index)
+	afterDropStatsLabel := fmt.Sprintf("after_drop_stat_%d", index)
+	matchLabel := opts.onMatchLabel
+	mismatchTail := asm.Instructions{
+		asm.Mov.Imm(asm.R4, 0).WithSymbol(mismatchLabel),
+	}
+	skipLabel := mismatchLabel
+
+	// Change labels if it's a drop filter with drop stats
+	if useDropStats {
+		matchLabel = fmt.Sprintf("inc_drop_stat_%d", index)
+		skipLabel = afterDropStatsLabel
+		mismatchTail = asm.Instructions{
+			asm.Ja.Label(skipLabel),
+		}
+	}
 
 	if filter.Pid != 0 {
 		insts = append(insts,
 			// == 0, no match
-			asm.JEq.Imm(cbpfcOpts.Result, 0, mismatchLabel).WithSymbol(resultLabel),
+			asm.JEq.Imm(cbpfcOpts.Result, 0, skipLabel).WithSymbol(resultLabel),
 
 			// check the pid
 			// load the pid from the packet
 			asm.LoadMem(asm.R7, opts.eventPtrReg, structRawPacketEventPidOffset, asm.Word),
-			asm.JEq.Imm(asm.R7, int32(filter.Pid), opts.onMatchLabel),
-			asm.Mov.Imm(asm.R4, 0).WithSymbol(mismatchLabel), // nop instruction, just hold the symbol
+			asm.JEq.Imm(asm.R7, int32(filter.Pid), matchLabel),
 		)
+		insts = append(insts, mismatchTail...)
 	} else if !filter.CGroupPathKey.IsNull() {
 		// use the cgroup id which the inode of the cgroup path
 		insts = append(insts,
 			// == 0, no match
-			asm.JEq.Imm(cbpfcOpts.Result, 0, mismatchLabel).WithSymbol(resultLabel),
+			asm.JEq.Imm(cbpfcOpts.Result, 0, skipLabel).WithSymbol(resultLabel),
 
 			// load the cgroup id from the packet
 			asm.LoadMem(asm.R7, opts.eventPtrReg, structRawPacketEventCgroupIdOffset, asm.DWord),
@@ -169,21 +237,33 @@ func FilterToInsts(index int, filter Filter, opts ProgOpts) (asm.Instructions, e
 
 			// check the cgroup id
 			asm.LoadImm(asm.R4, int64(filter.CGroupPathKey.Inode), asm.DWord),
-			asm.JEq.Reg(asm.R7, asm.R4, opts.onMatchLabel),
-			asm.Mov.Imm(asm.R4, 0).WithSymbol(mismatchLabel), // nop instruction, just hold the symbol
+			asm.JEq.Reg(asm.R7, asm.R4, matchLabel),
+		)
+		insts = append(insts, mismatchTail...)
+	} else if useDropStats {
+		insts = append(insts,
+			asm.JEq.Imm(cbpfcOpts.Result, 0, skipLabel).WithSymbol(resultLabel),
+			asm.Ja.Label(matchLabel),
 		)
 	} else {
 		insts = append(insts,
-			asm.JNE.Imm(cbpfcOpts.Result, 0, opts.onMatchLabel).WithSymbol(resultLabel),
+			asm.JNE.Imm(cbpfcOpts.Result, 0, matchLabel).WithSymbol(resultLabel),
 		)
 	}
+
+	if useDropStats {
+		insts = append(insts, dropStatsIncrementInsts(index, opts.dropStatsMapFd, opts.onMatchLabel)...)
+		insts = append(insts, asm.Mov.Imm(asm.R4, 0).WithSymbol(afterDropStatsLabel)) // nop instruction, just hold the symbol
+	}
+
 	return insts, nil
 }
 
-// we want to creates progs like that
-// prog1 -> tc1 -> footer -> prog2 -> tc2 -> footer -> progN -> footer
-// where each prog is like this
-// header -> filter 1 -> filter 2 -> ... -> filter n -> footer
+// we want to create progs like that
+// prog1 -> prog2 -> ... -> progN
+//
+// where each prog is:
+// header -> filter 1 -> filter 2 -> ... -> filter n -> [tail_call] -> footer
 func filtersToProgs(filters []Filter, opts ProgOpts, headerInsts, footerInsts asm.Instructions) ([]asm.Instructions, *multierror.Error) {
 	var (
 		progInsts []asm.Instructions
@@ -386,3 +466,11 @@ func DropActionsToProgramSpecs(rawPacketEventMapFd, clsRouterMapFd int, filters
 
 	return progSpecs, mErr.ErrorOrNil()
 }
+
+// FormatProgramInstructions returns BPF instructions with their indices, useful when
+// debugging verifier errors such as "unreachable insn N".
+func FormatProgramInstructions(insts asm.Instructions) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "% 1v", insts)
+	return b.String()
+}

pkg/security/ebpf/probes/rawpacket/pcap_unsupported.go

@@ -40,6 +40,11 @@ func (opts *ProgOpts) WithGetCurrentCgroupID(_ bool) *ProgOpts {
 	return opts
 }
 
+// WithDropStatsMapFd sets the map fd used to count dropped packets per filter index.
+func (opts *ProgOpts) WithDropStatsMapFd(_ int) *ProgOpts {
+	return opts
+}
+
 // FilterToInsts compile a bpf filter expression
 func FilterToInsts(_ int, _ Filter, _ ProgOpts) (asm.Instructions, error) {
 	return asm.Instructions{}, errors.New("not supported")

pkg/security/ebpf/tests/raw_packet_test.go

@@ -84,7 +84,7 @@ func testRawPacketFilter(t *testing.T, filters []rawpacket.Filter, progName stri
 	assert.Equal(t, expRetCode, code, "return code error: %v, please check the kernel structure offset of raw_packet_event", err)
 }
 
-func testRawPacketDropAction(t *testing.T, filters []rawpacket.Filter, progName string, expRetCode int64, expProgNum int, opts rawpacket.ProgOpts, catchCompilerError bool) {
+func testRawPacketDropAction(t *testing.T, filters []rawpacket.Filter, progName string, expRetCode int64, expProgNum int, opts rawpacket.ProgOpts, catchCompilerError bool, withDropStats bool) {
 	var ctx baloum.StdContext
 
 	vm := newVM(t)
@@ -101,6 +101,12 @@ func testRawPacketDropAction(t *testing.T, filters []rawpacket.Filter, progName
 	routerMap, err := vm.LoadMap(inactiveName)
 	assert.Nil(t, err, "map not found")
 
+	if withDropStats {
+		droppedPacketsMap, err := vm.LoadMap("dropped_packets")
+		assert.Nil(t, err, "dropped_packets map not found")
+		opts.WithDropStatsMapFd(droppedPacketsMap.FD())
+	}
+
 	progSpecs, err := rawpacket.DropActionsToProgramSpecs(rawPacketEventMap.FD(), routerMap.FD(), filters, opts)
 	if err != nil {
 		if catchCompilerError {
@@ -293,7 +299,7 @@ func TestRawPacketDropAction(t *testing.T) {
 				Pid:       123,
 			},
 		}
-		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", 255, 1, rawpacket.DefaultProgOpts(), true)
+		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", 255, 1, rawpacket.DefaultProgOpts(), true, false)
 	})
 
 	t.Run("syn-port-std-pid-ko", func(t *testing.T) {
@@ -305,7 +311,7 @@ func TestRawPacketDropAction(t *testing.T) {
 				Pid:       999,
 			},
 		}
-		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", probes.TCActUnspec, 1, rawpacket.DefaultProgOpts(), true)
+		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", probes.TCActUnspec, 1, rawpacket.DefaultProgOpts(), true, false)
 	})
 
 	t.Run("syn-port-std-cgroup-ok", func(t *testing.T) {
@@ -319,7 +325,7 @@ func TestRawPacketDropAction(t *testing.T) {
 				},
 			},
 		}
-		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", 255, 1, rawpacket.DefaultProgOpts(), true)
+		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", 255, 1, rawpacket.DefaultProgOpts(), true, false)
 	})
 
 	t.Run("syn-port-std-cgroup-ko", func(t *testing.T) {
@@ -333,7 +339,19 @@ func TestRawPacketDropAction(t *testing.T) {
 				},
 			},
 		}
-		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", probes.TCActUnspec, 1, rawpacket.DefaultProgOpts(), true)
+		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", probes.TCActUnspec, 1, rawpacket.DefaultProgOpts(), true, false)
+	})
+
+	t.Run("syn-port-std-drop-stats", func(t *testing.T) {
+		filters := []rawpacket.Filter{
+			{
+				RuleID:    "ok",
+				BPFFilter: "tcp dst port 5555 and tcp[tcpflags] == tcp-syn",
+				Policy:    rawpacket.PolicyDrop,
+				Pid:       123,
+			},
+		}
+		testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", 255, 1, rawpacket.DefaultProgOpts(), true, true)
 	})
 }
 
@@ -362,5 +380,5 @@ func TestRawPacketActionWithInvalidFilter(t *testing.T) {
 
 	// catchCompilerError is false: the invalid filter is expected to fail compilation and
 	// be skipped, but the valid filter should still produce a single program (and match).
-	testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", 255, 1, rawpacket.DefaultProgOpts(), false)
+	testRawPacketDropAction(t, filters, "test/raw_packet_drop_action", 255, 1, rawpacket.DefaultProgOpts(), false, false)
 }

pkg/security/metrics/metrics.go

@@ -593,6 +593,10 @@ var (
 	// MetricEventSampleSampled is the name of the metric used to report events that were sampled in kernel
 	// Tags: event_type
 	MetricEventSampleSampled = newRuntimeMetric(".event_sample.sampled")
+
+	// MetricRawPacketDropped is the name of the metric used to count packets dropped by network_filter actions
+	// Tags: rule_id
+	MetricRawPacketDropped = newRuntimeMetric(".network.raw_packet.dropped")
 )
 
 var (

pkg/security/probe/BUILD.bazel

@@ -96,6 +96,7 @@ go_library(
             "//pkg/security/probe/monitors/discarder",
             "//pkg/security/probe/monitors/dns",
             "//pkg/security/probe/monitors/eventsample",
+            "//pkg/security/probe/monitors/rawpacketdrop",
             "//pkg/security/probe/monitors/syscalls",
             "//pkg/security/probe/procfs",
             "//pkg/security/probe/sysctl",
@@ -213,6 +214,7 @@ go_library(
             "//pkg/security/probe/monitors/discarder",
             "//pkg/security/probe/monitors/dns",
             "//pkg/security/probe/monitors/eventsample",
+            "//pkg/security/probe/monitors/rawpacketdrop",
             "//pkg/security/probe/monitors/syscalls",
             "//pkg/security/probe/procfs",
             "//pkg/security/probe/sysctl",

pkg/security/probe/monitors/rawpacketdrop/BUILD.bazel

@@ -0,0 +1,25 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "rawpacketdrop",
+    srcs = ["rawpacketdrop_monitor.go"],
+    importpath = "github.com/DataDog/datadog-agent/pkg/security/probe/monitors/rawpacketdrop",
+    visibility = ["//visibility:public"],
+    deps = select({
+        "@rules_go//go/platform:android": [
+            "//pkg/security/metrics",
+            "//pkg/security/probe/managerhelper",
+            "@com_github_cilium_ebpf//:ebpf",
+            "@com_github_datadog_datadog_go_v5//statsd",
+            "@com_github_datadog_ebpf_manager//:ebpf-manager",
+        ],
+        "@rules_go//go/platform:linux": [
+            "//pkg/security/metrics",
+            "//pkg/security/probe/managerhelper",
+            "@com_github_cilium_ebpf//:ebpf",
+            "@com_github_datadog_datadog_go_v5//statsd",
+            "@com_github_datadog_ebpf_manager//:ebpf-manager",
+        ],
+        "//conditions:default": [],
+    }),
+)