How to populate an array of maps during loading?

[efagerho]

I'm testing out ebpf-go for the first time and I'm wondering how to populate an array of maps when loading the program. I have the following XDP program

#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/udp.h>
#include <linux/tcp.h>
#include <linux/in.h>
#include <linux/time.h>

#define MAX_CPUs 128

struct flow_key {
    __u32 src_ip;
    __u16 src_port;
};

struct flow_entry {
    __u64 timestamp_ns;
    __u32 dst_ip;
    __u16 dst_port;
    __u16 src_port;
};

struct port_assignment {
    __u32 dst_ip;
    __u16 dst_port;
    __u16 src_port;
};

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY);
    __uint(max_entries, 1);
    __type(key, __u32);
    __type(value, __u32);
} next_cpu SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 512000);
    __type(key, struct flow_key);
    __type(value, struct flow_entry);
} flows SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
    __uint(max_entries, MAX_CPUs + 1);
    __type(key, __u32);
    __type(value, __u32);
    __array(values, flows);
} per_cpu_flows SEC(".maps");

struct timer_elem {
    struct bpf_timer timer;
};

#define FLOW_TIMEOUT_NS ((__u64) 1 * 1000000000ULL)

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY);
    __uint(max_entries, 1);
    __type(key, __u32);
    __type(value, __u32);
    __uint(pinning, LIBBPF_PIN_BY_NAME);
} global_gc_timer_set SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY);
    __uint(max_entries, 1);
    __type(key, __u32);
    __type(value, struct timer_elem);
} global_gc_timer SEC(".maps");

static long flow_inspector(void *map, const struct flow_key *key, struct flow_entry *flow, void *ctx) {
    __u64 now = bpf_ktime_get_ns();

    if (flow->timestamp_ns + FLOW_TIMEOUT_NS < now) {
        bpf_map_delete_elem(map, key);
    }

    return 0;
}

static int expire_flows(void *map, void *key, struct timer_elem *timer) {
    __u32 index = 0;
    bpf_printk("Running GC on flows\n");

    __u32 *cpu = bpf_map_lookup_elem(&next_cpu, &index);
    if (!cpu) {
        bpf_printk("Could not retrieve next CPU for GC\n");
        return 0;
    }

    void *flows_to_gc = bpf_map_lookup_elem(&per_cpu_flows, cpu);
    if (!flows_to_gc) {
        bpf_printk("Could not retrieve flows for CPU %d, %ld\n", *cpu, flows_to_gc);
        return 0;
    }

    bpf_for_each_map_elem(flows_to_gc, flow_inspector, 0, 0);

    *cpu = (*cpu == MAX_CPUs) ? 0 : *cpu + 1;
    long ret = bpf_timer_start(&timer->timer, FLOW_TIMEOUT_NS, 0);
    bpf_printk("bpf_timer_start return value: %ld\n", ret);
    return 0;
}

static __always_inline void ensure_timer_started() {
    __u32 key = 0;

    __u32 *timer_set = bpf_map_lookup_elem(&global_gc_timer_set, &key);
    if (timer_set != 0 && !*timer_set) {
        *timer_set = 1;

        struct timer_elem* timer = bpf_map_lookup_elem(&global_gc_timer, &key);
        if (!timer) {
            return;
        }

	long ret;

        bpf_printk("Attaching global GC timer\n");
        ret = bpf_timer_init(&timer->timer, &global_gc_timer, CLOCK_MONOTONIC);
        bpf_printk("bpf_timer_init return value: %ld\n", ret);

        ret = bpf_timer_set_callback(&timer->timer, expire_flows);
        bpf_printk("bpf_timer_set_callback return value: %ld\n", ret);

        ret = bpf_timer_start(&timer->timer, FLOW_TIMEOUT_NS, 0);
        bpf_printk("bpf_timer_start return value: %ld\n", ret);
    }
}

SEC("xdp")
int udp_router(struct xdp_md *ctx) {
    ensure_timer_started();

    void *data = (void *)(long)ctx->data;
    void *data_end = (void *)(long)ctx->data_end;

    struct ethhdr *eth = data;
    if ((void *)(eth + 1) > data_end)
        return XDP_PASS;

    if (eth->h_proto != bpf_htons(ETH_P_IP))
        return XDP_PASS;

    struct iphdr *ip = data + sizeof(*eth);
    if ((void *)(ip + 1) > data_end)
        return XDP_PASS;

    if (ip->protocol != IPPROTO_UDP)
        return XDP_PASS;

    struct udphdr *udp = (void *)ip + ip->ihl * 4;
    if ((void*)(udp + 1) > data_end)
        return XDP_PASS;

    return XDP_PASS;
}

char _license[] SEC("license") = "GPL";

My Go program for loading it is the following:

package main

import (
	"log/slog"
	"net"
	"os"
	"os/signal"
	"runtime"
	"time"

	"github.com/cilium/ebpf"
	"github.com/cilium/ebpf/link"
	"github.com/cilium/ebpf/rlimit"
)

func main() {
	logger := slog.Default()
	ifaceName := "ens1"

	// Remove resource limits for kernels <5.11.
	if err := rlimit.RemoveMemlock(); err != nil {
		logger.Error("Removing memlock", "error", err)
		os.Exit(1)
	}

	// Load the compiled eBPF ELF and load it into the kernel.
	var objs udp_routerObjects
	if err := loadUdp_routerObjects(&objs, &ebpf.CollectionOptions{
		Maps: ebpf.MapOptions{
			PinPath: "/sys/fs/bpf",
		},
	}); err != nil {
		logger.Error("Loading eBPF objects", "error", err)
		os.Exit(1)
	}
	defer objs.Close()

	// Creating flow tables for each CPU
	cpus := runtime.NumCPU()

	for cpu := 0; cpu < cpus; cpu++ {
		if err := objs.udp_routerMaps.PerCpuFlows.Update(&cpu, objs.udp_routerMaps.Flows, ebpf.UpdateAny); err != nil {
			logger.Error("Inserting map", "cpu", cpu, "error", err)
			os.Exit(1)
		}
	}

	iface, err := net.InterfaceByName(ifaceName)
	if err != nil {
		logger.Error("Getting interface", "interface", ifaceName, "error", err)
		os.Exit(1)
	}

	mode := link.XDPGenericMode

	// Attach count_packets to the network interface.
	link, err := link.AttachXDP(link.XDPOptions{
		Program:   objs.UdpRouter,
		Interface: iface.Index,
		Flags:     mode,
	})
	if err != nil {
		logger.Error("Attaching XDP", "interface", iface.Name, "error", err)
		os.Exit(1)
	}
	defer link.Close()

	logger.Info("Starting UDP Router", "interface", iface.Name)

	// Event loop
	tick := time.Tick(time.Second)
	stop := make(chan os.Signal, 5)
	signal.Notify(stop, os.Interrupt)

	for {
		select {
		case <-tick:
			logger.Info("Alive and well...")
		case <-stop:
			logger.Info("Received signal, exiting..")
			return
		}
	}
}

However, it fails in the loop, where it's trying to create the flow tables for each CPU. I couldn't find in the manual details on how to initialize such arrays, so I could be doing it wrong. I see with strace one syscall that returns a not supported error, but can't really figure out what operation is causing it.

Any ideas?

[ti-mo]

@efagerho Thanks for the question! Could you please trim it down to the essentials? (most of the code isn't relevant to this issue) Also, including the error message would be useful.

There are a few traps when dealing with map-in-maps, you're certainly not alone. :) The flows (objs.udp_routerMaps.Flows in Go) is an instance of a map, not a Spec. The code as provided above will make all $numcpus keys point to the same map. As such, you'll probably want to do the following:

for cpu := range cpus {
	// Instantiate a new inner flows map for each CPU.
	m, err := ebpf.NewMap(udp_routerMapSpecs.PerCpuFlows.InnerMap)
	if err != nil {
		panic(err)
	}
	defer m.Close()

	// Insert it into the outer map.
	if err := objs.udp_routerMaps.PerCpuFlows.Update(&cpu, m, ebpf.UpdateAny); err != nil {
		logger.Error("Inserting map", "cpu", cpu, "error", err)
		os.Exit(1)
	}
}

The error you're getting is probably from loadUdp_routerObjects (again, hard to tell without clear output) because you've specified __array(values, flows);, flows being a map declaration (technically a global variable), not an inner map type. Try this:

struct flows {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 512000);
    __type(key, struct flow_key);
    __type(value, struct flow_entry);
}

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
    __uint(max_entries, MAX_CPUs + 1);
    __type(key, __u32);
    __type(value, __u32);
    __array(values, struct flows);
} per_cpu_flows SEC(".maps");

This avoids instantiating a global flows map that's never inserted into the outer map. Note that there's currently an LLVM compiler bug that requires you to reference struct flows somewhere in your program; don't attempt to compile and load this snippet as-is.

Also see https://github.com/cilium/ebpf/blob/main/examples/map_in_map/main.go for more MapSpec twiddling.