feat(tracing): add chunked tail-call traceparent scanner for large HTTP headers (fixes #1381)

When a Traceparent header is preceded by large headers (e.g. Authorization
tokens, X-Custom-Data payloads), it can land beyond the initial 1 KB eBPF
capture window and be silently missed. This change adds a tail-call based
chunked scanner that walks the request buffer in 956-byte steps, stopping at
the end-of-headers marker or when the traceparent is found.

bpf/common/trace_util.h:
- Add bpf_strstr_tp_eoh(): single bpf_loop pass that finds either the
  traceparent or the \r\n\r\n end-of-headers marker, whichever comes first.
  Uses the existing k_tp_pos_not_found sentinel from the preceding bugfix PR.

bpf/generictracer/k_tracer_tailcall.h:
- Add k_tail_parse_traceparent_http = 10. k_tail_continue_netfd_read moves
  from index 10 to 11. WARNING: changing jump_table indices is a hard ABI
  break; the BPF objects and both Go loaders (generictracer and gotracer)
  must be updated atomically — a full pod restart is required.

bpf/generictracer/protocol_http.h:
- Add obi_parse_traceparent_http: the new tail-call program. Iterates up to
  MAX_CHUNKS (29) times, calling bpf_strstr_tp_eoh on each 956-byte slice.
  Stops at EOH, traceparent found, or iteration limit.

bpf/generictracer/protocol_handler.c:
- Dispatch to obi_parse_traceparent_http from the still_reading path in
  obi_handle_buf_with_args when chunked scanning is configured.

bpf/generictracer/k_tracer.c:
- Support ITER_UBUF and single-segment ITER_IOVEC in return_recvmsg; set
  u_buf_is_user=1 so downstream code uses bpf_probe_read_user.

bpf/generictracer/k_tracer_defs.h:
- Rename handle_buf_with_connection to handle_buf_with_connection_ext;
  add handle_user_buf_with_connection wrapper that sets u_buf_is_user=1.

bpf/generictracer/ssl_defs.h, java_tls.c:
- Switch SSL/Java-TLS paths to handle_user_buf_with_connection.

pkg/internal/ebpf/generictracer/generictracer.go:
- Register obi_parse_traceparent_http in the jump table at index 10.
- Write bpf_max_request_tp_parse_size_kb from MaxRequestTPParseSizeKB;
  set to 0 on legacy kernels that lack bpf_loop.

pkg/internal/ebpf/gotracer/gotracer.go:
- Update k_tail_continue_netfd_read registration from index 10 to 11.

pkg/ebpf/common/http_transform.go:
- Use event.OriginalTraceId (not event.Tp.TraceId) as the large-buffer
  map key in extractTCPLargeBuffer. The scanner may overwrite Tp.TraceId
  after the buffer is stored; OriginalTraceId is stable.

pkg/ebpf/common/common.go:
- Register a dummy stub for obi_parse_traceparent_http on legacy kernels
  that do not support bpf_loop (dummy.Copy() from the preceding bugfix PR).

pkg/config/ebpf_tracer.go, pkg/obi/config.go:
- Add MaxRequestTPParseSizeKB (default 4, range 4-27 KB). Controls the
  maximum request size scanned by the chunked parser.

devdocs/config/CONFIG.md, config-schema.json: document the new knob.

internal/test/integration/components/tpclient/service.js:
- Add /with-huge-tp endpoint; refactor /with-tp span-ID increment to use
  the new nextTraceparent() helper.

internal/test/integration/traceparent_extraction_test.go:
- Add testWithHugeHeadersTraceparent: sends a 2500-byte filler header
  before Traceparent on the HTTP/kprobe path and asserts that all spans
  in the tpclient-a/b/c chain carry the static trace ID.

Author: smehboub

bpf/common/trace_util.h

@@ -98,6 +98,73 @@ static int tp_match(u32 index, void *data) {
     return 0;
 }
 
+// Combined traceparent + end-of-headers search context.
+// Used by bpf_strstr_tp_eoh to find both in a single bpf_loop pass.
+struct callback_ctx_eoh {
+    unsigned char *buf;
+    u32 tp_pos;
+    u32 eoh_pos;
+};
+
+// Searches for traceparent and \r\n\r\n in a single pass.
+// Stops at whichever comes first:
+//   - traceparent found → records tp_pos, stops
+//   - \r\n\r\n found    → records eoh_pos, stops (end of headers reached)
+//
+// The guard uses TRACE_PARENT_HEADER_LEN (68 bytes) as the cutoff for both
+// checks. is_eoh only needs 4 bytes, so the last 64 bytes of the buffer are
+// not checked for EOH here. Any EOH in that window is covered by the 68-byte
+// chunk overlap: the next chunk starts TRACE_PARENT_HEADER_LEN bytes before
+// the end of the current one, so the overlap bytes [956..1023] are rescanned
+// at local indices [0..67] in the next iteration.
+static int tp_eoh_match(u32 index, void *data) {
+    if (index >= (TRACE_BUF_SIZE - TRACE_PARENT_HEADER_LEN)) {
+        return 1;
+    }
+
+    struct callback_ctx_eoh *ctx = data;
+    unsigned char *s = &ctx->buf[index];
+
+    if (is_eoh(s)) {
+        ctx->eoh_pos = index;
+        return 1;
+    }
+
+    if (is_traceparent(s)) {
+        ctx->tp_pos = index;
+        return 1;
+    }
+
+    return 0;
+}
+
+// Like bpf_strstr_tp_loop but also stops at the end-of-headers marker.
+// Sets *eoh_found=true if \r\n\r\n was reached before any traceparent.
+// Callers must not tail-call to the next chunk when *eoh_found is true.
+static __always_inline unsigned char *
+bpf_strstr_tp_eoh(unsigned char *buf, const u16 buf_len, bool *eoh_found) {
+    *eoh_found = false;
+    if (!g_bpf_traceparent_enabled) {
+        return NULL;
+    }
+
+    struct callback_ctx_eoh data = {
+        .buf = buf, .tp_pos = k_tp_pos_not_found, .eoh_pos = k_tp_pos_not_found};
+
+    bpf_loop((u32)buf_len, tp_eoh_match, &data, 0);
+
+    if (data.eoh_pos != k_tp_pos_not_found) {
+        *eoh_found = true;
+    }
+
+    if (data.tp_pos != k_tp_pos_not_found) {
+        return (data.tp_pos > (TRACE_BUF_SIZE - TRACE_PARENT_HEADER_LEN)) ? NULL
+                                                                          : &buf[data.tp_pos];
+    }
+
+    return NULL;
+}
+
 static __always_inline unsigned char *bpf_strstr_tp_loop(unsigned char *buf, const u16 buf_len) {
     if (!g_bpf_traceparent_enabled) {
         return NULL;

bpf/generictracer/java_tls.c

@@ -160,7 +160,7 @@ int BPF_KPROBE(obi_kprobe_sys_ioctl) {
         void *buf = arg + 1 + sizeof(connection_info_t) + sizeof(u32);
         const u64 zero = 0;
         bpf_map_update_elem(&active_ssl_connections, &p_conn, &zero, BPF_ANY);
-        handle_buf_with_connection(ctx, &p_conn, buf, len, WITH_SSL, op, orig_dport);
+        handle_user_buf_with_connection(ctx, &p_conn, buf, len, WITH_SSL, op, orig_dport);
     }
 
     return 0;

bpf/generictracer/k_tracer.c

@@ -963,6 +963,8 @@ static __always_inline int return_recvmsg(void *ctx, struct sock *in_sock, u64 i
     }
 
     unsigned char *buf = 0;
+    u64 orig_ubuf = 0;
+    int full_copied_len = copied_len;
     if (args) {
         iovec_iter_ctx *iov_ctx = (iovec_iter_ctx *)&args->iovec_ctx;
 
@@ -972,6 +974,22 @@ static __always_inline int return_recvmsg(void *ctx, struct sock *in_sock, u64 i
             goto done;
         }
 
+        if (bpf_core_enum_value_exists(enum iter_type___dummy, ITER_UBUF) &&
+            iov_ctx->iter_type == bpf_core_enum_value(enum iter_type___dummy, ITER_UBUF)) {
+            orig_ubuf = (u64)iov_ctx->ubuf;
+        } else if (iov_ctx->iter_type == bpf_core_enum_value(enum iter_type, ITER_IOVEC) &&
+                   iov_ctx->nr_segs == 1) {
+            // On kernels < 6.0, ITER_UBUF does not exist and single-buffer recvmsg calls
+            // use ITER_IOVEC with exactly one segment.  In that case the entire receive
+            // payload is in one contiguous userspace buffer; capture its base pointer so
+            // the chunked traceparent scanner can read beyond the 8 KB iovec scratch cap
+            // using bpf_probe_read_user (same mechanism as ITER_UBUF on newer kernels).
+            struct iovec vec;
+            if (bpf_probe_read_kernel(&vec, sizeof(vec), &iov_ctx->iov[0]) == 0 && vec.iov_base) {
+                orig_ubuf = (u64)vec.iov_base;
+            }
+        }
+
         buf = iovec_memory();
         if (buf) {
             copied_len = read_iovec_ctx(iov_ctx, buf, copied_len);
@@ -1025,8 +1043,16 @@ static __always_inline int return_recvmsg(void *ctx, struct sock *in_sock, u64 i
             if (buf && copied_len) {
                 bpf_map_delete_elem(&active_recv_args, &id);
                 // doesn't return must be logically last statement
-                handle_buf_with_connection(
-                    ctx, &info, buf, copied_len, NO_SSL, TCP_RECV, orig_dport);
+                handle_buf_with_connection_ext(ctx,
+                                               &info,
+                                               buf,
+                                               copied_len,
+                                               NO_SSL,
+                                               TCP_RECV,
+                                               orig_dport,
+                                               orig_ubuf,
+                                               full_copied_len,
+                                               0);
             }
         } else {
             bpf_dbg_printk("identified SSL connection, ignoring: [%llx]...", *ssl);

bpf/generictracer/k_tracer_defs.h

@@ -56,6 +56,11 @@ static __always_inline call_protocol_args_t *make_protocol_args(const pid_connec
     args->u_buf = (u64)u_buf;
     args->lw_thread = lw_thread;
     args->protocols = protocols;
+    args->orig_buf = 0;
+    args->full_bytes_len = 0;
+    args->niter = 0;
+    args->is_append = 0;
+    args->u_buf_is_user = 0;
     args->protocol_type = protocol_type_for_conn_info(info);
 
     args->pid_conn = *info;
@@ -64,13 +69,16 @@ static __always_inline call_protocol_args_t *make_protocol_args(const pid_connec
     return args;
 }
 
-static __always_inline void handle_buf_with_connection(void *ctx,
-                                                       pid_connection_info_t *pid_conn,
-                                                       void *u_buf,
-                                                       int bytes_len,
-                                                       u8 ssl,
-                                                       u8 direction,
-                                                       u16 orig_dport) {
+static __always_inline void handle_buf_with_connection_ext(void *ctx,
+                                                           pid_connection_info_t *pid_conn,
+                                                           void *u_buf,
+                                                           int bytes_len,
+                                                           u8 ssl,
+                                                           u8 direction,
+                                                           u16 orig_dport,
+                                                           u64 orig_buf,
+                                                           u32 full_bytes_len,
+                                                           u8 u_buf_is_user) {
     call_protocol_args_t *args = make_protocol_args(pid_conn,
                                                     k_lw_thread_none,
                                                     k_protocol_selector_all,
@@ -83,9 +91,34 @@ static __always_inline void handle_buf_with_connection(void *ctx,
         return;
     }
 
+    args->orig_buf = orig_buf;
+    args->full_bytes_len = full_bytes_len;
+    args->u_buf_is_user = u_buf_is_user;
     bpf_tail_call(ctx, &jump_table, k_tail_handle_buf_with_args);
 }
 
+static __always_inline void handle_buf_with_connection(void *ctx,
+                                                       pid_connection_info_t *pid_conn,
+                                                       void *u_buf,
+                                                       int bytes_len,
+                                                       u8 ssl,
+                                                       u8 direction,
+                                                       u16 orig_dport) {
+    handle_buf_with_connection_ext(
+        ctx, pid_conn, u_buf, bytes_len, ssl, direction, orig_dport, 0, 0, 0);
+}
+
+static __always_inline void handle_user_buf_with_connection(void *ctx,
+                                                            pid_connection_info_t *pid_conn,
+                                                            void *u_buf,
+                                                            int bytes_len,
+                                                            u8 ssl,
+                                                            u8 direction,
+                                                            u16 orig_dport) {
+    handle_buf_with_connection_ext(
+        ctx, pid_conn, u_buf, bytes_len, ssl, direction, orig_dport, 0, 0, 1);
+}
+
 static __always_inline void handle_light_weight_thread_buf(void *ctx,
                                                            const lw_thread_t lw_thread,
                                                            protocol_selector_t protocols,
@@ -101,6 +134,7 @@ static __always_inline void handle_light_weight_thread_buf(void *ctx,
         return;
     }
 
+    args->u_buf_is_user = 1;
     bpf_tail_call(ctx, &jump_table, k_tail_handle_buf_with_args);
 }
 

bpf/generictracer/k_tracer_tailcall.h

@@ -24,5 +24,13 @@ enum {
     k_tail_protocol_http2_grpc_handle_end_frame = 7,
     k_tail_handle_buf_with_args = 8,
     k_tail_continue_protocol_http_tp = 9,
-    k_tail_continue_netfd_read = 10,
+    k_tail_parse_traceparent_http = 10,
+    // WARNING: k_tail_continue_netfd_read moved from index 10 to 11 when the
+    // chunked traceparent scanner was added at index 10. This affects gotracer
+    // (go_net.c), which calls this index. Changing jump_table indices is a
+    // hard breaking change during a rolling upgrade: the BPF objects and the
+    // Go loader programs (both generictracer and gotracer) must be updated
+    // atomically — a full pod restart is required; in-place BPF reload is not
+    // safe.
+    k_tail_continue_netfd_read = 11,
 };

bpf/generictracer/protocol_handler.c

@@ -142,20 +142,33 @@ int obi_handle_buf_with_args(void *ctx) {
                     packet_type = PACKET_TYPE_RESPONSE;
                 }
 
-                http_send_large_buffer(info,
-                                       (void *)args->u_buf,
-                                       args->bytes_len,
-                                       packet_type,
-                                       args->direction,
-                                       k_large_buf_action_append);
-
                 if (reading) {
+                    const u32 prev_len = info->len;
                     info->len += args->bytes_len;
+                    if (g_bpf_traceparent_enabled && capture_header_buffer &&
+                        bpf_max_request_tp_parse_size_kb > 0 &&
+                        prev_len < (u32)bpf_max_request_tp_parse_size_kb * 1024) {
+                        args->packet_type = packet_type;
+                        args->is_append = 1;
+                        args->niter = 0;
+                        bpf_tail_call(ctx, &jump_table, k_tail_parse_traceparent_http);
+                        // tail-call failed — fall through
+                    }
                 } else if (responding) {
                     info->end_monotime_ns = bpf_ktime_get_ns();
                     bpf_d_printk("bytes len %d, new bytes %d", info->resp_len, args->bytes_len);
                     info->resp_len += args->bytes_len;
                 }
+
+                // TP parsing not needed or tail-call failed: emit large buffer now.
+                // When the tail-call succeeds, obi_parse_traceparent_http emits
+                // it at done: instead, so this path is only reached as a fallback.
+                http_send_large_buffer(info,
+                                       (void *)args->u_buf,
+                                       args->bytes_len,
+                                       packet_type,
+                                       args->direction,
+                                       k_large_buf_action_append);
             }
         } else if (args->protocols.tcp && !info) {
             // SSL requests will see both TCP traffic and text traffic, ignore the TCP if