compiler, library: Add BTF field-info relocation intrinsics

BTF, the BPF Type Format, encodes type information for both the running
Linux kernel and compiled eBPF programs. An eBPF object can carry
relocation records that describe field and aggregate accesses in terms
of BTF types instead of fixed offsets; at load time, the loader compares
the program's BTF with the kernel's BTF and rewrites those accesses to
the correct offsets for the target kernel. This mechanism is often
referred to as "CO-RE relocations" or "BTF relocations".

`offset_of` always folds to a plain layout constant and does not
preserve enough information for BTF CO-RE relocation emission. As a
result, it is not suitable for relocatable field queries on BPF targets.

Add three intrinsics for BTF field metadata queries:

* `btf_field_byte_offset`
* `btf_field_byte_size`
* `btf_field_exists`

Their availability is hidden behind the `btf_relocations` feature gate.

Unlike `offset_of`, they remain visible to backend codegen and can lower
to relocatable field-info queries instead of immediate layout constants.

For LLVM, lower these intrinsics through `@llvm.bpf.preserve.field.info`
with the corresponding query kind. The necessary
`@llvm.preserve.{struct,array,union}.access.index` chain is constructed
internally during lowering, but it is not exposed as part of the
user-facing API.

On targets or backends without BTF relocation support, fall back to:

* The field offset for `btf_field_byte_offset`.
* The field size for `btf_field_byte_size`.
* `true` for `btf_field_exists`.

The language-level design for this feature is proposed in
rust-lang/rfcs#3966.

Author: vadorovsky

compiler/rustc_abi/src/lib.rs

@@ -96,6 +96,7 @@ bitflags! {
         /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.
         const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5;
         const IS_SCALABLE        = 1 << 6;
+        const IS_BTF             = 1 << 7;
          // Any of these flags being set prevent field reordering optimisation.
         const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
@@ -203,6 +204,11 @@ impl ReprOptions {
         self.flags.contains(ReprFlags::IS_LINEAR)
     }
 
+    #[inline]
+    pub fn btf(&self) -> bool {
+        self.flags.contains(ReprFlags::IS_BTF)
+    }
+
     /// Returns the discriminant type, given these `repr` options.
     /// This must only be called on enums!
     ///

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -217,6 +217,14 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                                 "SIMD types are experimental and possibly buggy"
                             );
                         }
+                        if item.has_name(sym::btf) {
+                            gate!(
+                                &self,
+                                btf_relocations,
+                                attr.span,
+                                "BTF relocations are experimental"
+                            );
+                        }
                     }
                 }
             }

compiler/rustc_attr_parsing/src/attributes/repr.rs

@@ -143,6 +143,7 @@ fn parse_repr<S: Stage>(cx: &AcceptContext<'_, '_, S>, param: &MetaItemParser) -
         (Some(sym::C), ArgParser::NoArgs) => Some(ReprC),
         (Some(sym::simd), ArgParser::NoArgs) => Some(ReprSimd),
         (Some(sym::transparent), ArgParser::NoArgs) => Some(ReprTransparent),
+        (Some(sym::btf), ArgParser::NoArgs) => Some(ReprBtf),
         (Some(name @ int_pat!()), ArgParser::NoArgs) => {
             // int_pat!() should make sure it always parses
             Some(ReprInt(int_type_of_word(name).unwrap()))
@@ -154,6 +155,7 @@ fn parse_repr<S: Stage>(cx: &AcceptContext<'_, '_, S>, param: &MetaItemParser) -
                 | name @ sym::C
                 | name @ sym::simd
                 | name @ sym::transparent
+                | name @ sym::btf
                 | name @ int_pat!(),
             ),
             ArgParser::NameValue(_),
@@ -167,6 +169,7 @@ fn parse_repr<S: Stage>(cx: &AcceptContext<'_, '_, S>, param: &MetaItemParser) -
                 | name @ sym::C
                 | name @ sym::simd
                 | name @ sym::transparent
+                | name @ sym::btf
                 | name @ int_pat!(),
             ),
             ArgParser::List(_),

compiler/rustc_codegen_llvm/src/builder.rs

@@ -8,14 +8,15 @@ pub(crate) mod gpu_offload;
 
 use libc::{c_char, c_uint};
 use rustc_abi as abi;
-use rustc_abi::{Align, Size, WrappingRange};
+use rustc_abi::{Align, FieldIdx, Size, VariantIdx, WrappingRange};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
+use rustc_middle::bug;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, TargetFeature, TargetFeatureKind};
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
@@ -34,6 +35,7 @@ use crate::abi::FnAbiLlvmExt;
 use crate::attributes;
 use crate::common::Funclet;
 use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
+use crate::debuginfo::metadata::type_di_node;
 use crate::llvm::{
     self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE,
     ToLlvmBool, Type, Value,
@@ -936,6 +938,172 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    /* BTF relocations */
+    fn btf_preserve_array_access_index(
+        &mut self,
+        base_ty: Ty<'tcx>,
+        ty: &'ll Type,
+        ptr: &'ll Value,
+        dimension: u64,
+        index: u64,
+    ) -> &'ll Value {
+        if self.cx.tcx.sess.target.arch != Arch::Bpf || self.cx.dbg_cx.is_none() {
+            let mut indices = Vec::with_capacity(dimension as usize + 1);
+            for _ in 0..dimension {
+                indices.push(self.const_usize(0));
+            }
+            indices.push(self.const_usize(index));
+            return self.inbounds_gep(ty, ptr, &indices);
+        }
+        let dbg_info: &'ll Metadata = type_di_node(self.cx, base_ty);
+        unsafe {
+            llvm::LLVMRustBuildPreserveArrayAccessIndex(
+                self.llbuilder,
+                ty,
+                ptr,
+                dimension as c_uint,
+                index as c_uint,
+                Some(dbg_info),
+            )
+        }
+    }
+
+    fn btf_preserve_struct_access_index(
+        &mut self,
+        base_ty: Ty<'tcx>,
+        ty: &'ll Type,
+        ptr: &'ll Value,
+        gep_index: u64,
+        field_index: u64,
+    ) -> &'ll Value {
+        if self.cx.tcx.sess.target.arch != Arch::Bpf || self.cx.dbg_cx.is_none() {
+            let zero = self.const_usize(0);
+            let gep_index = self.const_usize(gep_index);
+            return self.inbounds_gep(ty, ptr, &[zero, gep_index]);
+        }
+        let dbg_info: &'ll Metadata = type_di_node(self.cx, base_ty);
+        unsafe {
+            llvm::LLVMRustBuildPreserveStructAccessIndex(
+                self.llbuilder,
+                ty,
+                ptr,
+                gep_index as c_uint,
+                field_index as c_uint,
+                Some(dbg_info),
+            )
+        }
+    }
+
+    fn btf_preserve_union_access_index(
+        &mut self,
+        base_ty: Ty<'tcx>,
+        ptr: &'ll Value,
+        field_index: u64,
+    ) -> &'ll Value {
+        if self.cx.tcx.sess.target.arch != Arch::Bpf || self.cx.dbg_cx.is_none() {
+            return ptr;
+        }
+        let dbg_info: &'ll Metadata = type_di_node(self.cx, base_ty);
+        unsafe {
+            llvm::LLVMRustBuildPreserveUnionAccessIndex(
+                self.llbuilder,
+                ptr,
+                field_index as c_uint,
+                Some(dbg_info),
+            )
+        }
+    }
+
+    fn btf_field_info(
+        &mut self,
+        base_ty: Ty<'tcx>,
+        variant: VariantIdx,
+        field: FieldIdx,
+        kind: u32,
+    ) -> &'ll Value {
+        const BPF_FIELD_BYTE_OFFSET: u32 = 0;
+        const BPF_FIELD_BYTE_SIZE: u32 = 1;
+        const BPF_FIELD_EXISTS: u32 = 2;
+
+        fn llvm_struct_field_index<'ll, 'tcx>(
+            bx: &Builder<'_, 'll, 'tcx>,
+            layout: TyAndLayout<'tcx>,
+            field_index: usize,
+        ) -> usize {
+            let mut llvm_index = 0;
+            let mut offset = Size::ZERO;
+
+            for i in layout.fields.index_by_increasing_offset() {
+                let target_offset = layout.fields.offset(i as usize);
+                if target_offset != offset {
+                    llvm_index += 1;
+                }
+
+                if i as usize == field_index {
+                    return llvm_index;
+                }
+
+                let field = layout.field(bx.cx(), i);
+                llvm_index += 1;
+                offset = target_offset + field.size;
+            }
+
+            bug!("field index {field_index} not found in layout {layout:#?}")
+        }
+
+        let layout = self.layout_of(base_ty);
+        let cx = ty::layout::LayoutCx::new(self.tcx, self.typing_env());
+        let layout = layout.for_variant(&cx, variant);
+        let offset = layout.fields.offset(field.index()).bytes() as u32;
+        let field_layout = layout.field(self.cx(), field.index());
+        let size = field_layout.size.bytes() as u32;
+
+        if self.cx.tcx.sess.target.arch != Arch::Bpf || self.cx.dbg_cx.is_none() {
+            return match kind {
+                BPF_FIELD_BYTE_OFFSET => self.const_u32(offset),
+                BPF_FIELD_BYTE_SIZE => self.const_u32(size),
+                BPF_FIELD_EXISTS => self.const_u32(1),
+                _ => bug!("unsupported static fallback for btf_field_info kind {kind}"),
+            };
+        }
+
+        let base = self.const_null(self.type_ptr());
+        let field_ptr = match base_ty.kind() {
+            ty::Adt(adt, _) if adt.is_union() || adt.is_enum() => {
+                return match kind {
+                    BPF_FIELD_BYTE_OFFSET => self.const_u32(offset),
+                    BPF_FIELD_BYTE_SIZE => self.const_u32(size),
+                    BPF_FIELD_EXISTS => self.const_u32(1),
+                    _ => bug!("unsupported union or enum fallback for btf_field_info kind {kind}"),
+                };
+            }
+            ty::Adt(..) | ty::Tuple(..) => {
+                let llvm_index = llvm_struct_field_index(self, layout, field.index());
+                self.btf_preserve_struct_access_index(
+                    base_ty,
+                    self.cx().backend_type(layout),
+                    base,
+                    llvm_index as u64,
+                    field.index() as u64,
+                )
+            }
+            _ => {
+                return match kind {
+                    BPF_FIELD_BYTE_OFFSET => self.const_u32(offset),
+                    BPF_FIELD_BYTE_SIZE => self.const_u32(size),
+                    BPF_FIELD_EXISTS => self.const_u32(1),
+                    _ => bug!("unsupported fallback for btf_field_info kind {kind}"),
+                };
+            }
+        };
+
+        self.call_intrinsic(
+            "llvm.bpf.preserve.field.info",
+            &[self.val_ty(field_ptr)],
+            &[field_ptr, self.const_u64(kind.into())],
+        )
+    }
+
     /* Casts */
     fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1399,6 +1399,30 @@ unsafe extern "C" {
         Flags: GEPNoWrapFlags,
     ) -> &'a Value;
 
+    // BTF relocations
+    pub(crate) fn LLVMRustBuildPreserveArrayAccessIndex<'a>(
+        B: &Builder<'a>,
+        ElTy: &'a Type,
+        Base: &'a Value,
+        Dimension: c_uint,
+        LastIndex: c_uint,
+        DbgInfo: Option<&'a Metadata>,
+    ) -> &'a Value;
+    pub(crate) fn LLVMRustBuildPreserveUnionAccessIndex<'a>(
+        B: &Builder<'a>,
+        Base: &'a Value,
+        FieldIndex: c_uint,
+        DbgInfo: Option<&'a Metadata>,
+    ) -> &'a Value;
+    pub(crate) fn LLVMRustBuildPreserveStructAccessIndex<'a>(
+        B: &Builder<'a>,
+        ElTy: &'a Type,
+        Base: &'a Value,
+        Index: c_uint,
+        FieldIndex: c_uint,
+        DbgInfo: Option<&'a Metadata>,
+    ) -> &'a Value;
+
     // Casts
     pub(crate) fn LLVMBuildTrunc<'a>(
         B: &Builder<'a>,

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -1,4 +1,4 @@
-use rustc_abi::{Align, WrappingRange};
+use rustc_abi::{Align, FieldIdx, VariantIdx, WrappingRange};
 use rustc_middle::mir::SourceInfo;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{bug, span_bug};
@@ -9,7 +9,7 @@ use rustc_target::spec::Arch;
 use super::FunctionCx;
 use super::operand::OperandRef;
 use super::place::PlaceRef;
-use crate::common::{AtomicRmwBinOp, SynchronizationScope};
+use crate::common::{AtomicRmwBinOp, IntPredicate, SynchronizationScope};
 use crate::errors::InvalidMonomorphization;
 use crate::traits::*;
 use crate::{MemFlags, meth, size_of_val};
@@ -579,6 +579,36 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
+            sym::btf_field_byte_offset | sym::btf_field_byte_size | sym::btf_field_exists => {
+                let tp_ty = fn_args.type_at(0);
+                let Some(variant) = bx.const_to_opt_uint(args[0].immediate()) else {
+                    span_bug!(span, "`btf_field_*` variant must be a constant");
+                };
+                let Some(field) = bx.const_to_opt_uint(args[1].immediate()) else {
+                    span_bug!(span, "`btf_field_*` field must be a constant");
+                };
+                let variant: u32 = variant.try_into().unwrap_or_else(|_| {
+                    span_bug!(span, "`btf_field_*` variant does not fit in u32");
+                });
+                let kind = match name {
+                    sym::btf_field_byte_offset => 0,
+                    sym::btf_field_byte_size => 1,
+                    sym::btf_field_exists => 2,
+                    _ => bug!(),
+                };
+                let llval = bx.btf_field_info(
+                    tp_ty,
+                    VariantIdx::from_u32(variant),
+                    FieldIdx::from_usize(field as usize),
+                    kind,
+                );
+                if result.layout.ty.is_bool() {
+                    bx.icmp(IntPredicate::IntNE, llval, bx.const_u32(0))
+                } else {
+                    bx.zext(llval, bx.type_isize())
+                }
+            }
+
             sym::cold_path => {
                 // This is a no-op. The intrinsic is just a hint to the optimizer.
                 return Ok(());