diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index ee46b49a094c6..7499f312afddb 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use std::{io, iter, slice}; use object::read::archive::ArchiveFile; +use rustc_abi::{Align, Size}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; @@ -22,14 +23,18 @@ use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel use rustc_session::config::{self, CrateType, Lto}; use tracing::{debug, info}; +use llvm::Linkage::*; + use crate::back::write::{ self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode, }; +use crate::builder::{SBuilder, UNNAMED}; use crate::errors::{ DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro, }; +use crate::common::AsCCharPtr; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, build_string}; +use crate::llvm::{self, build_string, Linkage}; use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes}; /// We keep track of the computed LTO cache keys from the previous @@ -39,11 +44,11 @@ const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { CrateType::Executable - | CrateType::Dylib - | CrateType::Staticlib - | CrateType::Cdylib - | CrateType::ProcMacro - | CrateType::Sdylib => true, + | CrateType::Dylib + | CrateType::Staticlib + | CrateType::Cdylib + | CrateType::ProcMacro + | CrateType::Sdylib => true, CrateType::Rlib => false, } } @@ -113,7 +118,7 @@ fn prepare_lto( cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); symbols_below_threshold .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); - } + } let archive_data = unsafe { Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib")) @@ -127,7 +132,7 @@ fn prepare_lto( std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c)) }) }) - .filter(|&(name, _)| looks_like_rust_object_file(name)); + .filter(|&(name, _)| looks_like_rust_object_file(name)); for (name, child) in obj_files { info!("adding bitcode from {}", name); match get_bitcode_slice_from_object_data( @@ -295,7 +300,7 @@ fn fat_lto( let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; (cost, i) }) - .max(); + .max(); // If we found a costliest module, we're good to go. Otherwise all our // inputs were serialized which could happen in the case, for example, that @@ -506,7 +511,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len(), ) - .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; + .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; let data = ThinData(data); @@ -628,6 +633,293 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) { llvm::set_rust_rules(true); } +fn gen_globals<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Type, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Type) { + let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry"); + let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments"); + let tptr = cx.type_ptr(); + let ti64 = cx.type_i64(); + let ti32 = cx.type_i32(); + let ti16 = cx.type_i16(); + let ti8 = cx.type_i8(); + let tarr = cx.type_array(ti32, 3); + + // @0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 + let unknown_txt = ";unknown;unknown;0;0;;"; + let c_entry_name = CString::new(unknown_txt).unwrap(); + let c_val = c_entry_name.as_bytes_with_nul(); + let initializer = crate::common::bytes_in_context(cx.llcx, c_val); + let at_zero = add_unnamed_global(&cx, &"", initializer, PrivateLinkage); + llvm::set_alignment(at_zero, Align::ONE); + + // @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8 + let struct_ident_ty = cx.type_named_struct("struct.ident_t"); + let struct_elems: Vec<&llvm::Value> = vec![cx.get_const_i32(0), cx.get_const_i32(2), cx.get_const_i32(0), cx.get_const_i32(22), at_zero]; + let struct_elems_ty: Vec<_> = struct_elems.iter().map(|&x| cx.val_ty(x)).collect(); + let initializer = crate::common::named_struct(struct_ident_ty, &struct_elems); + cx.set_struct_body(struct_ident_ty, &struct_elems_ty, false); + let at_one = add_unnamed_global(&cx, &"", initializer, PrivateLinkage); + llvm::set_alignment(at_one, Align::EIGHT); + + // coppied from LLVM + // typedef struct { + // uint64_t Reserved; + // uint16_t Version; + // uint16_t Kind; + // uint32_t Flags; + // void *Address; + // char *SymbolName; + // uint64_t Size; + // uint64_t Data; + // void *AuxAddr; + // } __tgt_offload_entry; + let entry_elements = vec![ti64, ti16, ti16, ti32, tptr, tptr, ti64, ti64, tptr]; + let kernel_elements = vec![ti32, ti32, tptr, tptr, tptr, tptr, tptr, tptr, ti64, ti64, tarr, tarr, ti32]; + + cx.set_struct_body(offload_entry_ty, &entry_elements, false); + cx.set_struct_body(kernel_arguments_ty, &kernel_elements, false); + let global = cx.declare_global("my_struct_global", offload_entry_ty); + let global = cx.declare_global("my_struct_global2", kernel_arguments_ty); + //@my_struct_global = external global %struct.__tgt_offload_entry + //@my_struct_global2 = external global %struct.__tgt_kernel_arguments + dbg!(&offload_entry_ty); + dbg!(&kernel_arguments_ty); + //LLVMTypeRef elements[9] = {i64Ty, i16Ty, i16Ty, i32Ty, ptrTy, ptrTy, i64Ty, i64Ty, ptrTy}; + //LLVMStructSetBody(structTy, elements, 9, 0); + + // New, to test memtransfer + // ; Function Attrs: nounwind + // declare void @__tgt_target_data_begin_mapper(ptr, i64, i32, ptr, ptr, ptr, ptr, ptr, ptr) #3 + // + // ; Function Attrs: nounwind + // declare void @__tgt_target_data_update_mapper(ptr, i64, i32, ptr, ptr, ptr, ptr, ptr, ptr) #3 + // + // ; Function Attrs: nounwind + // declare void @__tgt_target_data_end_mapper(ptr, i64, i32, ptr, ptr, ptr, ptr, ptr, ptr) #3 + + let mapper_begin = "__tgt_target_data_begin_mapper"; + let mapper_update = String::from("__tgt_target_data_update_mapper"); + let mapper_end = String::from("__tgt_target_data_end_mapper"); + let args = vec![tptr, ti64, ti32, tptr, tptr, tptr, tptr, tptr, tptr]; + let mapper_fn_ty = cx.type_func(&args, cx.type_void()); + let foo = crate::declare::declare_simple_fn(&cx, &mapper_begin, llvm::CallConv::CCallConv, llvm::UnnamedAddr::No, llvm::Visibility::Default, mapper_fn_ty); + let bar = crate::declare::declare_simple_fn(&cx, &mapper_update, llvm::CallConv::CCallConv, llvm::UnnamedAddr::No, llvm::Visibility::Default, mapper_fn_ty); + let baz = crate::declare::declare_simple_fn(&cx, &mapper_end, llvm::CallConv::CCallConv, llvm::UnnamedAddr::No, llvm::Visibility::Default, mapper_fn_ty); + let nounwind = llvm::AttributeKind::NoUnwind.create_attr(cx.llcx); + attributes::apply_to_llfn(foo, Function, &[nounwind]); + attributes::apply_to_llfn(bar, Function, &[nounwind]); + attributes::apply_to_llfn(baz, Function, &[nounwind]); + + (offload_entry_ty, at_one, foo, bar, baz, mapper_fn_ty) +} + +fn add_priv_unnamed_arr<'ll>(cx: &SimpleCx<'ll>, name: &str, vals: &[u64]) -> &'ll llvm::Value { + let ti64 = cx.type_i64(); + let size_ty = cx.type_array(ti64, vals.len() as u64); + let mut size_val = Vec::with_capacity(vals.len()); + for &val in vals { + size_val.push(cx.get_const_i64(val)); + } + let initializer = cx.const_array(ti64, &size_val); + add_unnamed_global(cx, name, initializer, PrivateLinkage) +} + +fn add_unnamed_global<'ll>(cx: &SimpleCx<'ll>, name: &str, initializer: &'ll llvm::Value, l: Linkage) -> &'ll llvm::Value { + let llglobal = add_global(cx, name, initializer, l); + unsafe {llvm::LLVMSetUnnamedAddress(llglobal, llvm::UnnamedAddr::Global)}; + llglobal +} + +fn add_global<'ll>(cx: &SimpleCx<'ll>, name: &str, initializer: &'ll llvm::Value, l: Linkage) -> &'ll llvm::Value { + let c_name = CString::new(name).unwrap(); + let llglobal: &'ll llvm::Value = llvm::add_global(cx.llmod, cx.val_ty(initializer), &c_name); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, l); + llvm::set_initializer(llglobal, initializer); + llglobal +} + + + +fn gen_define_handling<'ll>(cx: &'ll SimpleCx<'_>, kernel: &'ll llvm::Value, offload_entry_ty: &'ll llvm::Type, num: i64) -> &'ll llvm::Value { + let types = cx.func_params_types(cx.get_type_of_global(kernel)); + // It seems like non-pointer values are automatically mapped. So here, we focus on pointer (or + // reference) types. + let num_ptr_types = types.iter().map(|&x| matches!(cx.type_kind(x), rustc_codegen_ssa::common::TypeKind::Pointer)).count(); + + // We do not know their size anymore at this level, so hardcode a placeholder. + // A follow-up pr will track these from the frontend, where we still have Rust types. + // Then, we will be able to figure out that e.g. `&[f32;1024]` will result in 32*1024 bytes. + // I decided that 1024 bytes is a great placeholder value for now. + let o_sizes = add_priv_unnamed_arr(&cx, &format!(".offload_sizes.{num}"), &vec![1024;num_ptr_types]); + // Here we figure out whether something needs to be copied to the gpu (=1), from the gpu (=2), + // or both to and from the gpu (=3). Other values shouldn't affect us for now. + // A non-mutable reference or pointer will be 1, an array that's not read, but fully overwritten + // will be 2. For now, everything is 3, untill we have our frontend set up. + let o_types = add_priv_unnamed_arr(&cx, &format!(".offload_maptypes.{num}"), &vec![3;num_ptr_types]); + // Next: For each function, generate these three entries. A weak constant, + // the llvm.rodata entry name, and the omp_offloading_entries value + + // @.__omp_offloading_86fafab6_c40006a1__Z3fooPSt7complexIdES1_S0_m_l7.region_id = weak constant i8 0 + // @.offloading.entry_name = internal unnamed_addr constant [66 x i8] c"__omp_offloading_86fafab6_c40006a1__Z3fooPSt7complexIdES1_S0_m_l7\00", section ".llvm.rodata.offloading", align 1 + let name = format!(".kernel_{num}.region_id"); + let initializer = cx.get_const_i8(0); + let region_id = add_unnamed_global(&cx, &name, initializer, WeakAnyLinkage); + + let c_entry_name = CString::new(format!("kernel_{num}")).unwrap(); + let c_val = c_entry_name.as_bytes_with_nul(); + let foo = format!(".offloading.entry_name.{num}"); + + let initializer = crate::common::bytes_in_context(cx.llcx, c_val); + let llglobal = add_unnamed_global(&cx, &foo, initializer, InternalLinkage); + llvm::set_alignment(llglobal, Align::ONE); + let c_section_name = CString::new(".llvm.rodata.offloading").unwrap(); + llvm::set_section(llglobal, &c_section_name); + + + // Not actively used yet, for calling real kernels + let name = format!(".offloading.entry.kernel_{num}"); + let ci64_0 = cx.get_const_i64(0); + let ci16_1 = cx.get_const_i16(1); + let elems: Vec<&llvm::Value> = vec![ci64_0, ci16_1, ci16_1, cx.get_const_i32(0), region_id, llglobal, ci64_0, ci64_0, cx.const_null(cx.type_ptr())]; + + let initializer = crate::common::named_struct(offload_entry_ty, &elems); + let c_name = CString::new(name).unwrap(); + let llglobal = llvm::add_global(cx.llmod, offload_entry_ty, &c_name); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, WeakAnyLinkage); + llvm::set_initializer(llglobal, initializer); + llvm::set_alignment(llglobal, Align::ONE); + let c_section_name = CString::new(".omp_offloading_entries").unwrap(); + llvm::set_section(llglobal, &c_section_name); + // rustc + // @.offloading.entry.kernel_3 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @.kernel_3.region_id, ptr @.offloading.entry_name.3, i64 0, i64 0, ptr null }, section ".omp_offloading_entries", align 1 + // clang + // @.offloading.entry.__omp_offloading_86fafab6_c40006a1__Z3fooPSt7complexIdES1_S0_m_l7 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @.__omp_offloading_86fafab6_c40006a1__Z3fooPSt7complexIdES1_S0_m_l7.region_id, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "omp_offloading_entries", align 1 + + + // + // 1. @.offload_sizes.{num} = private unnamed_addr constant [4 x i64] [i64 8, i64 0, i64 16, i64 0] + // 2. @.offload_maptypes + // 3. @.__omp_offloading__fnc_name_ = weak constant i8 0 + // 4. @.offloading.entry_name = internal unnamed_addr constant [66 x i8] c"__omp_offloading_86fafab6_c40006a1__Z3fooPSt7complexIdES1_S0_m_l7\00", section ".llvm.rodata.offloading", align 1 + // 5. @.offloading.entry.__omp_offloading_86fafab6_c40006a1__Z3fooPSt7complexIdES1_S0_m_l7 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @.__omp_offloading_86fafab6_c40006a1__Z3fooPSt7complexIdES1_S0_m_l7.region_id, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "omp_offloading_entries", align 1 + o_types +} + + +// For each kernel *call*, we now use some of our previous declared globals to move data to and from +// the gpu. We don't have a proper frontend yet, so we assume that every call to a kernel function +// from main is intended to run on the GPU. For now, we only handle the data transfer part of it. +// If two consecutive kernels use the same memory, we still move it to the host and back to the gpu. +// Since in our frontend users (by default) don't have to specify data transfer, this is something +// we should optimize in the future! We also assume that everything should be copied back and forth, +// but sometimes we can directly zero-allocate on the device and only move back, or if something is +// immutable, we might only copy it to the device, but not back. +// +// Current steps: +// 0. Alloca some variables for the following steps +// 1. set insert point before kernel call. +// 2. generate all the GEPS and stores, to be used in 3) +// 3. generate __tgt_target_data_begin calls to move data to the GPU +// +// unchanged: keep kernel call. Later move the kernel to the GPU +// +// 4. set insert point after kernel call. +// 5. generate all the GEPS and stores, to be used in 6) +// 6. generate __tgt_target_data_end calls to move data from the GPU +fn gen_call_handling<'ll>(cx: &'ll SimpleCx<'_>, kernels: &[&'ll llvm::Value], s_ident_t: &'ll llvm::Value, begin: &'ll llvm::Value, update: &'ll llvm::Value, end: &'ll llvm::Value, fn_ty: &'ll llvm::Type, o_types: &[&'ll llvm::Value]) { + + let main_fn = cx.get_function("main"); + if let Some(main_fn) = main_fn { + let kernel_name = "kernel_1"; + let call = unsafe {llvm::LLVMRustGetFunctionCall(main_fn, kernel_name.as_c_char_ptr(), kernel_name.len())}; + let kernel_call = if call.is_some() { + dbg!("found kernel call"); + call.unwrap() + } else { + return; + }; + let kernel_call_bb = unsafe {llvm::LLVMGetInstructionParent(kernel_call)}; + let called = unsafe {llvm::LLVMGetCalledValue(kernel_call).unwrap()}; + let mut builder = SBuilder::build(cx, kernel_call_bb); + + let types = cx.func_params_types(cx.get_type_of_global(called)); + dbg!(&types); + let num_args = types.len() as u64; + let mut names: Vec<&llvm::Value> = Vec::with_capacity(num_args as usize); + + // Step 0) + unsafe{llvm::LLVMRustPositionBuilderPastAllocas(builder.llbuilder, main_fn)}; + let ty = cx.type_array(cx.type_ptr(), num_args); + // Baseptr are just the input pointer to the kernel, stored in a local alloca + let a1 = builder.my_alloca2(ty, Align::EIGHT, ".offload_baseptrs"); + // Ptrs are the result of a gep into the baseptr, at least for our trivial types. + let a2 = builder.my_alloca2(ty, Align::EIGHT, ".offload_ptrs"); + // These represent the sizes in bytes, e.g. the entry for `&[f64; 16]` will be 8*16. + let ty2 = cx.type_array(cx.type_i64(), num_args); + let a4 = builder.my_alloca2(ty2, Align::EIGHT, ".offload_sizes"); + // Now we allocate once per function param, a copy to be passed to one of our maps. + let mut vals = vec![]; + let mut geps = vec![]; + let i32_0 = cx.get_const_i32(0); + for (index, in_ty) in types.iter().enumerate() { + // get function arg, store it into the alloca, and read it. + let p = llvm::get_param(called, index as u32); + let name = llvm::get_value_name(p); + let name = str::from_utf8(name).unwrap(); + let arg_name = CString::new(format!("{name}.addr")).unwrap(); + let alloca = unsafe {llvm::LLVMBuildAlloca(builder.llbuilder, in_ty, arg_name.as_ptr())}; + builder.store(p, alloca, Align::EIGHT); + let val = builder.load(in_ty, alloca, Align::EIGHT); + let gep = builder.inbounds_gep(cx.type_f32(), val, &[i32_0]); + vals.push(val); + geps.push(gep); + } + + + // Step 1) + unsafe {llvm::LLVMRustPositionBefore(builder.llbuilder, kernel_call)}; + for i in 0..num_args { + let idx = cx.get_const_i32(i); + let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, idx]); + builder.store(vals[i as usize], gep1, Align::EIGHT); + let gep2 = builder.inbounds_gep(ty, a2, &[i32_0, idx]); + builder.store(geps[i as usize], gep2, Align::EIGHT); + let gep3 = builder.inbounds_gep(ty2, a4, &[i32_0, idx]); + builder.store(cx.get_const_i64(1024), gep3, Align::EIGHT); + } + + // Step 2) + let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]); + let gep2 = builder.inbounds_gep(ty, a2, &[i32_0, i32_0]); + let gep3 = builder.inbounds_gep(ty2, a4, &[i32_0, i32_0]); + + let nullptr = cx.const_null(cx.type_ptr()); + let o_type = o_types[0]; + let args = vec![s_ident_t, cx.get_const_i64(u64::MAX), cx.get_const_i32(num_args), gep1, gep2, gep3, o_type, nullptr, nullptr]; + builder.call(fn_ty, begin, &args, None); + + // Step 4) + unsafe {llvm::LLVMRustPositionAfter(builder.llbuilder, kernel_call)}; + + let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]); + let gep2 = builder.inbounds_gep(ty, a2, &[i32_0, i32_0]); + let gep3 = builder.inbounds_gep(ty2, a4, &[i32_0, i32_0]); + + let nullptr = cx.const_null(cx.type_ptr()); + let o_type = o_types[0]; + let args = vec![s_ident_t, cx.get_const_i64(u64::MAX), cx.get_const_i32(num_args), gep1, gep2, gep3, o_type, nullptr, nullptr]; + builder.call(fn_ty, end, &args, None); + + // call void @__tgt_target_data_begin_mapper(ptr @1, i64 -1, i32 3, ptr %27, ptr %28, ptr %29, ptr @.offload_maptypes, ptr null, ptr null) + // call void @__tgt_target_data_update_mapper(ptr @1, i64 -1, i32 2, ptr %46, ptr %47, ptr %48, ptr @.offload_maptypes.1, ptr null, ptr null) + // call void @__tgt_target_data_end_mapper(ptr @1, i64 -1, i32 3, ptr %49, ptr %50, ptr %51, ptr @.offload_maptypes, ptr null, ptr null) + // What is @1? Random but fixed: + // @0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 + // @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8 + } +} + pub(crate) fn run_pass_manager( cgcx: &CodegenContext, dcx: DiagCtxtHandle<'_>, @@ -653,6 +945,7 @@ pub(crate) fn run_pass_manager( // We then run the llvm_optimize function a second time, to optimize the code which we generated // in the enzyme differentiation pass. let enable_ad = config.autodiff.contains(&config::AutoDiff::Enable); + let enable_gpu = true;//config.offload.contains(&config::Offload::Enable); let stage = if thin { write::AutodiffStage::PreAD } else { @@ -667,6 +960,33 @@ pub(crate) fn run_pass_manager( write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; } + if cfg!(llvm_enzyme) && enable_gpu && !thin { + // first we need to add all the fun to the host module + // %struct.__tgt_offload_entry = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr } + // %struct.__tgt_kernel_arguments = type { i32, i32, ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, [3 x i32], [3 x i32], i32 } + let cx = + SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size); + if cx.get_function("gen_tgt_offload").is_some() { + + let (offload_entry_ty, at_one, begin, update, end, fn_ty) = gen_globals(&cx); + + dbg!("created struct"); + let mut o_types = vec![]; + let mut kernels = vec![]; + for num in 0..9 { + let kernel = cx.get_function(&format!("kernel_{num}")); + if let Some(kernel) = kernel{ + o_types.push(gen_define_handling(&cx, kernel, offload_entry_ty, num)); + kernels.push(kernel); + } + } + dbg!("gen_call_handling"); + gen_call_handling(&cx, &kernels, at_one, begin, update, end, fn_ty, &o_types); + } else { + dbg!("no marker found"); + } + } + if cfg!(llvm_enzyme) && enable_ad && !thin { let cx = SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size); @@ -935,7 +1255,7 @@ impl ThinLTOKeysMap { .expect("Invalid ThinLTO module key"); (module_name_to_str(name).to_string(), key) }) - .collect(); + .collect(); Self { keys } } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 167678c2ff136..eb9808f837def 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -88,6 +88,7 @@ impl<'a, 'll> SBuilder<'a, 'll> { }; call } + } impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { @@ -118,6 +119,63 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { } bx } + + pub(crate) fn my_alloca2(&mut self, ty: &'ll Type, align: Align, name: &str) -> &'ll Value { + let val = unsafe { + let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED); + llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); + // Cast to default addrspace if necessary + llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx.type_ptr(), UNNAMED) + }; + if name != "" { + let name = std::ffi::CString::new(name).unwrap(); + unsafe {llvm::set_value_name(val, &name.as_bytes())}; + } + val + } + + pub(crate) fn inbounds_gep( + &mut self, + ty: &'ll Type, + ptr: &'ll Value, + indices: &[&'ll Value], + ) -> &'ll Value { + unsafe { + llvm::LLVMBuildGEPWithNoWrapFlags( + self.llbuilder, + ty, + ptr, + indices.as_ptr(), + indices.len() as c_uint, + UNNAMED, + GEPNoWrapFlags::InBounds, + ) + } + } + + pub(crate) fn store( + &mut self, + val: &'ll Value, + ptr: &'ll Value, + align: Align, + ) -> &'ll Value { + debug!("Store {:?} -> {:?}", val, ptr); + assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer); + unsafe { + let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); + llvm::LLVMSetAlignment(store, align.bytes() as c_uint); + store + } + } + + pub(crate) fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { + unsafe { + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + llvm::LLVMSetAlignment(load, align.bytes() as c_uint); + load + } + } + } /// Empty string, to be used where LLVM expects an instruction name, indicating @@ -1261,7 +1319,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) .expect("LLVM does not have support for cleanupret"); - } + } } fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { @@ -1631,14 +1689,14 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { debug!( "type mismatch in function call of {:?}. \ Expected {:?} for param {}, got {:?}; injecting bitcast", - llfn, expected_ty, i, actual_ty + llfn, expected_ty, i, actual_ty ); self.bitcast(actual_val, expected_ty) } else { actual_val } }) - .collect(); + .collect(); Cow::Owned(casted_args) } @@ -1791,48 +1849,48 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; if self.tcx.sess.is_sanitizer_cfi_enabled() && let Some(fn_abi) = fn_abi - && is_indirect_call - { - if let Some(fn_attrs) = fn_attrs - && fn_attrs.no_sanitize.contains(SanitizerSet::CFI) + && is_indirect_call { - return; - } + if let Some(fn_attrs) = fn_attrs + && fn_attrs.no_sanitize.contains(SanitizerSet::CFI) + { + return; + } - let mut options = cfi::TypeIdOptions::empty(); - if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { - options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS); - } - if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { - options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS); - } + let mut options = cfi::TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS); + } - let typeid = if let Some(instance) = instance { - cfi::typeid_for_instance(self.tcx, instance, options) - } else { - cfi::typeid_for_fnabi(self.tcx, fn_abi, options) - }; - let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); - let dbg_loc = self.get_dbg_loc(); - - // Test whether the function pointer is associated with the type identifier. - let cond = self.type_test(llfn, typeid_metadata); - let bb_pass = self.append_sibling_block("type_test.pass"); - let bb_fail = self.append_sibling_block("type_test.fail"); - self.cond_br(cond, bb_pass, bb_fail); - - self.switch_to_block(bb_fail); - if let Some(dbg_loc) = dbg_loc { - self.set_dbg_loc(dbg_loc); - } - self.abort(); - self.unreachable(); + let typeid = if let Some(instance) = instance { + cfi::typeid_for_instance(self.tcx, instance, options) + } else { + cfi::typeid_for_fnabi(self.tcx, fn_abi, options) + }; + let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + let dbg_loc = self.get_dbg_loc(); + + // Test whether the function pointer is associated with the type identifier. + let cond = self.type_test(llfn, typeid_metadata); + let bb_pass = self.append_sibling_block("type_test.pass"); + let bb_fail = self.append_sibling_block("type_test.fail"); + self.cond_br(cond, bb_pass, bb_fail); + + self.switch_to_block(bb_fail); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } + self.abort(); + self.unreachable(); - self.switch_to_block(bb_pass); - if let Some(dbg_loc) = dbg_loc { - self.set_dbg_loc(dbg_loc); + self.switch_to_block(bb_pass); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } } - } } // Emits KCFI operand bundles. @@ -1847,31 +1905,31 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call - { - if let Some(fn_attrs) = fn_attrs - && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI) { - return None; - } + if let Some(fn_attrs) = fn_attrs + && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI) + { + return None; + } - let mut options = kcfi::TypeIdOptions::empty(); - if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { - options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS); - } - if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { - options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS); - } + let mut options = kcfi::TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS); + } - let kcfi_typeid = if let Some(instance) = instance { - kcfi::typeid_for_instance(self.tcx, instance, options) + let kcfi_typeid = if let Some(instance) = instance { + kcfi::typeid_for_instance(self.tcx, instance, options) + } else { + kcfi::typeid_for_fnabi(self.tcx, fn_abi, options) + }; + + Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)])) } else { - kcfi::typeid_for_fnabi(self.tcx, fn_abi, options) + None }; - - Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; kcfi_bundle } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 3cfa96393e920..8bf6a059d8195 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -99,14 +99,14 @@ impl<'ll, CX: Borrow>> BackendTypes for GenericCx<'ll, CX> { type DIVariable = &'ll llvm::debuginfo::DIVariable; } -impl<'ll> CodegenCx<'ll, '_> { +impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { pub(crate) fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { let len = u64::try_from(elts.len()).expect("LLVMConstArray2 elements len overflow"); unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) } } pub(crate) fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { - bytes_in_context(self.llcx, bytes) + bytes_in_context(self.llcx(), bytes) } pub(crate) fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { @@ -119,6 +119,10 @@ impl<'ll> CodegenCx<'ll, '_> { r } } + + pub(crate) fn const_null(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstNull(t) } + } } impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { @@ -373,6 +377,14 @@ pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> & } } +pub(crate) fn named_struct<'ll>( + ty: &'ll Type, + elts: &[&'ll Value], +) -> &'ll Value { + let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); + unsafe { llvm::LLVMConstNamedStruct(ty, elts.as_ptr(), len) } +} + fn struct_in_context<'ll>( llcx: &'ll llvm::Context, elts: &[&'ll Value], diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 8d6e1d8941b72..ef29494a3737d 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -685,6 +685,21 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { unsafe { llvm::LLVMConstInt(ty, n, llvm::False) } } + pub(crate) fn get_const_i32(&self, n: u64) -> &'ll Value { + let ty = unsafe { llvm::LLVMInt32TypeInContext(self.llcx()) }; + unsafe { llvm::LLVMConstInt(ty, n, llvm::False) } + } + + pub(crate) fn get_const_i16(&self, n: u64) -> &'ll Value { + let ty = unsafe { llvm::LLVMInt16TypeInContext(self.llcx()) }; + unsafe { llvm::LLVMConstInt(ty, n, llvm::False) } + } + + pub(crate) fn get_const_i8(&self, n: u64) -> &'ll Value { + let ty = unsafe { llvm::LLVMInt8TypeInContext(self.llcx()) }; + unsafe { llvm::LLVMConstInt(ty, n, llvm::False) } + } + pub(crate) fn get_function(&self, name: &str) -> Option<&'ll Value> { let name = SmallCStr::new(name); unsafe { llvm::LLVMGetNamedFunction((**self).borrow().llmod, name.as_ptr()) } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 2419ec1f88854..12e7f45be41f7 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -99,6 +99,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { ) } } + } impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { @@ -215,7 +216,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { llfn } +} +impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { /// Declare a global with an intention to define it. /// /// Use this function when you intend to define a global. This function will @@ -234,13 +237,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// Use this function when you intend to define a global without a name. pub(crate) fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod, ty) } + unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod(), ty) } } /// Gets declared value by name. pub(crate) fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { debug!("get_declared_value(name={:?})", name); - unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_c_char_ptr(), name.len()) } + unsafe { llvm::LLVMRustGetNamedValue(self.llmod(), name.as_c_char_ptr(), name.len()) } } /// Gets defined or externally defined (AvailableExternally linkage) value by diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 2ad39fc853819..097ca72bc8aa2 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -5,7 +5,7 @@ use libc::{c_char, c_uint}; use super::MetadataKindId; use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value}; -use crate::llvm::Bool; +use crate::llvm::{Bool, Builder}; #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { @@ -32,6 +32,14 @@ unsafe extern "C" { index: c_uint, kind: AttributeKind, ); + pub(crate) fn LLVMRustPositionBefore<'a>(B: &'a Builder<'_>, I: &'a Value); + pub(crate) fn LLVMRustPositionAfter<'a>(B: &'a Builder<'_>, I: &'a Value); + pub(crate) fn LLVMRustGetFunctionCall( + F: &Value, + name: *const c_char, + NameLen: libc::size_t, + ) -> Option<&Value>; + } unsafe extern "C" { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e27fbf94f341d..5a057435ac7f7 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1139,6 +1139,11 @@ unsafe extern "C" { Count: c_uint, Packed: Bool, ) -> &'a Value; + pub(crate) fn LLVMConstNamedStruct<'a>( + StructTy: &'a Type, + ConstantVals: *const &'a Value, + Count: c_uint, + ) -> &'a Value; pub(crate) fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions @@ -1209,6 +1214,8 @@ unsafe extern "C" { ) -> &'a BasicBlock; // Operations on instructions + pub(crate) fn LLVMGetInstructionParent(Inst: &Value) -> &BasicBlock; + pub(crate) fn LLVMGetCalledValue(CallInst: &Value) -> Option<&Value>; pub(crate) fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; pub(crate) fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; pub(crate) fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>; @@ -2554,6 +2561,7 @@ unsafe extern "C" { pub(crate) fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); + pub(crate) fn LLVMRustPositionBuilderPastAllocas<'a>(B: &Builder<'a>, Fn: &'a Value); pub(crate) fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); pub(crate) fn LLVMRustSetModulePICLevel(M: &Module); diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index a41ca8ce28bce..91d16f6b99e1c 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -121,6 +121,7 @@ pub struct ModuleConfig { pub emit_lifetime_markers: bool, pub llvm_plugins: Vec, pub autodiff: Vec, + pub offload: Vec, } impl ModuleConfig { @@ -270,6 +271,7 @@ impl ModuleConfig { emit_lifetime_markers: sess.emit_lifetime_markers(), llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]), autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]), + offload: if_regular!(sess.opts.unstable_opts.offload.clone(), vec![]), } } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 90aa9188c8300..3064f89f8c178 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1591,12 +1591,50 @@ extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, LLVMValueRef Dst, MaybeAlign(DstAlign), IsVolatile)); } +extern "C" void LLVMRustPositionBuilderPastAllocas(LLVMBuilderRef B, + LLVMValueRef Fn) { + Function *F = unwrap(Fn); + unwrap(B)->SetInsertPointPastAllocas(F); +} extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, LLVMBasicBlockRef BB) { auto Point = unwrap(BB)->getFirstInsertionPt(); unwrap(B)->SetInsertPoint(unwrap(BB), Point); } +extern "C" void LLVMRustPositionBefore(LLVMBuilderRef B, LLVMValueRef Instr) { + if (auto I = dyn_cast(unwrap(Instr))) { + unwrap(B)->SetInsertPoint(I); + } +} + +extern "C" void LLVMRustPositionAfter(LLVMBuilderRef B, LLVMValueRef Instr) { + if (auto I = dyn_cast(unwrap(Instr))) { + auto J = I->getNextNonDebugInstruction(); + unwrap(B)->SetInsertPoint(J); + } +} + +extern "C" LLVMValueRef +LLVMRustGetFunctionCall(LLVMValueRef Fn, const char *Name, size_t NameLen) { + auto targetName = StringRef(Name, NameLen); + Function *F = unwrap(Fn); + for (auto &BB : *F) { + for (auto &I : BB) { + if (auto *callInst = llvm::dyn_cast(&I)) { + const llvm::Function *calledFunc = callInst->getCalledFunction(); + if (calledFunc && calledFunc->getName() == targetName) { + // Found a call to the target function + llvm::errs() << "Found call: " << *callInst << "\n"; + return wrap(callInst); + } + } + } + } + + return nullptr; +} + extern "C" bool LLVMRustConstIntGetZExtValue(LLVMValueRef CV, uint64_t *value) { auto C = unwrap(CV); if (C->getBitWidth() > 64) diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 60e1b465ba96d..2c2f82461ec6f 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -226,6 +226,13 @@ pub enum CoverageLevel { Mcdc, } +// The different settings that the `-Z offload` flag can have. +#[derive(Clone, Copy, PartialEq, Hash, Debug)] +pub enum Offload { + /// Enable the llvm offload pipeline + Enable, +} + /// The different settings that the `-Z autodiff` flag can have. #[derive(Clone, Copy, PartialEq, Hash, Debug)] pub enum AutoDiff { @@ -3061,7 +3068,7 @@ pub(crate) mod dep_tracking { }; use super::{ - AutoDiff, BranchProtection, CFGuard, CFProtection, CollapseMacroDebuginfo, CoverageOptions, + AutoDiff, Offload, BranchProtection, CFGuard, CFProtection, CollapseMacroDebuginfo, CoverageOptions, CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, LtoCli, MirStripDebugInfo, NextSolverConfig, OomStrategy, OptLevel, OutFileName, @@ -3110,6 +3117,7 @@ pub(crate) mod dep_tracking { impl_dep_tracking_hash_via_hash!( AutoDiff, + Offload, bool, usize, NonZero, diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 5b4068740a159..4f9e7b4d9a88d 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -712,6 +712,7 @@ mod desc { pub(crate) const parse_list_with_polarity: &str = "a comma-separated list of strings, with elements beginning with + or -"; pub(crate) const parse_autodiff: &str = "a comma separated list of settings: `Enable`, `PrintSteps`, `PrintTA`, `PrintAA`, `PrintPerf`, `PrintModBefore`, `PrintModAfter`, `PrintModFinal`, `PrintPasses`, `NoPostopt`, `LooseTypes`, `Inline`"; + pub(crate) const parse_offload: &str = "a comma separated list of settings: `Enable`"; pub(crate) const parse_comma_list: &str = "a comma-separated list of strings"; pub(crate) const parse_opt_comma_list: &str = parse_comma_list; pub(crate) const parse_number: &str = "a number"; @@ -1343,6 +1344,27 @@ pub mod parse { } } + pub(crate) fn parse_offload(slot: &mut Vec, v: Option<&str>) -> bool { + let Some(v) = v else { + *slot = vec![]; + return true; + }; + let mut v: Vec<&str> = v.split(",").collect(); + v.sort_unstable(); + for &val in v.iter() { + let variant = match val { + "Enable" => Offload::Enable, + _ => { + // FIXME(ZuseZ4): print an error saying which value is not recognized + return false; + } + }; + slot.push(variant); + } + + true + } + pub(crate) fn parse_autodiff(slot: &mut Vec, v: Option<&str>) -> bool { let Some(v) = v else { *slot = vec![]; @@ -2372,6 +2394,11 @@ options! { "do not use unique names for text and data sections when -Z function-sections is used"), normalize_docs: bool = (false, parse_bool, [TRACKED], "normalize associated items in rustdoc when generating documentation"), + offload: Vec = (Vec::new(), parse_offload, [TRACKED], + "a list of offload flags to enable + Mandatory setting: + `=Enable` + Currently the only option available"), on_broken_pipe: OnBrokenPipe = (OnBrokenPipe::Default, parse_on_broken_pipe, [TRACKED], "behavior of std::io::ErrorKind::BrokenPipe (SIGPIPE)"), oom: OomStrategy = (OomStrategy::Abort, parse_oom_strategy, [TRACKED],