Fix calling custom call conv functions from inline asm

Author: bjorn3

example/std_example.rs

@@ -1,3 +1,4 @@
+#![cfg_attr(target_arch = "x86_64", feature(abi_custom))]
 #![feature(core_intrinsics, coroutines, coroutine_trait, repr_simd, tuple_trait, unboxed_closures)]
 #![allow(internal_features)]
 
@@ -167,6 +168,9 @@ fn main() {
 
     rust_call_abi();
 
+    #[cfg(target_arch = "x86_64")]
+    inline_asm_call_custom_abi();
+
     const fn no_str() -> Option<Box<str>> {
         None
     }
@@ -678,3 +682,17 @@ fn map(a: Option<(u8, Box<Instruction>)>) -> Option<Box<Instruction>> {
         Some((_, instr)) => Some(instr),
     }
 }
+
+#[cfg(target_arch = "x86_64")]
+fn inline_asm_call_custom_abi() {
+    use std::arch::{asm, naked_asm};
+
+    #[unsafe(naked)]
+    unsafe extern "custom" fn double() {
+        naked_asm!("add rax, rax", "ret");
+    }
+
+    let mut x: u64 = 21;
+    unsafe { asm!("call {}", sym double, inout("rax") x) };
+    assert_eq!(x, 42);
+}

src/inline_asm.rs

@@ -3,8 +3,10 @@
 use std::fmt::Write;
 
 use cranelift_codegen::isa::CallConv;
+use rustc_abi::CanonAbi;
 use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::LangItem;
+use rustc_middle::ty::layout::FnAbiOf;
 use rustc_span::sym;
 use rustc_target::asm::*;
 use rustc_target::spec::Arch;
@@ -120,21 +122,30 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                     .unwrap();
                     let symbol = fx.tcx.symbol_name(instance);
 
-                    // Pass a wrapper rather than the function itself as the function itself may not
-                    // be exported from the main codegen unit and may thus be unreachable from the
-                    // object file created by an external assembler.
-                    let wrapper_name = format!(
-                        "{}__inline_asm_{}_wrapper_n{}",
-                        fx.symbol_name,
-                        fx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
-                        fx.inline_asm_index,
-                    );
-                    fx.inline_asm_index += 1;
-                    let sig =
-                        get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
-                    create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name);
-
-                    CInlineAsmOperand::Symbol { symbol: wrapper_name }
+                    if FullyMonomorphizedLayoutCx(fx.tcx)
+                        .fn_abi_of_instance(instance, ty::List::empty())
+                        .conv
+                        == CanonAbi::Custom
+                    {
+                        // We can't create a wrapper for custom ABI functions.
+                        CInlineAsmOperand::Symbol { symbol: symbol.name.to_owned() }
+                    } else {
+                        // Pass a wrapper rather than the function itself as the function itself may not
+                        // be exported from the main codegen unit and may thus be unreachable from the
+                        // object file created by an external assembler.
+                        let wrapper_name = format!(
+                            "{}__inline_asm_{}_wrapper_n{}",
+                            fx.symbol_name,
+                            fx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
+                            fx.inline_asm_index,
+                        );
+                        fx.inline_asm_index += 1;
+                        let sig =
+                            get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
+                        create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name);
+
+                        CInlineAsmOperand::Symbol { symbol: wrapper_name }
+                    }
                 } else {
                     span_bug!(span, "invalid type for asm sym (fn)");
                 }