Skip to content

Commit 63a56f3

Browse files
jacquesguangithub-actions[bot]
jacquesguan
authored and
github-actions[bot]
committed
Automerge: [CIR][RISCV] Support Zihintntl builtins (#205468)
Include __builtin_riscv_ntl_load and __builtin_riscv_ntl_store.
2 parents 430944a + cc79c8c commit 63a56f3

4 files changed

Lines changed: 187 additions & 3 deletions

File tree

‎clang/lib/CIR/CodeGen/CIRGenBuiltinRISCV.cpp‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ CIRGenFunction::emitRISCVBuiltinExpr(unsigned builtinID, const CallExpr *e) {
3838
ASTContext::GetBuiltinTypeError error;
3939
getContext().GetBuiltinType(builtinID, error, &iceArguments);
4040
assert(error == ASTContext::GE_None && "Should not codegen an error");
41+
42+
if (builtinID == RISCV::BI__builtin_riscv_ntl_load)
43+
iceArguments |= (1 << 1);
44+
if (builtinID == RISCV::BI__builtin_riscv_ntl_store)
45+
iceArguments |= (1 << 2);
46+
4147
for (auto [idx, arg] : llvm::enumerate(e->arguments()))
4248
ops.push_back(emitScalarOrConstFoldImmArg(iceArguments, idx, arg));
4349

@@ -154,9 +160,31 @@ CIRGenFunction::emitRISCVBuiltinExpr(unsigned builtinID, const CallExpr *e) {
154160
// Zihintntl
155161
case RISCV::BI__builtin_riscv_ntl_load:
156162
case RISCV::BI__builtin_riscv_ntl_store: {
157-
cgm.errorNYI(e->getSourceRange(),
158-
std::string("unimplemented RISC-V builtin call: ") +
159-
getContext().BuiltinInfo.getName(builtinID));
163+
unsigned domainVal = 5; // Default __RISCV_NTLH_ALL
164+
unsigned domainArgNo =
165+
builtinID == RISCV::BI__builtin_riscv_ntl_load ? 1 : 2;
166+
if (e->getNumArgs() > domainArgNo) {
167+
const std::optional<llvm::APSInt> result =
168+
e->getArg(domainArgNo)->getIntegerConstantExpr(getContext());
169+
assert(result && "Expected NTLH domain argument to be a constant");
170+
domainVal = result->getZExtValue();
171+
}
172+
173+
mlir::Location loc = getLoc(e->getSourceRange());
174+
mlir::Attribute domainAttr = builder.getI32IntegerAttr(domainVal);
175+
Address addr(ops[0],
176+
cgm.getNaturalPointeeTypeAlignment(e->getArg(0)->getType()));
177+
if (builtinID == RISCV::BI__builtin_riscv_ntl_load) {
178+
auto load = builder.createLoad(loc, addr, /*isVolatile=*/false,
179+
/*isNontemporal=*/true);
180+
load->setAttr("cir.riscv_nontemporal_domain", domainAttr);
181+
return load.getResult();
182+
}
183+
184+
mlir::Value val = emitToMemory(ops[1], e->getArg(1)->getType());
185+
auto store = builder.createStore(loc, val, addr, /*isVolatile=*/false,
186+
/*isNontemporal=*/true);
187+
store->setAttr("cir.riscv_nontemporal_domain", domainAttr);
160188
return mlir::Value{};
161189
}
162190

‎clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2026,6 +2026,8 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
20262026
op.getIsVolatile(), /*isNonTemporal=*/op.getIsNontemporal(),
20272027
/*isInvariant=*/op.getInvariant(), /*isInvariantGroup=*/false, ordering,
20282028
llvmSyncScope.value_or(std::string()));
2029+
if (mlir::Attribute domain = op->getAttr("cir.riscv_nontemporal_domain"))
2030+
newLoad->setAttr("cir.riscv_nontemporal_domain", domain);
20292031

20302032
// Convert adapted result to its original type if needed.
20312033
mlir::Value result =
@@ -2081,6 +2083,8 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
20812083
op.getIsVolatile(),
20822084
/*isNonTemporal=*/op.getIsNontemporal(), /*isInvariantGroup=*/false,
20832085
memorder, llvmSyncScope.value_or(std::string()));
2086+
if (mlir::Attribute domain = op->getAttr("cir.riscv_nontemporal_domain"))
2087+
storeOp->setAttr("cir.riscv_nontemporal_domain", domain);
20842088
rewriter.replaceOp(op, storeOp);
20852089
assert(!cir::MissingFeatures::opLoadStoreTbaa());
20862090
return mlir::LogicalResult::success();

‎clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVMIR.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
#include "clang/CIR/Dialect/IR/CIRDialect.h"
1919
#include "llvm/ADT/ArrayRef.h"
2020
#include "llvm/IR/Constant.h"
21+
#include "llvm/IR/Constants.h"
2122
#include "llvm/IR/GlobalVariable.h"
23+
#include "llvm/IR/Metadata.h"
2224

2325
using namespace llvm;
2426

@@ -59,11 +61,34 @@ class CIRDialectLLVMIRTranslationInterface
5961
} else if (auto mod = dyn_cast<mlir::ModuleOp>(op)) {
6062
if (mlir::failed(amendModule(mod, attribute, moduleTranslation)))
6163
return mlir::failure();
64+
} else if (attribute.getName() == "cir.riscv_nontemporal_domain") {
65+
if (mlir::failed(amendRISCVNontemporalDomain(op, instructions, attribute,
66+
moduleTranslation)))
67+
return mlir::failure();
6268
}
6369
return mlir::success();
6470
}
6571

6672
private:
73+
mlir::LogicalResult amendRISCVNontemporalDomain(
74+
mlir::Operation *op, llvm::ArrayRef<llvm::Instruction *> instructions,
75+
mlir::NamedAttribute attribute,
76+
mlir::LLVM::ModuleTranslation &moduleTranslation) const {
77+
auto domain = mlir::dyn_cast<mlir::IntegerAttr>(attribute.getValue());
78+
if (!domain)
79+
return op->emitError()
80+
<< "expected cir.riscv_nontemporal_domain to be an integer";
81+
82+
llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext();
83+
llvm::MDNode *node = llvm::MDNode::get(
84+
llvmContext, llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
85+
llvm::Type::getInt32Ty(llvmContext),
86+
domain.getValue().getZExtValue())));
87+
for (llvm::Instruction *inst : instructions)
88+
inst->setMetadata("riscv-nontemporal-domain", node);
89+
return mlir::success();
90+
}
91+
6792
// Translate CIR function attributes to LLVM function attributes.
6893
mlir::LogicalResult
6994
amendFunction(mlir::LLVM::LLVMFuncOp func,

‎clang/test/CIR/CodeGenBuiltins/RISCV/riscv-zihintntl.c‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// REQUIRES: riscv-registered-target
2+
// RUN: %clang_cc1 -triple riscv32 -target-feature +zihintntl -fclangir -emit-cir %s -o - | FileCheck %s --check-prefix=CIR
3+
// RUN: %clang_cc1 -triple riscv64 -target-feature +zihintntl -fclangir -emit-cir %s -o - | FileCheck %s --check-prefix=CIR
4+
// RUN: %clang_cc1 -triple riscv32 -target-feature +zihintntl -fclangir -emit-llvm %s -o - | FileCheck %s --check-prefix=LLVM
5+
// RUN: %clang_cc1 -triple riscv64 -target-feature +zihintntl -fclangir -emit-llvm %s -o - | FileCheck %s --check-prefix=LLVM
6+
// RUN: %clang_cc1 -triple riscv32 -target-feature +zihintntl -emit-llvm %s -o - | FileCheck %s --check-prefix=LLVM
7+
// RUN: %clang_cc1 -triple riscv64 -target-feature +zihintntl -emit-llvm %s -o - | FileCheck %s --check-prefix=LLVM
8+
9+
#include <riscv_ntlh.h>
10+
11+
signed char sc;
12+
unsigned char uc;
13+
signed short ss;
14+
unsigned short us;
15+
signed int si;
16+
unsigned int ui;
17+
signed long long sll;
18+
unsigned long long ull;
19+
_Float16 h1, h2;
20+
float f1, f2;
21+
double d1, d2;
22+
typedef int v4si __attribute__((vector_size(16)));
23+
typedef signed short v8ss __attribute__((vector_size(16)));
24+
typedef signed char v16sc __attribute__((vector_size(16)));
25+
v4si v4si1, v4si2;
26+
v8ss v8ss1, v8ss2;
27+
v16sc v16sc1, v16sc2;
28+
29+
// CIR-LABEL: cir.func{{.*}} @ntl_load_all_sizes(
30+
// LLVM-LABEL: @ntl_load_all_sizes(
31+
void ntl_load_all_sizes(void) {
32+
// CIR: cir.load nontemporal align(1) {{.*}} {cir.riscv_nontemporal_domain = 2 : i32}
33+
// LLVM: load i8, ptr {{.*}}, align 1, !nontemporal [[NT:![0-9]+]], !riscv-nontemporal-domain [[D2:![0-9]+]]
34+
uc = __riscv_ntl_load(&sc, __RISCV_NTLH_INNERMOST_PRIVATE);
35+
// CIR: cir.load nontemporal align(1) {{.*}} {cir.riscv_nontemporal_domain = 3 : i32}
36+
// LLVM: load i8, ptr {{.*}}, align 1, !nontemporal [[NT]], !riscv-nontemporal-domain [[D3:![0-9]+]]
37+
sc = __riscv_ntl_load(&uc, __RISCV_NTLH_ALL_PRIVATE);
38+
// CIR: cir.load nontemporal align(2) {{.*}} {cir.riscv_nontemporal_domain = 4 : i32}
39+
// LLVM: load i16, ptr {{.*}}, align 2, !nontemporal [[NT]], !riscv-nontemporal-domain [[D4:![0-9]+]]
40+
us = __riscv_ntl_load(&ss, __RISCV_NTLH_INNERMOST_SHARED);
41+
// CIR: cir.load nontemporal align(2) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
42+
// LLVM: load i16, ptr {{.*}}, align 2, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5:![0-9]+]]
43+
ss = __riscv_ntl_load(&us, __RISCV_NTLH_ALL);
44+
// CIR: cir.load nontemporal align(4) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
45+
// LLVM: load i32, ptr {{.*}}, align 4, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
46+
ui = __riscv_ntl_load(&si);
47+
// CIR: cir.load nontemporal align(4) {{.*}} {cir.riscv_nontemporal_domain = 2 : i32}
48+
// LLVM: load i32, ptr {{.*}}, align 4, !nontemporal [[NT]], !riscv-nontemporal-domain [[D2]]
49+
si = __riscv_ntl_load(&ui, __RISCV_NTLH_INNERMOST_PRIVATE);
50+
// CIR: cir.load nontemporal align(8) {{.*}} {cir.riscv_nontemporal_domain = 3 : i32}
51+
// LLVM: load i64, ptr {{.*}}, align 8, !nontemporal [[NT]], !riscv-nontemporal-domain [[D3]]
52+
ull = __riscv_ntl_load(&sll, __RISCV_NTLH_ALL_PRIVATE);
53+
// CIR: cir.load nontemporal align(8) {{.*}} {cir.riscv_nontemporal_domain = 4 : i32}
54+
// LLVM: load i64, ptr {{.*}}, align 8, !nontemporal [[NT]], !riscv-nontemporal-domain [[D4]]
55+
sll = __riscv_ntl_load(&ull, __RISCV_NTLH_INNERMOST_SHARED);
56+
// CIR: cir.load nontemporal align(2) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
57+
// LLVM: load half, ptr {{.*}}, align 2, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
58+
h1 = __riscv_ntl_load(&h2, __RISCV_NTLH_ALL);
59+
// CIR: cir.load nontemporal align(4) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
60+
// LLVM: load float, ptr {{.*}}, align 4, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
61+
f1 = __riscv_ntl_load(&f2);
62+
// CIR: cir.load nontemporal align(8) {{.*}} {cir.riscv_nontemporal_domain = 2 : i32}
63+
// LLVM: load double, ptr {{.*}}, align 8, !nontemporal [[NT]], !riscv-nontemporal-domain [[D2]]
64+
d1 = __riscv_ntl_load(&d2, __RISCV_NTLH_INNERMOST_PRIVATE);
65+
// CIR: cir.load nontemporal align(16) {{.*}} {cir.riscv_nontemporal_domain = 3 : i32}
66+
// LLVM: load <4 x i32>, ptr {{.*}}, align 16, !nontemporal [[NT]], !riscv-nontemporal-domain [[D3]]
67+
v4si1 = __riscv_ntl_load(&v4si2, __RISCV_NTLH_ALL_PRIVATE);
68+
// CIR: cir.load nontemporal align(16) {{.*}} {cir.riscv_nontemporal_domain = 4 : i32}
69+
// LLVM: load <8 x i16>, ptr {{.*}}, align 16, !nontemporal [[NT]], !riscv-nontemporal-domain [[D4]]
70+
v8ss1 = __riscv_ntl_load(&v8ss2, __RISCV_NTLH_INNERMOST_SHARED);
71+
// CIR: cir.load nontemporal align(16) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
72+
// LLVM: load <16 x i8>, ptr {{.*}}, align 16, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
73+
v16sc1 = __riscv_ntl_load(&v16sc2, __RISCV_NTLH_ALL);
74+
}
75+
76+
// CIR-LABEL: cir.func{{.*}} @ntl_store_all_sizes(
77+
// LLVM-LABEL: @ntl_store_all_sizes(
78+
void ntl_store_all_sizes(void) {
79+
// CIR: cir.store nontemporal align(1) {{.*}} {cir.riscv_nontemporal_domain = 2 : i32}
80+
// LLVM: store i8 {{.*}}, ptr {{.*}}, align 1, !nontemporal [[NT]], !riscv-nontemporal-domain [[D2]]
81+
__riscv_ntl_store(&uc, 1, __RISCV_NTLH_INNERMOST_PRIVATE);
82+
// CIR: cir.store nontemporal align(1) {{.*}} {cir.riscv_nontemporal_domain = 3 : i32}
83+
// LLVM: store i8 {{.*}}, ptr {{.*}}, align 1, !nontemporal [[NT]], !riscv-nontemporal-domain [[D3]]
84+
__riscv_ntl_store(&sc, 1, __RISCV_NTLH_ALL_PRIVATE);
85+
// CIR: cir.store nontemporal align(2) {{.*}} {cir.riscv_nontemporal_domain = 4 : i32}
86+
// LLVM: store i16 {{.*}}, ptr {{.*}}, align 2, !nontemporal [[NT]], !riscv-nontemporal-domain [[D4]]
87+
__riscv_ntl_store(&us, 1, __RISCV_NTLH_INNERMOST_SHARED);
88+
// CIR: cir.store nontemporal align(2) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
89+
// LLVM: store i16 {{.*}}, ptr {{.*}}, align 2, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
90+
__riscv_ntl_store(&ss, 1, __RISCV_NTLH_ALL);
91+
// CIR: cir.store nontemporal align(4) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
92+
// LLVM: store i32 {{.*}}, ptr {{.*}}, align 4, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
93+
__riscv_ntl_store(&ui, 1);
94+
// CIR: cir.store nontemporal align(4) {{.*}} {cir.riscv_nontemporal_domain = 2 : i32}
95+
// LLVM: store i32 {{.*}}, ptr {{.*}}, align 4, !nontemporal [[NT]], !riscv-nontemporal-domain [[D2]]
96+
__riscv_ntl_store(&si, 1, __RISCV_NTLH_INNERMOST_PRIVATE);
97+
// CIR: cir.store nontemporal align(8) {{.*}} {cir.riscv_nontemporal_domain = 3 : i32}
98+
// LLVM: store i64 {{.*}}, ptr {{.*}}, align 8, !nontemporal [[NT]], !riscv-nontemporal-domain [[D3]]
99+
__riscv_ntl_store(&ull, 1, __RISCV_NTLH_ALL_PRIVATE);
100+
// CIR: cir.store nontemporal align(8) {{.*}} {cir.riscv_nontemporal_domain = 4 : i32}
101+
// LLVM: store i64 {{.*}}, ptr {{.*}}, align 8, !nontemporal [[NT]], !riscv-nontemporal-domain [[D4]]
102+
__riscv_ntl_store(&sll, 1, __RISCV_NTLH_INNERMOST_SHARED);
103+
// CIR: cir.store nontemporal align(2) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
104+
// LLVM: store half {{.*}}, ptr {{.*}}, align 2, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
105+
__riscv_ntl_store(&h1, 1.0, __RISCV_NTLH_ALL);
106+
// CIR: cir.store nontemporal align(4) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
107+
// LLVM: store float {{.*}}, ptr {{.*}}, align 4, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
108+
__riscv_ntl_store(&f1, 1.0);
109+
// CIR: cir.store nontemporal align(8) {{.*}} {cir.riscv_nontemporal_domain = 2 : i32}
110+
// LLVM: store double {{.*}}, ptr {{.*}}, align 8, !nontemporal [[NT]], !riscv-nontemporal-domain [[D2]]
111+
__riscv_ntl_store(&d1, 1.0, __RISCV_NTLH_INNERMOST_PRIVATE);
112+
// CIR: cir.store nontemporal align(16) {{.*}} {cir.riscv_nontemporal_domain = 3 : i32}
113+
// LLVM: store <4 x i32> {{.*}}, ptr {{.*}}, align 16, !nontemporal [[NT]], !riscv-nontemporal-domain [[D3]]
114+
__riscv_ntl_store(&v4si1, v4si2, __RISCV_NTLH_ALL_PRIVATE);
115+
// CIR: cir.store nontemporal align(16) {{.*}} {cir.riscv_nontemporal_domain = 4 : i32}
116+
// LLVM: store <8 x i16> {{.*}}, ptr {{.*}}, align 16, !nontemporal [[NT]], !riscv-nontemporal-domain [[D4]]
117+
__riscv_ntl_store(&v8ss1, v8ss2, __RISCV_NTLH_INNERMOST_SHARED);
118+
// CIR: cir.store nontemporal align(16) {{.*}} {cir.riscv_nontemporal_domain = 5 : i32}
119+
// LLVM: store <16 x i8> {{.*}}, ptr {{.*}}, align 16, !nontemporal [[NT]], !riscv-nontemporal-domain [[D5]]
120+
__riscv_ntl_store(&v16sc1, v16sc2, __RISCV_NTLH_ALL);
121+
}
122+
123+
// LLVM-DAG: [[NT]] = !{i32 1}
124+
// LLVM-DAG: [[D2]] = !{i32 2}
125+
// LLVM-DAG: [[D3]] = !{i32 3}
126+
// LLVM-DAG: [[D4]] = !{i32 4}
127+
// LLVM-DAG: [[D5]] = !{i32 5}

0 commit comments

Comments
 (0)