[Asan][RISCV] Enhance getTgtMemIntrinsic() to allow Asan instrument t… #135198
+2,443
−25
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
…arget memory intrinsic operations.
|
@llvm/pr-subscribers-llvm-analysis @llvm/pr-subscribers-backend-amdgpu Author: Hank Chang (HankChang736) Changes…arget intrinsic. Previously, AddressSanitizer treated target memory intrinsics as black boxes, which prevented ASan from accurately instrumenting them. This patch enhance TargetTransformInfo (TTI) hook - Note :
Fixing 133361. Patch is 179.69 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/135198.diff 13 Files Affected:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 2efca0d1d754f..47613d0ea3fe8 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -31,6 +31,7 @@
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/InstructionCost.h"
+#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
#include <functional>
#include <optional>
#include <utility>
@@ -1665,7 +1666,9 @@ class TargetTransformInfo {
/// will contain additional information - whether the intrinsic may write
/// or read to memory, volatility and the pointer. Info is undefined
/// if false is returned.
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const;
/// \returns The maximum element size, in bytes, for an element
/// unordered-atomic memory intrinsic.
@@ -2289,8 +2292,9 @@ class TargetTransformInfo::Concept {
getAddressComputationCost(Type *Ty, ScalarEvolution *SE, const SCEV *Ptr) = 0;
virtual InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) = 0;
- virtual bool getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) = 0;
+ virtual bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) = 0;
virtual unsigned getAtomicMemIntrinsicMaxElementSize() const = 0;
virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
Type *ExpectedType) = 0;
@@ -3060,9 +3064,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) override {
return Impl.getCostOfKeepingLiveOverCall(Tys);
}
- bool getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) override {
- return Impl.getTgtMemIntrinsic(Inst, Info);
+ bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand>
+ *Interesting = nullptr) override {
+ return Impl.getTgtMemIntrinsic(Inst, Info, Interesting);
}
unsigned getAtomicMemIntrinsicMaxElementSize() const override {
return Impl.getAtomicMemIntrinsicMaxElementSize();
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 3fe0a9101fdee..9ed8d48c255a6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -904,7 +904,9 @@ class TargetTransformInfoImplBase {
return 0;
}
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const {
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const {
return false;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 4fea4e5711f5a..88ab23cd9abcf 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1268,9 +1268,10 @@ TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
}
-bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) const {
- return TTIImpl->getTgtMemIntrinsic(Inst, Info);
+bool TargetTransformInfo::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
+ return TTIImpl->getTgtMemIntrinsic(Inst, Info, Interesting);
}
unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b1d8277182add..007e1912ab9dd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4874,8 +4874,9 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
}
}
-bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) {
+bool AArch64TTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) {
switch (Inst->getIntrinsicID()) {
default:
break;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ae0df6b895ec8..0068fbcb0176d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -259,7 +259,9 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
Type *ExpectedType);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr);
bool isElementTypeLegalForScalableVector(Type *Ty) const {
if (Ty->isPointerTy())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 09f7877b13b3a..11e1db1806dc5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -494,8 +494,9 @@ unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
return 8;
}
-bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) const {
+bool GCNTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
switch (Inst->getIntrinsicID()) {
case Intrinsic::amdgcn_ds_ordered_add:
case Intrinsic::amdgcn_ds_ordered_swap: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index f5062070ac6f4..6bd5a5d8ec51b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -151,7 +151,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
std::optional<uint32_t> AtomicCpySize) const;
unsigned getMaxInterleaveFactor(ElementCount VF);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const;
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 3a4c2fcad8c83..c35dd7aec0be4 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -974,8 +974,9 @@ bool PPCTTIImpl::shouldBuildRelLookupTables() const {
return BaseT::shouldBuildRelLookupTables();
}
-bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) {
+bool PPCTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) {
switch (Inst->getIntrinsicID()) {
case Intrinsic::ppc_altivec_lvx:
case Intrinsic::ppc_altivec_lvxl:
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index bf3ddad134e14..65f6bd98f4624 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -70,7 +70,9 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI,
DominatorTree *DT, AssumptionCache *AC,
TargetLibraryInfo *LibInfo);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr);
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 9b91de36a688a..53ad495ff2195 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -15,6 +15,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/PatternMatch.h"
#include <cmath>
#include <optional>
@@ -43,6 +44,87 @@ static cl::opt<unsigned>
"vectorization while tail-folding."),
cl::init(5), cl::Hidden);
+bool RISCVTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
+ const DataLayout &DL = getDataLayout();
+ Intrinsic::ID IID = Inst->getIntrinsicID();
+ LLVMContext &C = Inst->getContext();
+ bool HasMask = false;
+ bool HasInteresting = (Interesting == nullptr) ? false : true;
+
+ switch (IID) {
+ case Intrinsic::riscv_vle_mask:
+ case Intrinsic::riscv_vse_mask:
+ HasMask = true;
+ [[fallthrough]];
+ case Intrinsic::riscv_vle:
+ case Intrinsic::riscv_vse: {
+ // Intrinsic interface:
+ // riscv_vle(merge, ptr, vl)
+ // riscv_vle_mask(merge, ptr, mask, vl, policy)
+ // riscv_vse(val, ptr, vl)
+ // riscv_vse_mask(val, ptr, mask, vl, policy)
+ bool IsWrite = Inst->getType()->isVoidTy();
+ Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
+ const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
+ unsigned VLIndex = RVVIInfo->VLOperand;
+ unsigned PtrOperandNo = VLIndex - 1 - HasMask;
+ MaybeAlign Alignment =
+ Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
+ Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
+ Value *Mask = ConstantInt::getTrue(MaskType);
+ if (HasMask)
+ Mask = Inst->getArgOperand(VLIndex - 1);
+ Value *EVL = Inst->getArgOperand(VLIndex);
+ if (HasInteresting)
+ Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment,
+ Mask, EVL);
+ return true;
+ }
+ case Intrinsic::riscv_vlse_mask:
+ case Intrinsic::riscv_vsse_mask:
+ HasMask = true;
+ [[fallthrough]];
+ case Intrinsic::riscv_vlse:
+ case Intrinsic::riscv_vsse: {
+ // Intrinsic interface:
+ // riscv_vlse(merge, ptr, stride, vl)
+ // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
+ // riscv_vsse(val, ptr, stride, vl)
+ // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
+ bool IsWrite = Inst->getType()->isVoidTy();
+ Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
+ const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
+ unsigned VLIndex = RVVIInfo->VLOperand;
+ unsigned PtrOperandNo = VLIndex - 2 - HasMask;
+ MaybeAlign Alignment =
+ Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
+
+ Value *Stride = Inst->getArgOperand(PtrOperandNo + 1);
+ // Use the pointer alignment as the element alignment if the stride is a
+ // multiple of the pointer alignment. Otherwise, the element alignment
+ // should be the greatest common divisor of pointer alignment and stride.
+ // For simplicity, just consider unalignment for elements.
+ unsigned PointerAlign = Alignment.valueOrOne().value();
+ if (!isa<ConstantInt>(Stride) ||
+ cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
+ Alignment = Align(1);
+
+ Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
+ Value *Mask = ConstantInt::getTrue(MaskType);
+ if (HasMask)
+ Mask = Inst->getArgOperand(VLIndex - 1);
+ Value *EVL = Inst->getArgOperand(VLIndex);
+ if (HasInteresting)
+ Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment,
+ Mask, EVL, Stride);
+ return true;
+ }
+ }
+ return false;
+}
+
InstructionCost
RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
TTI::TargetCostKind CostKind) {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index c61dd1507f168..5d534b908f8f8 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -150,6 +150,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
TTI::PeelingPreferences &PP);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const;
+
unsigned getMinVectorRegisterBitWidth() const {
return ST->useRVVForFixedLengthVectors() ? 16 : 0;
}
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index a4c6c61e57998..5d8889809f845 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -29,6 +29,7 @@
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/StackSafetyAnalysis.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/Demangle/Demangle.h"
@@ -799,7 +800,8 @@ struct AddressSanitizer {
bool ignoreAccess(Instruction *Inst, Value *Ptr);
void getInterestingMemoryOperands(
- Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
+ Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
+ const TargetTransformInfo *TTI);
void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
InterestingMemoryOperand &O, bool UseCalls,
@@ -839,7 +841,8 @@ struct AddressSanitizer {
void instrumentMemIntrinsic(MemIntrinsic *MI, RuntimeCallInserter &RTCI);
Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
bool suppressInstrumentationSiteForDebug(int &Instrumented);
- bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI);
+ bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI);
bool maybeInsertAsanInitAtFunctionEntry(Function &F);
bool maybeInsertDynamicShadowAtFunctionEntry(Function &F);
void markEscapedLocalAllocas(Function &F);
@@ -1327,7 +1330,8 @@ PreservedAnalyses AddressSanitizerPass::run(Module &M,
Options.MaxInlinePoisoningSize, Options.CompileKernel, Options.Recover,
Options.UseAfterScope, Options.UseAfterReturn);
const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
- Modified |= FunctionSanitizer.instrumentFunction(F, &TLI);
+ const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(F);
+ Modified |= FunctionSanitizer.instrumentFunction(F, &TLI, &TTI);
}
Modified |= ModuleSanitizer.instrumentModule();
if (!Modified)
@@ -1465,7 +1469,8 @@ bool AddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
}
void AddressSanitizer::getInterestingMemoryOperands(
- Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
+ Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
+ const TargetTransformInfo *TTI) {
// Do not instrument the load fetching the dynamic shadow address.
if (LocalDynamicShadow == I)
return;
@@ -1583,6 +1588,11 @@ void AddressSanitizer::getInterestingMemoryOperands(
break;
}
default:
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ MemIntrinsicInfo DummyInfo;
+ if (TTI->getTgtMemIntrinsic(II, DummyInfo, &Interesting))
+ return;
+ }
for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
ignoreAccess(I, CI->getArgOperand(ArgNo)))
@@ -2998,7 +3008,8 @@ bool AddressSanitizer::suppressInstrumentationSiteForDebug(int &Instrumented) {
}
bool AddressSanitizer::instrumentFunction(Function &F,
- const TargetLibraryInfo *TLI) {
+ const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI) {
bool FunctionModified = false;
// Do not apply any instrumentation for naked functions.
@@ -3051,7 +3062,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
if (Inst.hasMetadata(LLVMContext::MD_nosanitize))
continue;
SmallVector<InterestingMemoryOperand, 1> InterestingOperands;
- getInterestingMemoryOperands(&Inst, InterestingOperands);
+ getInterestingMemoryOperands(&Inst, InterestingOperands, TTI);
if (!InterestingOperands.empty()) {
for (auto &Operand : InterestingOperands) {
diff --git a/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll
new file mode 100644
index 0000000000000..7571679659533
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll
@@ -0,0 +1,2304 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=asan \
+; RUN: -asan-instrumentation-with-call-threshold=0 -S | FileCheck %s
+
+declare <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32(
+ <vscale x 1 x i32>,
+ <vscale x 1 x i32>*,
+ i64)
+define <vscale x 1 x i32> @intrinsic_vle_v_nxv1i32_nxv1i32(<vscale x 1 x i32>* align 4 %0, i64 %1) sanitize_address {
+; CHECK-LABEL: @intrinsic_vle_v_nxv1i32_nxv1i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr @__asan_shadow_memory_dynamic_address, align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[TMP1:%.*]], 0
+; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP12:%.*]]
+; CHECK: 4:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[TMP5]])
+; CHECK-NEXT: br label [[DOTSPLIT:%.*]]
+; CHECK: .split:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP4]] ], [ [[IV_NEXT:%.*]], [[TMP11:%.*]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 1 x i1> splat (i1 true), i64 [[IV]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP11]]
+; CHECK: 8:
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr <vscale x 1 x i32>, ptr [[TMP0:%.*]], i64 0, i64 [[IV]]
+; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP9]] to i64
+; CHECK-NEXT: call void @__asan_load4(i64 [[TMP10]])
+; CHECK-NEXT: br label [[TMP11]]
+; CHECK: 11:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP6]]
+; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]]
+; CHECK: .split.split:
+; CHECK-NEXT: br label [[TMP12]]
+; CHECK: 12:
+; CHECK-NEXT: [[A:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32.i64(<vscale x 1 x i32> undef, ptr [[TMP0]], i64 [[TMP1]])
+; CHECK-NE...
[truncated]
|
|
@llvm/pr-subscribers-llvm-transforms Author: Hank Chang (HankChang736) Changes…arget intrinsic. Previously, AddressSanitizer treated target memory intrinsics as black boxes, which prevented ASan from accurately instrumenting them. This patch enhance TargetTransformInfo (TTI) hook - Note :
Fixing 133361. Patch is 179.69 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/135198.diff 13 Files Affected:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 2efca0d1d754f..47613d0ea3fe8 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -31,6 +31,7 @@
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/InstructionCost.h"
+#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
#include <functional>
#include <optional>
#include <utility>
@@ -1665,7 +1666,9 @@ class TargetTransformInfo {
/// will contain additional information - whether the intrinsic may write
/// or read to memory, volatility and the pointer. Info is undefined
/// if false is returned.
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const;
/// \returns The maximum element size, in bytes, for an element
/// unordered-atomic memory intrinsic.
@@ -2289,8 +2292,9 @@ class TargetTransformInfo::Concept {
getAddressComputationCost(Type *Ty, ScalarEvolution *SE, const SCEV *Ptr) = 0;
virtual InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) = 0;
- virtual bool getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) = 0;
+ virtual bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) = 0;
virtual unsigned getAtomicMemIntrinsicMaxElementSize() const = 0;
virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
Type *ExpectedType) = 0;
@@ -3060,9 +3064,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) override {
return Impl.getCostOfKeepingLiveOverCall(Tys);
}
- bool getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) override {
- return Impl.getTgtMemIntrinsic(Inst, Info);
+ bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand>
+ *Interesting = nullptr) override {
+ return Impl.getTgtMemIntrinsic(Inst, Info, Interesting);
}
unsigned getAtomicMemIntrinsicMaxElementSize() const override {
return Impl.getAtomicMemIntrinsicMaxElementSize();
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 3fe0a9101fdee..9ed8d48c255a6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -904,7 +904,9 @@ class TargetTransformInfoImplBase {
return 0;
}
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const {
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const {
return false;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 4fea4e5711f5a..88ab23cd9abcf 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1268,9 +1268,10 @@ TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
}
-bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) const {
- return TTIImpl->getTgtMemIntrinsic(Inst, Info);
+bool TargetTransformInfo::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
+ return TTIImpl->getTgtMemIntrinsic(Inst, Info, Interesting);
}
unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b1d8277182add..007e1912ab9dd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4874,8 +4874,9 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
}
}
-bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) {
+bool AArch64TTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) {
switch (Inst->getIntrinsicID()) {
default:
break;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ae0df6b895ec8..0068fbcb0176d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -259,7 +259,9 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
Type *ExpectedType);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr);
bool isElementTypeLegalForScalableVector(Type *Ty) const {
if (Ty->isPointerTy())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 09f7877b13b3a..11e1db1806dc5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -494,8 +494,9 @@ unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
return 8;
}
-bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) const {
+bool GCNTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
switch (Inst->getIntrinsicID()) {
case Intrinsic::amdgcn_ds_ordered_add:
case Intrinsic::amdgcn_ds_ordered_swap: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index f5062070ac6f4..6bd5a5d8ec51b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -151,7 +151,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
std::optional<uint32_t> AtomicCpySize) const;
unsigned getMaxInterleaveFactor(ElementCount VF);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const;
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 3a4c2fcad8c83..c35dd7aec0be4 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -974,8 +974,9 @@ bool PPCTTIImpl::shouldBuildRelLookupTables() const {
return BaseT::shouldBuildRelLookupTables();
}
-bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) {
+bool PPCTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) {
switch (Inst->getIntrinsicID()) {
case Intrinsic::ppc_altivec_lvx:
case Intrinsic::ppc_altivec_lvxl:
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index bf3ddad134e14..65f6bd98f4624 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -70,7 +70,9 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI,
DominatorTree *DT, AssumptionCache *AC,
TargetLibraryInfo *LibInfo);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr);
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 9b91de36a688a..53ad495ff2195 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -15,6 +15,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/PatternMatch.h"
#include <cmath>
#include <optional>
@@ -43,6 +44,87 @@ static cl::opt<unsigned>
"vectorization while tail-folding."),
cl::init(5), cl::Hidden);
+bool RISCVTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
+ const DataLayout &DL = getDataLayout();
+ Intrinsic::ID IID = Inst->getIntrinsicID();
+ LLVMContext &C = Inst->getContext();
+ bool HasMask = false;
+ bool HasInteresting = (Interesting == nullptr) ? false : true;
+
+ switch (IID) {
+ case Intrinsic::riscv_vle_mask:
+ case Intrinsic::riscv_vse_mask:
+ HasMask = true;
+ [[fallthrough]];
+ case Intrinsic::riscv_vle:
+ case Intrinsic::riscv_vse: {
+ // Intrinsic interface:
+ // riscv_vle(merge, ptr, vl)
+ // riscv_vle_mask(merge, ptr, mask, vl, policy)
+ // riscv_vse(val, ptr, vl)
+ // riscv_vse_mask(val, ptr, mask, vl, policy)
+ bool IsWrite = Inst->getType()->isVoidTy();
+ Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
+ const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
+ unsigned VLIndex = RVVIInfo->VLOperand;
+ unsigned PtrOperandNo = VLIndex - 1 - HasMask;
+ MaybeAlign Alignment =
+ Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
+ Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
+ Value *Mask = ConstantInt::getTrue(MaskType);
+ if (HasMask)
+ Mask = Inst->getArgOperand(VLIndex - 1);
+ Value *EVL = Inst->getArgOperand(VLIndex);
+ if (HasInteresting)
+ Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment,
+ Mask, EVL);
+ return true;
+ }
+ case Intrinsic::riscv_vlse_mask:
+ case Intrinsic::riscv_vsse_mask:
+ HasMask = true;
+ [[fallthrough]];
+ case Intrinsic::riscv_vlse:
+ case Intrinsic::riscv_vsse: {
+ // Intrinsic interface:
+ // riscv_vlse(merge, ptr, stride, vl)
+ // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
+ // riscv_vsse(val, ptr, stride, vl)
+ // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
+ bool IsWrite = Inst->getType()->isVoidTy();
+ Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
+ const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
+ unsigned VLIndex = RVVIInfo->VLOperand;
+ unsigned PtrOperandNo = VLIndex - 2 - HasMask;
+ MaybeAlign Alignment =
+ Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
+
+ Value *Stride = Inst->getArgOperand(PtrOperandNo + 1);
+ // Use the pointer alignment as the element alignment if the stride is a
+ // multiple of the pointer alignment. Otherwise, the element alignment
+ // should be the greatest common divisor of pointer alignment and stride.
+ // For simplicity, just consider unalignment for elements.
+ unsigned PointerAlign = Alignment.valueOrOne().value();
+ if (!isa<ConstantInt>(Stride) ||
+ cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
+ Alignment = Align(1);
+
+ Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
+ Value *Mask = ConstantInt::getTrue(MaskType);
+ if (HasMask)
+ Mask = Inst->getArgOperand(VLIndex - 1);
+ Value *EVL = Inst->getArgOperand(VLIndex);
+ if (HasInteresting)
+ Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment,
+ Mask, EVL, Stride);
+ return true;
+ }
+ }
+ return false;
+}
+
InstructionCost
RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
TTI::TargetCostKind CostKind) {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index c61dd1507f168..5d534b908f8f8 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -150,6 +150,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
TTI::PeelingPreferences &PP);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const;
+
unsigned getMinVectorRegisterBitWidth() const {
return ST->useRVVForFixedLengthVectors() ? 16 : 0;
}
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index a4c6c61e57998..5d8889809f845 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -29,6 +29,7 @@
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/StackSafetyAnalysis.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/Demangle/Demangle.h"
@@ -799,7 +800,8 @@ struct AddressSanitizer {
bool ignoreAccess(Instruction *Inst, Value *Ptr);
void getInterestingMemoryOperands(
- Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
+ Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
+ const TargetTransformInfo *TTI);
void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
InterestingMemoryOperand &O, bool UseCalls,
@@ -839,7 +841,8 @@ struct AddressSanitizer {
void instrumentMemIntrinsic(MemIntrinsic *MI, RuntimeCallInserter &RTCI);
Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
bool suppressInstrumentationSiteForDebug(int &Instrumented);
- bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI);
+ bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI);
bool maybeInsertAsanInitAtFunctionEntry(Function &F);
bool maybeInsertDynamicShadowAtFunctionEntry(Function &F);
void markEscapedLocalAllocas(Function &F);
@@ -1327,7 +1330,8 @@ PreservedAnalyses AddressSanitizerPass::run(Module &M,
Options.MaxInlinePoisoningSize, Options.CompileKernel, Options.Recover,
Options.UseAfterScope, Options.UseAfterReturn);
const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
- Modified |= FunctionSanitizer.instrumentFunction(F, &TLI);
+ const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(F);
+ Modified |= FunctionSanitizer.instrumentFunction(F, &TLI, &TTI);
}
Modified |= ModuleSanitizer.instrumentModule();
if (!Modified)
@@ -1465,7 +1469,8 @@ bool AddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
}
void AddressSanitizer::getInterestingMemoryOperands(
- Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
+ Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
+ const TargetTransformInfo *TTI) {
// Do not instrument the load fetching the dynamic shadow address.
if (LocalDynamicShadow == I)
return;
@@ -1583,6 +1588,11 @@ void AddressSanitizer::getInterestingMemoryOperands(
break;
}
default:
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ MemIntrinsicInfo DummyInfo;
+ if (TTI->getTgtMemIntrinsic(II, DummyInfo, &Interesting))
+ return;
+ }
for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
ignoreAccess(I, CI->getArgOperand(ArgNo)))
@@ -2998,7 +3008,8 @@ bool AddressSanitizer::suppressInstrumentationSiteForDebug(int &Instrumented) {
}
bool AddressSanitizer::instrumentFunction(Function &F,
- const TargetLibraryInfo *TLI) {
+ const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI) {
bool FunctionModified = false;
// Do not apply any instrumentation for naked functions.
@@ -3051,7 +3062,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
if (Inst.hasMetadata(LLVMContext::MD_nosanitize))
continue;
SmallVector<InterestingMemoryOperand, 1> InterestingOperands;
- getInterestingMemoryOperands(&Inst, InterestingOperands);
+ getInterestingMemoryOperands(&Inst, InterestingOperands, TTI);
if (!InterestingOperands.empty()) {
for (auto &Operand : InterestingOperands) {
diff --git a/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll
new file mode 100644
index 0000000000000..7571679659533
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll
@@ -0,0 +1,2304 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=asan \
+; RUN: -asan-instrumentation-with-call-threshold=0 -S | FileCheck %s
+
+declare <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32(
+ <vscale x 1 x i32>,
+ <vscale x 1 x i32>*,
+ i64)
+define <vscale x 1 x i32> @intrinsic_vle_v_nxv1i32_nxv1i32(<vscale x 1 x i32>* align 4 %0, i64 %1) sanitize_address {
+; CHECK-LABEL: @intrinsic_vle_v_nxv1i32_nxv1i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr @__asan_shadow_memory_dynamic_address, align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[TMP1:%.*]], 0
+; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP12:%.*]]
+; CHECK: 4:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[TMP5]])
+; CHECK-NEXT: br label [[DOTSPLIT:%.*]]
+; CHECK: .split:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP4]] ], [ [[IV_NEXT:%.*]], [[TMP11:%.*]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 1 x i1> splat (i1 true), i64 [[IV]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP11]]
+; CHECK: 8:
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr <vscale x 1 x i32>, ptr [[TMP0:%.*]], i64 0, i64 [[IV]]
+; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP9]] to i64
+; CHECK-NEXT: call void @__asan_load4(i64 [[TMP10]])
+; CHECK-NEXT: br label [[TMP11]]
+; CHECK: 11:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP6]]
+; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]]
+; CHECK: .split.split:
+; CHECK-NEXT: br label [[TMP12]]
+; CHECK: 12:
+; CHECK-NEXT: [[A:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32.i64(<vscale x 1 x i32> undef, ptr [[TMP0]], i64 [[TMP1]])
+; CHECK-NE...
[truncated]
|
|
@llvm/pr-subscribers-backend-powerpc Author: Hank Chang (HankChang736) Changes…arget intrinsic. Previously, AddressSanitizer treated target memory intrinsics as black boxes, which prevented ASan from accurately instrumenting them. This patch enhance TargetTransformInfo (TTI) hook - Note :
Fixing 133361. Patch is 179.69 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/135198.diff 13 Files Affected:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 2efca0d1d754f..47613d0ea3fe8 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -31,6 +31,7 @@
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/InstructionCost.h"
+#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
#include <functional>
#include <optional>
#include <utility>
@@ -1665,7 +1666,9 @@ class TargetTransformInfo {
/// will contain additional information - whether the intrinsic may write
/// or read to memory, volatility and the pointer. Info is undefined
/// if false is returned.
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const;
/// \returns The maximum element size, in bytes, for an element
/// unordered-atomic memory intrinsic.
@@ -2289,8 +2292,9 @@ class TargetTransformInfo::Concept {
getAddressComputationCost(Type *Ty, ScalarEvolution *SE, const SCEV *Ptr) = 0;
virtual InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) = 0;
- virtual bool getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) = 0;
+ virtual bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) = 0;
virtual unsigned getAtomicMemIntrinsicMaxElementSize() const = 0;
virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
Type *ExpectedType) = 0;
@@ -3060,9 +3064,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) override {
return Impl.getCostOfKeepingLiveOverCall(Tys);
}
- bool getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) override {
- return Impl.getTgtMemIntrinsic(Inst, Info);
+ bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand>
+ *Interesting = nullptr) override {
+ return Impl.getTgtMemIntrinsic(Inst, Info, Interesting);
}
unsigned getAtomicMemIntrinsicMaxElementSize() const override {
return Impl.getAtomicMemIntrinsicMaxElementSize();
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 3fe0a9101fdee..9ed8d48c255a6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -904,7 +904,9 @@ class TargetTransformInfoImplBase {
return 0;
}
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const {
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const {
return false;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 4fea4e5711f5a..88ab23cd9abcf 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1268,9 +1268,10 @@ TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
}
-bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) const {
- return TTIImpl->getTgtMemIntrinsic(Inst, Info);
+bool TargetTransformInfo::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
+ return TTIImpl->getTgtMemIntrinsic(Inst, Info, Interesting);
}
unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b1d8277182add..007e1912ab9dd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4874,8 +4874,9 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
}
}
-bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) {
+bool AArch64TTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) {
switch (Inst->getIntrinsicID()) {
default:
break;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ae0df6b895ec8..0068fbcb0176d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -259,7 +259,9 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
Type *ExpectedType);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr);
bool isElementTypeLegalForScalableVector(Type *Ty) const {
if (Ty->isPointerTy())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 09f7877b13b3a..11e1db1806dc5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -494,8 +494,9 @@ unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
return 8;
}
-bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) const {
+bool GCNTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
switch (Inst->getIntrinsicID()) {
case Intrinsic::amdgcn_ds_ordered_add:
case Intrinsic::amdgcn_ds_ordered_swap: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index f5062070ac6f4..6bd5a5d8ec51b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -151,7 +151,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
std::optional<uint32_t> AtomicCpySize) const;
unsigned getMaxInterleaveFactor(ElementCount VF);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr) const;
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 3a4c2fcad8c83..c35dd7aec0be4 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -974,8 +974,9 @@ bool PPCTTIImpl::shouldBuildRelLookupTables() const {
return BaseT::shouldBuildRelLookupTables();
}
-bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
- MemIntrinsicInfo &Info) {
+bool PPCTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) {
switch (Inst->getIntrinsicID()) {
case Intrinsic::ppc_altivec_lvx:
case Intrinsic::ppc_altivec_lvxl:
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index bf3ddad134e14..65f6bd98f4624 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -70,7 +70,9 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI,
DominatorTree *DT, AssumptionCache *AC,
TargetLibraryInfo *LibInfo);
- bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting = nullptr);
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 9b91de36a688a..53ad495ff2195 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -15,6 +15,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/PatternMatch.h"
#include <cmath>
#include <optional>
@@ -43,6 +44,87 @@ static cl::opt<unsigned>
"vectorization while tail-folding."),
cl::init(5), cl::Hidden);
+bool RISCVTTIImpl::getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const {
+ const DataLayout &DL = getDataLayout();
+ Intrinsic::ID IID = Inst->getIntrinsicID();
+ LLVMContext &C = Inst->getContext();
+ bool HasMask = false;
+ bool HasInteresting = (Interesting == nullptr) ? false : true;
+
+ switch (IID) {
+ case Intrinsic::riscv_vle_mask:
+ case Intrinsic::riscv_vse_mask:
+ HasMask = true;
+ [[fallthrough]];
+ case Intrinsic::riscv_vle:
+ case Intrinsic::riscv_vse: {
+ // Intrinsic interface:
+ // riscv_vle(merge, ptr, vl)
+ // riscv_vle_mask(merge, ptr, mask, vl, policy)
+ // riscv_vse(val, ptr, vl)
+ // riscv_vse_mask(val, ptr, mask, vl, policy)
+ bool IsWrite = Inst->getType()->isVoidTy();
+ Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
+ const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
+ unsigned VLIndex = RVVIInfo->VLOperand;
+ unsigned PtrOperandNo = VLIndex - 1 - HasMask;
+ MaybeAlign Alignment =
+ Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
+ Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
+ Value *Mask = ConstantInt::getTrue(MaskType);
+ if (HasMask)
+ Mask = Inst->getArgOperand(VLIndex - 1);
+ Value *EVL = Inst->getArgOperand(VLIndex);
+ if (HasInteresting)
+ Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment,
+ Mask, EVL);
+ return true;
+ }
+ case Intrinsic::riscv_vlse_mask:
+ case Intrinsic::riscv_vsse_mask:
+ HasMask = true;
+ [[fallthrough]];
+ case Intrinsic::riscv_vlse:
+ case Intrinsic::riscv_vsse: {
+ // Intrinsic interface:
+ // riscv_vlse(merge, ptr, stride, vl)
+ // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
+ // riscv_vsse(val, ptr, stride, vl)
+ // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
+ bool IsWrite = Inst->getType()->isVoidTy();
+ Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
+ const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
+ unsigned VLIndex = RVVIInfo->VLOperand;
+ unsigned PtrOperandNo = VLIndex - 2 - HasMask;
+ MaybeAlign Alignment =
+ Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
+
+ Value *Stride = Inst->getArgOperand(PtrOperandNo + 1);
+ // Use the pointer alignment as the element alignment if the stride is a
+ // multiple of the pointer alignment. Otherwise, the element alignment
+ // should be the greatest common divisor of pointer alignment and stride.
+ // For simplicity, just consider unalignment for elements.
+ unsigned PointerAlign = Alignment.valueOrOne().value();
+ if (!isa<ConstantInt>(Stride) ||
+ cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
+ Alignment = Align(1);
+
+ Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
+ Value *Mask = ConstantInt::getTrue(MaskType);
+ if (HasMask)
+ Mask = Inst->getArgOperand(VLIndex - 1);
+ Value *EVL = Inst->getArgOperand(VLIndex);
+ if (HasInteresting)
+ Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment,
+ Mask, EVL, Stride);
+ return true;
+ }
+ }
+ return false;
+}
+
InstructionCost
RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
TTI::TargetCostKind CostKind) {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index c61dd1507f168..5d534b908f8f8 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -150,6 +150,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
TTI::PeelingPreferences &PP);
+ bool getTgtMemIntrinsic(
+ IntrinsicInst *Inst, MemIntrinsicInfo &Info,
+ SmallVectorImpl<InterestingMemoryOperand> *Interesting) const;
+
unsigned getMinVectorRegisterBitWidth() const {
return ST->useRVVForFixedLengthVectors() ? 16 : 0;
}
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index a4c6c61e57998..5d8889809f845 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -29,6 +29,7 @@
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/StackSafetyAnalysis.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/Demangle/Demangle.h"
@@ -799,7 +800,8 @@ struct AddressSanitizer {
bool ignoreAccess(Instruction *Inst, Value *Ptr);
void getInterestingMemoryOperands(
- Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
+ Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
+ const TargetTransformInfo *TTI);
void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
InterestingMemoryOperand &O, bool UseCalls,
@@ -839,7 +841,8 @@ struct AddressSanitizer {
void instrumentMemIntrinsic(MemIntrinsic *MI, RuntimeCallInserter &RTCI);
Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
bool suppressInstrumentationSiteForDebug(int &Instrumented);
- bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI);
+ bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI);
bool maybeInsertAsanInitAtFunctionEntry(Function &F);
bool maybeInsertDynamicShadowAtFunctionEntry(Function &F);
void markEscapedLocalAllocas(Function &F);
@@ -1327,7 +1330,8 @@ PreservedAnalyses AddressSanitizerPass::run(Module &M,
Options.MaxInlinePoisoningSize, Options.CompileKernel, Options.Recover,
Options.UseAfterScope, Options.UseAfterReturn);
const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
- Modified |= FunctionSanitizer.instrumentFunction(F, &TLI);
+ const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(F);
+ Modified |= FunctionSanitizer.instrumentFunction(F, &TLI, &TTI);
}
Modified |= ModuleSanitizer.instrumentModule();
if (!Modified)
@@ -1465,7 +1469,8 @@ bool AddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
}
void AddressSanitizer::getInterestingMemoryOperands(
- Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
+ Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting,
+ const TargetTransformInfo *TTI) {
// Do not instrument the load fetching the dynamic shadow address.
if (LocalDynamicShadow == I)
return;
@@ -1583,6 +1588,11 @@ void AddressSanitizer::getInterestingMemoryOperands(
break;
}
default:
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ MemIntrinsicInfo DummyInfo;
+ if (TTI->getTgtMemIntrinsic(II, DummyInfo, &Interesting))
+ return;
+ }
for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
ignoreAccess(I, CI->getArgOperand(ArgNo)))
@@ -2998,7 +3008,8 @@ bool AddressSanitizer::suppressInstrumentationSiteForDebug(int &Instrumented) {
}
bool AddressSanitizer::instrumentFunction(Function &F,
- const TargetLibraryInfo *TLI) {
+ const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI) {
bool FunctionModified = false;
// Do not apply any instrumentation for naked functions.
@@ -3051,7 +3062,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
if (Inst.hasMetadata(LLVMContext::MD_nosanitize))
continue;
SmallVector<InterestingMemoryOperand, 1> InterestingOperands;
- getInterestingMemoryOperands(&Inst, InterestingOperands);
+ getInterestingMemoryOperands(&Inst, InterestingOperands, TTI);
if (!InterestingOperands.empty()) {
for (auto &Operand : InterestingOperands) {
diff --git a/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll
new file mode 100644
index 0000000000000..7571679659533
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/RISCV/asan-rvv-intrinsics.ll
@@ -0,0 +1,2304 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=asan \
+; RUN: -asan-instrumentation-with-call-threshold=0 -S | FileCheck %s
+
+declare <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32(
+ <vscale x 1 x i32>,
+ <vscale x 1 x i32>*,
+ i64)
+define <vscale x 1 x i32> @intrinsic_vle_v_nxv1i32_nxv1i32(<vscale x 1 x i32>* align 4 %0, i64 %1) sanitize_address {
+; CHECK-LABEL: @intrinsic_vle_v_nxv1i32_nxv1i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr @__asan_shadow_memory_dynamic_address, align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[TMP1:%.*]], 0
+; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP12:%.*]]
+; CHECK: 4:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[TMP5]])
+; CHECK-NEXT: br label [[DOTSPLIT:%.*]]
+; CHECK: .split:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP4]] ], [ [[IV_NEXT:%.*]], [[TMP11:%.*]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 1 x i1> splat (i1 true), i64 [[IV]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP11]]
+; CHECK: 8:
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr <vscale x 1 x i32>, ptr [[TMP0:%.*]], i64 0, i64 [[IV]]
+; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP9]] to i64
+; CHECK-NEXT: call void @__asan_load4(i64 [[TMP10]])
+; CHECK-NEXT: br label [[TMP11]]
+; CHECK: 11:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP6]]
+; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]]
+; CHECK: .split.split:
+; CHECK-NEXT: br label [[TMP12]]
+; CHECK: 12:
+; CHECK-NEXT: [[A:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32.i64(<vscale x 1 x i32> undef, ptr [[TMP0]], i64 [[TMP1]])
+; CHECK-NE...
[truncated]
|
arichardson
approved these changes
Apr 10, 2025
| MemIntrinsicInfo &Info) const { | ||
| bool GCNTTIImpl::getTgtMemIntrinsic( | ||
| IntrinsicInst *Inst, MemIntrinsicInfo &Info, | ||
| SmallVectorImpl<InterestingMemoryOperand> *Interesting) const { |
There was a problem hiding this comment.
Fill the InterestingMemoryOperand as well if available?
There was a problem hiding this comment.
@arichardson Thank you for the feedback, after few days of thinking. I'd like to rewrite this patch since this getTgtMemIntrinsic() return true means it fill MemIntrinsicInfo. However, I break this rule and it might cause further error in EarlyCSE where getTgtMemIntrinsic is used if we fill InterestingMemoryOperand. I will try to find a way to combine these, maybe change the return value to optional value or another better way.
| MemIntrinsicInfo &Info) { | ||
| bool AArch64TTIImpl::getTgtMemIntrinsic( | ||
| IntrinsicInst *Inst, MemIntrinsicInfo &Info, | ||
| SmallVectorImpl<InterestingMemoryOperand> *Interesting) { |
There was a problem hiding this comment.
Fill the InterestingMemoryOperand as well if available?
arichardson
requested changes
Apr 10, 2025
kito-cheng
reviewed
Apr 11, 2025
| if (HasInteresting) | ||
| Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment, | ||
| Mask, EVL); | ||
| return true; |
There was a problem hiding this comment.
Return true means we have fill up MemIntrinsicInfo, but seems like we didn't...
| if (HasInteresting) | ||
| Interesting->emplace_back(Inst, PtrOperandNo, IsWrite, Ty, Alignment, | ||
| Mask, EVL, Stride); | ||
| return true; |
| @@ -1665,7 +1666,9 @@ class TargetTransformInfo { | |||
| /// will contain additional information - whether the intrinsic may write | |||
| /// or read to memory, volatility and the pointer. Info is undefined | |||
| /// if false is returned. | |||
There was a problem hiding this comment.
We may update the comment to indicate the relationship between return value and Interesting?
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
…arget intrinsic.
Previously, AddressSanitizer treated target memory intrinsics as black boxes, which prevented ASan from accurately instrumenting them. This patch enhance TargetTransformInfo (TTI) hook -
getTgtMemIntrinsicto allow targets to describe the behavior of their memory intrinsics to AddressSanitizer.Note :
Fixing 133361.