Skip to content

Commit 62eeb9b

Browse files
ZentrikZentrik
committed
Cherry-pick "Fix late gc lowering pass for vector intrinsics #55864"
1 parent 0d46fa6 commit 62eeb9b

File tree

3 files changed

+75
-25
lines changed

3 files changed

+75
-25
lines changed

Diff for: ‎src/llvm-late-gc-lowering.cpp

+59-24
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// This file is a part of Julia. License is MIT: https://julialang.org/license
22

33
#include "llvm-gc-interface-passes.h"
4+
#include "llvm/IR/Intrinsics.h"
5+
#include "llvm/Support/Casting.h"
46

57
#define DEBUG_TYPE "late_lower_gcroot"
68

@@ -171,39 +173,54 @@ static std::pair<Value*,int> FindBaseValue(const State &S, Value *V, bool UseCac
171173
(void)LI;
172174
break;
173175
}
174-
else if (auto II = dyn_cast<IntrinsicInst>(CurrentV)) {
176+
else if (dyn_cast<IntrinsicInst>(CurrentV) != nullptr &&
177+
(cast<IntrinsicInst>(CurrentV)->getIntrinsicID() == Intrinsic::masked_load ||
178+
cast<IntrinsicInst>(CurrentV)->getIntrinsicID() == Intrinsic::masked_gather)) {
175179
// Some intrinsics behave like LoadInst followed by a SelectInst
176180
// This should never happen in a derived addrspace (since those cannot be stored to memory)
177181
// so we don't need to lift these operations, but we do need to check if it's loaded and continue walking the base pointer
178-
if (II->getIntrinsicID() == Intrinsic::masked_load ||
179-
II->getIntrinsicID() == Intrinsic::masked_gather) {
180-
if (auto VTy = dyn_cast<VectorType>(II->getType())) {
181-
if (hasLoadedTy(VTy->getElementType())) {
182-
Value *Mask = II->getOperand(2);
183-
Value *Passthrough = II->getOperand(3);
184-
if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue()) {
185-
assert(isa<UndefValue>(Passthrough) && "unimplemented");
186-
(void)Passthrough;
182+
auto II = dyn_cast<IntrinsicInst>(CurrentV);
183+
if (auto VTy = dyn_cast<VectorType>(II->getType())) {
184+
if (hasLoadedTy(VTy->getElementType())) {
185+
Value *Mask = II->getOperand(2);
186+
Value *Passthrough = II->getOperand(3);
187+
if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue()) {
188+
assert(isa<UndefValue>(Passthrough) && "unimplemented");
189+
(void)Passthrough;
190+
}
191+
CurrentV = II->getOperand(0);
192+
if (II->getIntrinsicID() == Intrinsic::masked_load) {
193+
fld_idx = -1;
194+
if (!isSpecialPtr(CurrentV->getType())) {
195+
CurrentV = ConstantPointerNull::get(PointerType::get(V->getContext(), 0));
187196
}
188-
CurrentV = II->getOperand(0);
189-
if (II->getIntrinsicID() == Intrinsic::masked_load) {
190-
fld_idx = -1;
191-
if (!isSpecialPtr(CurrentV->getType())) {
197+
} else {
198+
if (auto VTy2 = dyn_cast<VectorType>(CurrentV->getType())) {
199+
if (!isSpecialPtr(VTy2->getElementType())) {
192200
CurrentV = ConstantPointerNull::get(PointerType::get(V->getContext(), 0));
193-
}
194-
} else {
195-
if (auto VTy2 = dyn_cast<VectorType>(CurrentV->getType())) {
196-
if (!isSpecialPtr(VTy2->getElementType())) {
197-
CurrentV = ConstantPointerNull::get(PointerType::get(V->getContext(), 0));
198-
fld_idx = -1;
199-
}
201+
fld_idx = -1;
200202
}
201203
}
202-
continue;
203204
}
205+
continue;
206+
}
207+
}
208+
// In general a load terminates a walk
209+
break;
210+
}
211+
else if (dyn_cast<IntrinsicInst>(CurrentV) != nullptr && cast<IntrinsicInst>(CurrentV)->getIntrinsicID() == Intrinsic::vector_extract) {
212+
auto II = dyn_cast<IntrinsicInst>(CurrentV);
213+
if (auto VTy = dyn_cast<VectorType>(II->getType())) {
214+
if (hasLoadedTy(VTy->getElementType())) {
215+
Value *Idx = II->getOperand(1);
216+
if (!isa<ConstantInt>(Idx)) {
217+
assert(isa<UndefValue>(Idx) && "unimplemented");
218+
(void)Idx;
219+
}
220+
CurrentV = II->getOperand(0);
221+
fld_idx = -1;
222+
continue;
204223
}
205-
// In general a load terminates a walk
206-
break;
207224
}
208225
}
209226
else if (auto CI = dyn_cast<CallInst>(CurrentV)) {
@@ -530,6 +547,20 @@ SmallVector<int, 0> LateLowerGCFrame::NumberAllBase(State &S, Value *CurrentV) {
530547
Numbers = NumberAll(S, IEI->getOperand(0));
531548
int ElNumber = Number(S, IEI->getOperand(1));
532549
Numbers[idx] = ElNumber;
550+
} else if (dyn_cast<IntrinsicInst>(CurrentV) != nullptr && dyn_cast<IntrinsicInst>(CurrentV)->getIntrinsicID() == Intrinsic::vector_insert) {
551+
auto *VII = cast<IntrinsicInst>(CurrentV);
552+
// Vector insert is a bit like a shuffle so use the same approach
553+
SmallVector<int, 0> Numbers1 = NumberAll(S, VII->getOperand(0));
554+
SmallVector<int, 0> Numbers2 = NumberAll(S, VII->getOperand(1));
555+
unsigned first_idx = cast<ConstantInt>(VII->getOperand(2))->getZExtValue();
556+
for (unsigned i = 0; i < Numbers1.size(); ++i) {
557+
if (i < first_idx)
558+
Numbers.push_back(Numbers1[i]);
559+
else if (i - first_idx < Numbers2.size())
560+
Numbers.push_back(Numbers2[i - first_idx]);
561+
else
562+
Numbers.push_back(Numbers1[i]);
563+
}
533564
} else if (auto *IVI = dyn_cast<InsertValueInst>(CurrentV)) {
534565
Numbers = NumberAll(S, IVI->getAggregateOperand());
535566
auto Tracked = TrackCompositeType(IVI->getType());
@@ -1206,6 +1237,10 @@ State LateLowerGCFrame::LocalScan(Function &F) {
12061237
}
12071238
}
12081239
}
1240+
if (II->getIntrinsicID() == Intrinsic::vector_extract || II->getIntrinsicID() == Intrinsic::vector_insert) {
1241+
// These are not real defs
1242+
continue;
1243+
}
12091244
}
12101245
auto callee = CI->getCalledFunction();
12111246
if (callee && callee == typeof_func) {

Diff for: ‎test/llvmpasses/image-codegen.jl

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# RUN: export JULIA_LLVM_ARGS="--print-before=loop-vectorize --print-module-scope"
33
# RUN: rm -rf %t
44
# RUN: mkdir %t
5-
# RUN: julia --image-codegen --startup-file=no %s 2> %t/output.txt
5+
# RUN: julia --image-codegen -t1,0 --startup-file=no %s 2> %t/output.txt
66
# RUN: FileCheck %s < %t/output.txt
77

88
# COM: checks that global variables compiled in imaging codegen

Diff for: ‎test/llvmpasses/late-lower-gc.ll

+15
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,21 @@ define {} addrspace(10)* @gclift_switch({} addrspace(13)* addrspace(10)* %input,
164164
ret {} addrspace(10)* %ret
165165
}
166166

167+
; Shouldn't hang
168+
define void @vector_insert(<4 x {} addrspace(10)* > %0, <2 x {} addrspace(10)* > %1) {
169+
top:
170+
%pgcstack = call {}*** @julia.get_pgcstack()
171+
%2 = call <4 x {} addrspace(10)*> @llvm.vector.insert.v4p10.v2p10(<4 x {} addrspace(10)*> %0, <2 x {} addrspace(10)*> %1, i64 2)
172+
ret void
173+
}
174+
175+
define void @vector_extract(<4 x {} addrspace(10)* > %0, <2 x {} addrspace(10)* > %1) {
176+
top:
177+
%pgcstack = call {}*** @julia.get_pgcstack()
178+
%2 = call <2 x {} addrspace(10)*> @llvm.vector.extract.v2p10.v4p10(<4 x {} addrspace(10)* > %0, i64 2)
179+
ret void
180+
}
181+
167182
define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) {
168183
%v2 = call {}*** @julia.get_pgcstack()
169184
%e0 = extractvalue [2 x {} addrspace(10)* addrspace(11)*] %ar, 0

0 commit comments

Comments
 (0)