Skip to content

Commit e22295b

Browse files
snickolls-armsnickolls-arm
committed
Revamp the generation of runtime division checks on ARM64
Fixes #64795 This patch introduces a new compilation phase that passes over the GenTrees looking for GT_DIV/GT_UDIV nodes on integral types, and morphs the code to introduce the necessary conformance checks (overflow/divide-by-zero) early on in the compilation pipeline. Currently these are added during the Emit phase, meaning optimizations don't run on any code introduced. The aim is to allow the compiler to make decisions on code position and instruction selection for these checks. For example on ARM64 this enables certain scenarios to choose the cbz instruction over cmp/beq, can lead to more compact code. It also allows some of the comparisons in the checks to be hoisted out of loops.
1 parent 4de59c4 commit e22295b

12 files changed

+382
-97
lines changed

‎src/coreclr/jit/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ endif(CLR_CMAKE_TARGET_WIN32)
9393
set( JIT_SOURCES
9494
abi.cpp
9595
alloc.cpp
96+
arithchecks.cpp
9697
assertionprop.cpp
9798
bitset.cpp
9899
block.cpp

‎src/coreclr/jit/arithchecks.cpp

+270
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
#include "jitpch.h"
5+
#include "compiler.h"
6+
#include "gentree.h"
7+
#ifdef _MSC_VER
8+
#pragma hdrstop
9+
#endif
10+
11+
jitstd::vector<unsigned>& Compiler::getVisitedDivNodes()
12+
{
13+
if (visitedDivNodes == nullptr)
14+
{
15+
visitedDivNodes = new (getAllocator(CMK_Unknown)) jitstd::vector<unsigned>(getAllocator(CMK_Unknown));
16+
}
17+
return *visitedDivNodes;
18+
}
19+
20+
jitstd::vector<Compiler::ThrowHelper>& Compiler::getThrowHelperSet()
21+
{
22+
if (throwHelperSet == nullptr)
23+
{
24+
throwHelperSet = new (getAllocator(CMK_Unknown)) jitstd::vector<ThrowHelper>(getAllocator(CMK_Unknown));
25+
}
26+
return *throwHelperSet;
27+
}
28+
29+
void Compiler::fgInsertConditionalThrowException(Compiler::TreeCoord& code, SpecialCodeKind codeKind)
30+
{
31+
// Create a new basic block, splitting before the code containing the division.
32+
Statement* newStmt = nullptr;
33+
GenTree** splitNodeUse = nullptr;
34+
JITDUMP("[%06d] contains the division\n", code.tree->gtTreeID);
35+
BasicBlock* divBlock = fgSplitBlockBeforeTree(code.block, code.stmt, code.tree, &newStmt, &splitNodeUse);
36+
GenTree* tree = *splitNodeUse;
37+
JITDUMP("[%06d] contains the division after split\n", tree->gtTreeID);
38+
BasicBlock* checkBlock = code.block;
39+
40+
// The checking block is added after the original block that has been split. This is for convenience of
41+
// adding the checking statement to an empty basic block.
42+
checkBlock = fgNewBBafter(BBJ_ALWAYS, code.block, true);
43+
fgRedirectTargetEdge(code.block, checkBlock);
44+
45+
GenTree* dividend = tree->gtGetOp1();
46+
GenTree* divisor = tree->gtGetOp2();
47+
assert(varTypeIsIntegral(divisor) && varTypeIsIntegral(dividend));
48+
49+
GenTree* divisorCopy = nullptr;
50+
GenTree* dividendCopy = nullptr;
51+
52+
GenTree* condition = nullptr;
53+
CorInfoHelpFunc helper = CORINFO_HELP_UNDEF;
54+
switch (codeKind)
55+
{
56+
case SCK_DIV_BY_ZERO:
57+
{
58+
assert(codeKind == SCK_DIV_BY_ZERO);
59+
impCloneExpr(divisor, &divisorCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("cloned for runtime check"));
60+
61+
// (divisor == 0)
62+
condition = gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(0, genActualType(divisorCopy)));
63+
64+
helper = CORINFO_HELP_THROWDIVZERO;
65+
break;
66+
}
67+
case SCK_OVERFLOW:
68+
{
69+
impCloneExpr(divisor, &divisorCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("cloned for runtime check"));
70+
impCloneExpr(dividend, &dividendCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("cloned for runtime check"));
71+
72+
// (dividend < 0 && divisor == -1)
73+
GenTreeOp* const divisorIsMinusOne =
74+
gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(-1, genActualType(divisorCopy)));
75+
GenTreeOp* const dividendIsNegative =
76+
gtNewOperNode(GT_LT, TYP_INT, dividendCopy, gtNewIconNode(0, genActualType(dividendCopy)));
77+
GenTreeOp* const combinedTest = gtNewOperNode(GT_AND, TYP_INT, divisorIsMinusOne, dividendIsNegative);
78+
condition = gtNewOperNode(GT_EQ, TYP_INT, combinedTest, gtNewTrue());
79+
80+
helper = CORINFO_HELP_OVERFLOW;
81+
break;
82+
}
83+
default:
84+
noway_assert(!"unexpected exception code kind");
85+
}
86+
assert(condition != nullptr);
87+
assert(helper != CORINFO_HELP_UNDEF);
88+
89+
// Add another block near the checking block, this will be the throw block.
90+
// This block is explicitly placed after the checking block to produce debuggable code
91+
// when optimizations aren't enabled.
92+
BasicBlock* throwBlock = nullptr;
93+
94+
// Search for an already created throw block. We don't do this for debug code
95+
// so the throw code will stay in-line.
96+
if (!opts.compDbgCode)
97+
{
98+
for (const ThrowHelper& th : getThrowHelperSet())
99+
{
100+
if (th.For(helper, divBlock))
101+
{
102+
throwBlock = th.Block();
103+
break;
104+
}
105+
}
106+
}
107+
108+
if (throwBlock == nullptr)
109+
{
110+
// Insert a throw block close by, in the same EH region.
111+
throwBlock = fgNewBBafter(BBJ_THROW, checkBlock, true);
112+
113+
// Add exception raising code to the throw block.
114+
GenTreeCall* const thrw = gtNewHelperCallNode(helper, TYP_VOID);
115+
Statement* throwStmt = gtNewStmt(thrw);
116+
fgInsertStmtAtEnd(throwBlock, throwStmt);
117+
assert(throwBlock->lastStmt() == throwStmt);
118+
119+
if (!opts.compDbgCode)
120+
getThrowHelperSet().push_back(ThrowHelper(helper, throwBlock));
121+
}
122+
123+
// Add the compare and jump to the end of the checking block.
124+
condition->gtFlags |= GTF_RELOP_JMP_USED;
125+
GenTree* const cndJmp = gtNewOperNode(GT_JTRUE, TYP_VOID, condition);
126+
Statement* jmpStmt = gtNewStmt(cndJmp);
127+
fgInsertStmtAtEnd(checkBlock, jmpStmt);
128+
assert(checkBlock->lastStmt() == jmpStmt);
129+
130+
// Set the true edge to the exception block, false edge to the block where division occurs.
131+
FlowEdge* trueEdge = fgAddRefPred(throwBlock, checkBlock);
132+
FlowEdge* falseEdge = fgAddRefPred(divBlock, checkBlock);
133+
134+
// The exception case is considered unlikely in comparison to normal control flow.
135+
trueEdge->setLikelihood(0.01);
136+
falseEdge->setLikelihood(0.99);
137+
138+
checkBlock->SetCond(trueEdge, falseEdge);
139+
140+
JITDUMP(FMT_BB " contains the division\n", divBlock->bbNum);
141+
JITDUMP(FMT_BB " contains the check\n", checkBlock->bbNum);
142+
JITDUMP(FMT_BB " contains the throw\n", throwBlock->bbNum);
143+
144+
code.block = divBlock;
145+
code.stmt = divBlock->firstStmt();
146+
code.tree = tree;
147+
}
148+
149+
bool Compiler::fgInsertDivisionChecks(BasicBlock* block, Statement* stmt, GenTree* tree)
150+
{
151+
assert(tree->OperIs(GT_DIV, GT_UDIV));
152+
153+
// Check we haven't processed this DIV before.
154+
bool found = false;
155+
for (const unsigned& id : getVisitedDivNodes())
156+
{
157+
if (tree->gtTreeID == id)
158+
{
159+
found = true;
160+
break;
161+
}
162+
}
163+
if (found)
164+
return false;
165+
166+
// Only integral divisions will throw
167+
if (!(varTypeIsIntegral(tree->gtGetOp1()) && varTypeIsIntegral(tree->gtGetOp2())))
168+
{
169+
return false;
170+
}
171+
172+
bool modified = false;
173+
174+
TreeCoord divCode(block, stmt, tree);
175+
176+
// Check for division by -1 when the dividend is signed.
177+
// (NegativeVal / -1) => OverflowException
178+
if ((divCode.tree->OperExceptions(this) & ExceptionSetFlags::ArithmeticException) != ExceptionSetFlags::None)
179+
{
180+
assert(tree->OperIs(GT_DIV));
181+
JITDUMP(FMT_BB " " FMT_STMT " needs a signed overflow check\n", divCode.block->bbNum, divCode.stmt->GetID());
182+
fgInsertConditionalThrowException(divCode, SCK_OVERFLOW);
183+
modified = true;
184+
}
185+
186+
// Check for division by 0 - both unsigned and signed are affected.
187+
// (AnyVal / 0) => DivideByZeroException
188+
if ((divCode.tree->OperExceptions(this) & ExceptionSetFlags::DivideByZeroException) != ExceptionSetFlags::None)
189+
{
190+
JITDUMP(FMT_BB " " FMT_STMT " needs a divide by zero check\n", divCode.block->bbNum, divCode.stmt->GetID());
191+
fgInsertConditionalThrowException(divCode, SCK_DIV_BY_ZERO);
192+
modified = true;
193+
}
194+
195+
JITDUMP("Marking [%06d] as visited.\n", divCode.tree->gtTreeID);
196+
getVisitedDivNodes().push_back(divCode.tree->gtTreeID);
197+
198+
return modified;
199+
}
200+
201+
PhaseStatus Compiler::fgInsertArithmeticExceptions()
202+
{
203+
#if defined(TARGET_ARM64)
204+
struct Visitor : GenTreeVisitor<Visitor>
205+
{
206+
enum
207+
{
208+
DoPreOrder = true,
209+
};
210+
211+
Visitor(Compiler* comp, BasicBlock* block, Statement* stmt)
212+
: GenTreeVisitor(comp)
213+
, m_block(block)
214+
, m_stmt(stmt)
215+
{
216+
}
217+
218+
fgWalkResult PreOrderVisit(GenTree** use, GenTree* user)
219+
{
220+
if (!(*use)->OperIs(GT_DIV, GT_UDIV))
221+
{
222+
return WALK_CONTINUE;
223+
}
224+
225+
modified = m_compiler->fgInsertDivisionChecks(m_block, m_stmt, *use);
226+
227+
// If checks were inserted, the tree was split and the graph was modified,
228+
// so traversal needs to start again.
229+
if (modified)
230+
{
231+
return WALK_ABORT;
232+
}
233+
234+
return WALK_CONTINUE;
235+
}
236+
237+
BasicBlock* m_block;
238+
Statement* m_stmt;
239+
public:
240+
bool modified = false;
241+
};
242+
243+
bool lastModified = false;
244+
bool anyModified = false;
245+
do
246+
{
247+
lastModified = false;
248+
for (BasicBlock* const block : Blocks())
249+
{
250+
for (Statement* const stmt : block->Statements())
251+
{
252+
GenTree* tree = stmt->GetRootNode();
253+
Visitor visitor(this, block, stmt);
254+
visitor.WalkTree(&tree, nullptr);
255+
lastModified = visitor.modified;
256+
anyModified |= lastModified;
257+
258+
if (lastModified)
259+
break;
260+
}
261+
if (lastModified)
262+
break;
263+
}
264+
} while (lastModified);
265+
266+
return anyModified ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
267+
#else
268+
return PhaseStatus::MODIFIED_NOTHING;
269+
#endif
270+
}

‎src/coreclr/jit/codegen.h

+2
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,9 @@ class CodeGen final : public CodeGenInterface
794794
void genCodeForLongUMod(GenTreeOp* node);
795795
#endif // TARGET_X86
796796

797+
#if !defined(TARGET_ARM64)
797798
void genCodeForDivMod(GenTreeOp* treeNode);
799+
#endif
798800
void genCodeForMul(GenTreeOp* treeNode);
799801
void genCodeForIncSaturate(GenTree* treeNode);
800802
void genCodeForMulHi(GenTreeOp* treeNode);

‎src/coreclr/jit/codegenarm64.cpp

-89
Original file line numberDiff line numberDiff line change
@@ -3535,95 +3535,6 @@ void CodeGen::genCodeForBswap(GenTree* tree)
35353535
genProduceReg(tree);
35363536
}
35373537

3538-
//------------------------------------------------------------------------
3539-
// genCodeForDivMod: Produce code for a GT_DIV/GT_UDIV node. We don't see MOD:
3540-
// (1) integer MOD is morphed into a sequence of sub, mul, div in fgMorph;
3541-
// (2) float/double MOD is morphed into a helper call by front-end.
3542-
//
3543-
// Arguments:
3544-
// tree - the node
3545-
//
3546-
void CodeGen::genCodeForDivMod(GenTreeOp* tree)
3547-
{
3548-
assert(tree->OperIs(GT_DIV, GT_UDIV));
3549-
3550-
var_types targetType = tree->TypeGet();
3551-
emitter* emit = GetEmitter();
3552-
3553-
genConsumeOperands(tree);
3554-
3555-
if (varTypeIsFloating(targetType))
3556-
{
3557-
// Floating point divide never raises an exception
3558-
genCodeForBinary(tree);
3559-
}
3560-
else // an integer divide operation
3561-
{
3562-
// Generate the require runtime checks for GT_DIV or GT_UDIV.
3563-
3564-
GenTree* divisorOp = tree->gtGetOp2();
3565-
emitAttr size = EA_ATTR(genTypeSize(genActualType(tree->TypeGet())));
3566-
3567-
regNumber divisorReg = divisorOp->GetRegNum();
3568-
3569-
ExceptionSetFlags exSetFlags = tree->OperExceptions(compiler);
3570-
3571-
// (AnyVal / 0) => DivideByZeroException
3572-
if ((exSetFlags & ExceptionSetFlags::DivideByZeroException) != ExceptionSetFlags::None)
3573-
{
3574-
if (divisorOp->IsIntegralConst(0))
3575-
{
3576-
// We unconditionally throw a divide by zero exception
3577-
genJumpToThrowHlpBlk(EJ_jmp, SCK_DIV_BY_ZERO);
3578-
3579-
// We still need to call genProduceReg
3580-
genProduceReg(tree);
3581-
3582-
return;
3583-
}
3584-
else
3585-
{
3586-
// Check if the divisor is zero throw a DivideByZeroException
3587-
emit->emitIns_R_I(INS_cmp, size, divisorReg, 0);
3588-
genJumpToThrowHlpBlk(EJ_eq, SCK_DIV_BY_ZERO);
3589-
}
3590-
}
3591-
3592-
// (MinInt / -1) => ArithmeticException
3593-
if ((exSetFlags & ExceptionSetFlags::ArithmeticException) != ExceptionSetFlags::None)
3594-
{
3595-
// Signed-division might overflow.
3596-
3597-
assert(tree->OperIs(GT_DIV));
3598-
assert(!divisorOp->IsIntegralConst(0));
3599-
3600-
BasicBlock* sdivLabel = genCreateTempLabel();
3601-
GenTree* dividendOp = tree->gtGetOp1();
3602-
3603-
// Check if the divisor is not -1 branch to 'sdivLabel'
3604-
emit->emitIns_R_I(INS_cmp, size, divisorReg, -1);
3605-
3606-
inst_JMP(EJ_ne, sdivLabel);
3607-
// If control flow continues past here the 'divisorReg' is known to be -1
3608-
3609-
regNumber dividendReg = dividendOp->GetRegNum();
3610-
// At this point the divisor is known to be -1
3611-
//
3612-
// Issue the 'cmp dividendReg, 1' instruction.
3613-
// This is an alias to 'subs zr, dividendReg, 1' on ARM64 itself.
3614-
// This will set the V (overflow) flags only when dividendReg is MinInt
3615-
//
3616-
emit->emitIns_R_I(INS_cmp, size, dividendReg, 1);
3617-
genJumpToThrowHlpBlk(EJ_vs, SCK_ARITH_EXCPN); // if the V flags is set throw
3618-
// ArithmeticException
3619-
3620-
genDefineTempLabel(sdivLabel);
3621-
}
3622-
3623-
genCodeForBinary(tree); // Generate the sdiv instruction
3624-
}
3625-
}
3626-
36273538
// Generate code for CpObj nodes which copy structs that have interleaved
36283539
// GC pointers.
36293540
// For this case we'll generate a sequence of loads/stores in the case of struct

0 commit comments

Comments
 (0)