Skip to content

Commit b93a981

Browse files
jakobbotschjakobbotsch
authored
JIT: Switch remainder of lowering to new ABI info (#112544)
This required introducing some new information in `ABIPassingSegment` for stack-passed segments about whether they own the full stack slot or not. Apple arm64 notoriously has cases where multiple parameters are packed in the same stack slot.
1 parent fe8b347 commit b93a981

File tree

8 files changed

+120
-158
lines changed

8 files changed

+120
-158
lines changed

‎src/coreclr/jit/abi.cpp

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ bool ABIPassingSegment::IsPassedOnStack() const
3838
regNumber ABIPassingSegment::GetRegister() const
3939
{
4040
assert(IsPassedInRegister());
41-
return m_register;
41+
return static_cast<regNumber>(m_register);
4242
}
4343

4444
//-----------------------------------------------------------------------------
@@ -50,17 +50,17 @@ regNumber ABIPassingSegment::GetRegister() const
5050
//
5151
regMaskTP ABIPassingSegment::GetRegisterMask() const
5252
{
53-
assert(IsPassedInRegister());
54-
regMaskTP reg = genRegMask(m_register);
53+
regNumber reg = GetRegister();
54+
regMaskTP mask = genRegMask(reg);
5555

5656
#ifdef TARGET_ARM
57-
if (genIsValidFloatReg(m_register) && (Size == 8))
57+
if (genIsValidFloatReg(reg) && (Size == 8))
5858
{
59-
reg |= genRegMask(REG_NEXT(m_register));
59+
mask |= genRegMask(REG_NEXT(reg));
6060
}
6161
#endif
6262

63-
return reg;
63+
return mask;
6464
}
6565

6666
//-----------------------------------------------------------------------------
@@ -87,6 +87,21 @@ unsigned ABIPassingSegment::GetStackOffset() const
8787
return m_stackOffset;
8888
}
8989

90+
//-----------------------------------------------------------------------------
91+
// GetStackSize:
92+
// Get the amount of stack size consumed by this segment.
93+
//
94+
// Return Value:
95+
// Normally the size rounded up to the pointer size. For Apple's arm64 ABI,
96+
// however, some arguments do not get their own stack slots, in which case
97+
// the return value is the same as "Size".
98+
//
99+
unsigned ABIPassingSegment::GetStackSize() const
100+
{
101+
assert(IsPassedOnStack());
102+
return m_isFullStackSlot ? roundUp(Size, TARGET_POINTER_SIZE) : Size;
103+
}
104+
90105
//-----------------------------------------------------------------------------
91106
// GetRegisterType:
92107
// Return the smallest type larger or equal to Size that most naturally
@@ -97,8 +112,7 @@ unsigned ABIPassingSegment::GetStackOffset() const
97112
//
98113
var_types ABIPassingSegment::GetRegisterType() const
99114
{
100-
assert(IsPassedInRegister());
101-
if (genIsValidFloatReg(m_register))
115+
if (genIsValidFloatReg(GetRegister()))
102116
{
103117
switch (Size)
104118
{
@@ -157,7 +171,7 @@ ABIPassingSegment ABIPassingSegment::InRegister(regNumber reg, unsigned offset,
157171
{
158172
assert(reg != REG_NA);
159173
ABIPassingSegment segment;
160-
segment.m_register = reg;
174+
segment.m_register = static_cast<regNumberSmall>(reg);
161175
segment.m_stackOffset = 0;
162176
segment.Offset = offset;
163177
segment.Size = size;
@@ -187,6 +201,35 @@ ABIPassingSegment ABIPassingSegment::OnStack(unsigned stackOffset, unsigned offs
187201
return segment;
188202
}
189203

204+
//-----------------------------------------------------------------------------
205+
// OnStackWithoutConsumingFullSlot:
206+
// Create an ABIPassingSegment representing that a segment is passed on the
207+
// stack, and which does not gets its own full stack slot.
208+
//
209+
// Parameters:
210+
// stackOffset - Offset relative to the first stack parameter/argument
211+
// offset - The offset of the segment that is passed in the register
212+
// size - The size of the segment passed in the register
213+
//
214+
// Return Value:
215+
// New instance of ABIPassingSegment.
216+
//
217+
// Remarks:
218+
// This affects what ABIPassingSegment::GetStackSize() returns.
219+
//
220+
ABIPassingSegment ABIPassingSegment::OnStackWithoutConsumingFullSlot(unsigned stackOffset,
221+
unsigned offset,
222+
unsigned size)
223+
{
224+
ABIPassingSegment segment;
225+
segment.m_register = REG_NA;
226+
segment.m_stackOffset = stackOffset;
227+
segment.m_isFullStackSlot = false;
228+
segment.Offset = offset;
229+
segment.Size = size;
230+
return segment;
231+
}
232+
190233
//-----------------------------------------------------------------------------
191234
// ABIPassingInformation:
192235
// Construct an instance with the specified number of segments allocated in

‎src/coreclr/jit/abi.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ enum class WellKnownArg : unsigned;
88

99
class ABIPassingSegment
1010
{
11-
regNumber m_register = REG_NA;
12-
unsigned m_stackOffset = 0;
11+
regNumberSmall m_register = REG_NA;
12+
bool m_isFullStackSlot = true;
13+
unsigned m_stackOffset = 0;
1314

1415
public:
1516
bool IsPassedInRegister() const;
@@ -34,10 +35,16 @@ class ABIPassingSegment
3435
// offset, relative to the base of stack arguments.
3536
unsigned GetStackOffset() const;
3637

38+
// Get the size of stack consumed. Normally this is 'Size' rounded up to
39+
// the pointer size, but for apple arm64 ABI some primitives do not consume
40+
// full stack slots.
41+
unsigned GetStackSize() const;
42+
3743
var_types GetRegisterType() const;
3844

3945
static ABIPassingSegment InRegister(regNumber reg, unsigned offset, unsigned size);
4046
static ABIPassingSegment OnStack(unsigned stackOffset, unsigned offset, unsigned size);
47+
static ABIPassingSegment OnStackWithoutConsumingFullSlot(unsigned stackOffset, unsigned offset, unsigned size);
4148

4249
#ifdef DEBUG
4350
void Dump() const;

‎src/coreclr/jit/gentree.cpp

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1344,41 +1344,6 @@ bool CallArg::IsUserArg() const
13441344
}
13451345
}
13461346

1347-
#ifdef DEBUG
1348-
//---------------------------------------------------------------
1349-
// CheckIsStruct: Verify that the struct ABI information is consistent with the IR node.
1350-
//
1351-
void CallArg::CheckIsStruct()
1352-
{
1353-
GenTree* node = GetNode();
1354-
if (varTypeIsStruct(GetSignatureType()))
1355-
{
1356-
if (!varTypeIsStruct(node) && !node->OperIs(GT_FIELD_LIST))
1357-
{
1358-
// This is the case where we are passing a struct as a primitive type.
1359-
// On most targets, this is always a single register or slot.
1360-
// However, on ARM this could be two slots if it is TYP_DOUBLE.
1361-
bool isPassedAsPrimitiveType =
1362-
((AbiInfo.NumRegs == 1) || ((AbiInfo.NumRegs == 0) && (AbiInfo.ByteSize <= TARGET_POINTER_SIZE)));
1363-
#ifdef TARGET_ARM
1364-
if (!isPassedAsPrimitiveType)
1365-
{
1366-
if (node->TypeGet() == TYP_DOUBLE && AbiInfo.NumRegs == 0 && (AbiInfo.GetStackSlotsNumber() == 2))
1367-
{
1368-
isPassedAsPrimitiveType = true;
1369-
}
1370-
}
1371-
#endif // TARGET_ARM
1372-
assert(isPassedAsPrimitiveType);
1373-
}
1374-
}
1375-
else
1376-
{
1377-
assert(!varTypeIsStruct(node));
1378-
}
1379-
}
1380-
#endif
1381-
13821347
CallArgs::CallArgs()
13831348
: m_head(nullptr)
13841349
, m_lateHead(nullptr)

‎src/coreclr/jit/gentree.h

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4857,14 +4857,6 @@ class CallArg
48574857

48584858
#ifdef DEBUG
48594859
void Dump(Compiler* comp);
4860-
// Check that the value of 'AbiInfo.IsStruct' is consistent.
4861-
// A struct arg must be one of the following:
4862-
// - A node of struct type,
4863-
// - A GT_FIELD_LIST, or
4864-
// - A node of a scalar type, passed in a single register or slot
4865-
// (or two slots in the case of a struct pass on the stack as TYP_DOUBLE).
4866-
//
4867-
void CheckIsStruct();
48684860
#endif
48694861
};
48704862

‎src/coreclr/jit/lower.cpp

Lines changed: 36 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1583,44 +1583,42 @@ GenTree* Lowering::NewPutArg(GenTreeCall* call, GenTree* arg, CallArg* callArg,
15831583
assert(arg != nullptr);
15841584
assert(callArg != nullptr);
15851585

1586-
GenTree* putArg = nullptr;
1587-
1588-
bool isOnStack = (callArg->AbiInfo.GetRegNum() == REG_STK);
1586+
GenTree* putArg = nullptr;
1587+
const ABIPassingInformation& abiInfo = callArg->NewAbiInfo;
15891588

15901589
#if FEATURE_ARG_SPLIT
15911590
// Struct can be split into register(s) and stack on ARM
1592-
if (compFeatureArgSplit() && callArg->AbiInfo.IsSplit())
1591+
if (compFeatureArgSplit() && callArg->NewAbiInfo.IsSplitAcrossRegistersAndStack())
15931592
{
15941593
assert(arg->OperIs(GT_BLK, GT_FIELD_LIST) || arg->OperIsLocalRead());
1595-
// TODO: Need to check correctness for FastTailCall
1596-
if (call->IsFastTailCall())
1594+
assert(!call->IsFastTailCall());
1595+
1596+
#ifdef DEBUG
1597+
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
15971598
{
1598-
#ifdef TARGET_ARM
1599-
NYI_ARM("lower: struct argument by fast tail call");
1600-
#endif // TARGET_ARM
1599+
assert((i < abiInfo.NumSegments - 1) == abiInfo.Segment(i).IsPassedInRegister());
16011600
}
1601+
#endif
16021602

1603-
const unsigned slotNumber = callArg->AbiInfo.ByteOffset / TARGET_POINTER_SIZE;
1604-
const bool putInIncomingArgArea = call->IsFastTailCall();
1603+
unsigned numRegs = abiInfo.NumSegments - 1;
1604+
const ABIPassingSegment& stackSeg = abiInfo.Segment(abiInfo.NumSegments - 1);
16051605

1606-
putArg = new (comp, GT_PUTARG_SPLIT) GenTreePutArgSplit(arg, callArg->AbiInfo.ByteOffset,
1607-
#ifdef FEATURE_PUT_STRUCT_ARG_STK
1608-
callArg->AbiInfo.GetStackByteSize(),
1609-
#endif
1610-
callArg->AbiInfo.NumRegs, call, putInIncomingArgArea);
1606+
putArg = new (comp, GT_PUTARG_SPLIT)
1607+
GenTreePutArgSplit(arg, stackSeg.GetStackOffset(), stackSeg.GetStackSize(), abiInfo.NumSegments - 1, call,
1608+
/* putInIncomingArgArea */ false);
16111609

16121610
GenTreePutArgSplit* argSplit = putArg->AsPutArgSplit();
1613-
for (unsigned regIndex = 0; regIndex < callArg->AbiInfo.NumRegs; regIndex++)
1611+
for (unsigned regIndex = 0; regIndex < numRegs; regIndex++)
16141612
{
1615-
argSplit->SetRegNumByIdx(callArg->AbiInfo.GetRegNum(regIndex), regIndex);
1613+
argSplit->SetRegNumByIdx(abiInfo.Segment(regIndex).GetRegister(), regIndex);
16161614
}
16171615

16181616
if (arg->OperIs(GT_FIELD_LIST))
16191617
{
16201618
unsigned regIndex = 0;
16211619
for (GenTreeFieldList::Use& use : arg->AsFieldList()->Uses())
16221620
{
1623-
if (regIndex >= callArg->AbiInfo.NumRegs)
1621+
if (regIndex >= numRegs)
16241622
{
16251623
break;
16261624
}
@@ -1642,7 +1640,7 @@ GenTree* Lowering::NewPutArg(GenTreeCall* call, GenTree* arg, CallArg* callArg,
16421640
ClassLayout* layout = arg->GetLayout(comp);
16431641

16441642
// Set type of registers
1645-
for (unsigned index = 0; index < callArg->AbiInfo.NumRegs; index++)
1643+
for (unsigned index = 0; index < numRegs; index++)
16461644
{
16471645
argSplit->m_regType[index] = layout->GetGCPtrType(index);
16481646
}
@@ -1651,15 +1649,15 @@ GenTree* Lowering::NewPutArg(GenTreeCall* call, GenTree* arg, CallArg* callArg,
16511649
else
16521650
#endif // FEATURE_ARG_SPLIT
16531651
{
1654-
if (!isOnStack)
1652+
if (abiInfo.HasAnyRegisterSegment())
16551653
{
16561654
#if FEATURE_MULTIREG_ARGS
1657-
if ((callArg->AbiInfo.NumRegs > 1) && (arg->OperGet() == GT_FIELD_LIST))
1655+
if ((abiInfo.NumSegments > 1) && arg->OperIs(GT_FIELD_LIST))
16581656
{
16591657
unsigned int regIndex = 0;
16601658
for (GenTreeFieldList::Use& use : arg->AsFieldList()->Uses())
16611659
{
1662-
regNumber argReg = callArg->AbiInfo.GetRegNum(regIndex);
1660+
regNumber argReg = abiInfo.Segment(regIndex).GetRegister();
16631661
GenTree* curOp = use.GetNode();
16641662
var_types curTyp = curOp->TypeGet();
16651663

@@ -1678,74 +1676,27 @@ GenTree* Lowering::NewPutArg(GenTreeCall* call, GenTree* arg, CallArg* callArg,
16781676
else
16791677
#endif // FEATURE_MULTIREG_ARGS
16801678
{
1681-
putArg = comp->gtNewPutArgReg(type, arg, callArg->AbiInfo.GetRegNum());
1679+
assert(abiInfo.HasExactlyOneRegisterSegment());
1680+
putArg = comp->gtNewPutArgReg(type, arg, abiInfo.Segment(0).GetRegister());
16821681
}
16831682
}
16841683
else
16851684
{
1686-
// Mark this one as tail call arg if it is a fast tail call.
1687-
// This provides the info to put this argument in in-coming arg area slot
1688-
// instead of in out-going arg area slot.
1689-
1690-
#ifdef DEBUG
1691-
// Make sure state is correct. The PUTARG_STK has TYP_VOID, as it doesn't produce
1692-
// a result. So the type of its operand must be the correct type to push on the stack.
1693-
callArg->CheckIsStruct();
1685+
#ifdef FEATURE_SIMD
1686+
assert(arg->OperIsFieldList() || (genActualType(arg) == type) ||
1687+
(arg->TypeIs(TYP_SIMD16) && (type == TYP_SIMD12)));
1688+
#else
1689+
assert(arg->OperIsFieldList() || (genActualType(arg) == type));
16941690
#endif
1691+
assert(abiInfo.NumSegments == 1);
1692+
const ABIPassingSegment& stackSeg = abiInfo.Segment(0);
1693+
const bool putInIncomingArgArea = call->IsFastTailCall();
16951694

1696-
if ((arg->OperGet() != GT_FIELD_LIST))
1697-
{
1698-
#if defined(FEATURE_SIMD) && defined(FEATURE_PUT_STRUCT_ARG_STK)
1699-
if (type == TYP_SIMD12)
1700-
{
1701-
#if !defined(TARGET_64BIT)
1702-
assert(callArg->AbiInfo.ByteSize == 12);
1703-
#else // TARGET_64BIT
1704-
if (compAppleArm64Abi())
1705-
{
1706-
assert(callArg->AbiInfo.ByteSize == 12);
1707-
}
1708-
else
1709-
{
1710-
assert(callArg->AbiInfo.ByteSize == 16);
1711-
}
1712-
#endif // TARGET_64BIT
1713-
}
1714-
else
1715-
#endif // defined(FEATURE_SIMD) && defined(FEATURE_PUT_STRUCT_ARG_STK)
1716-
{
1717-
assert(genActualType(arg->TypeGet()) == type);
1718-
}
1719-
}
1720-
const unsigned slotNumber = callArg->AbiInfo.ByteOffset / TARGET_POINTER_SIZE;
1721-
const bool putInIncomingArgArea = call->IsFastTailCall();
1722-
1723-
putArg =
1724-
new (comp, GT_PUTARG_STK) GenTreePutArgStk(GT_PUTARG_STK, TYP_VOID, arg, callArg->AbiInfo.ByteOffset,
1695+
putArg = new (comp, GT_PUTARG_STK) GenTreePutArgStk(GT_PUTARG_STK, TYP_VOID, arg, stackSeg.GetStackOffset(),
17251696
#ifdef FEATURE_PUT_STRUCT_ARG_STK
1726-
callArg->AbiInfo.GetStackByteSize(),
1727-
#endif
1728-
call, putInIncomingArgArea);
1729-
1730-
#if defined(DEBUG) && defined(FEATURE_PUT_STRUCT_ARG_STK)
1731-
if (varTypeIsStruct(callArg->GetSignatureType()))
1732-
{
1733-
// We use GT_BLK only for non-SIMD struct arguments.
1734-
if (arg->OperIs(GT_BLK))
1735-
{
1736-
assert(!varTypeIsSIMD(arg));
1737-
}
1738-
else if (!arg->TypeIs(TYP_STRUCT))
1739-
{
1740-
#ifdef TARGET_ARM
1741-
assert((callArg->AbiInfo.GetStackSlotsNumber() == 1) ||
1742-
((arg->TypeGet() == TYP_DOUBLE) && (callArg->AbiInfo.GetStackSlotsNumber() == 2)));
1743-
#else
1744-
assert(varTypeIsSIMD(arg) || (callArg->AbiInfo.GetStackSlotsNumber() == 1));
1697+
stackSeg.GetStackSize(),
17451698
#endif
1746-
}
1747-
}
1748-
#endif // defined(DEBUG) && defined(FEATURE_PUT_STRUCT_ARG_STK)
1699+
call, putInIncomingArgArea);
17491700
}
17501701
}
17511702

@@ -1819,7 +1770,8 @@ void Lowering::LowerArg(GenTreeCall* call, CallArg* callArg, bool late)
18191770
}
18201771
#elif defined(TARGET_AMD64)
18211772
// TYP_SIMD8 parameters that are passed as longs
1822-
if (type == TYP_SIMD8 && genIsValidIntReg(callArg->AbiInfo.GetRegNum()))
1773+
if (type == TYP_SIMD8 && callArg->NewAbiInfo.HasExactlyOneRegisterSegment() &&
1774+
genIsValidIntReg(callArg->NewAbiInfo.Segment(0).GetRegister()))
18231775
{
18241776
GenTree* bitcast = comp->gtNewBitCastNode(TYP_LONG, arg);
18251777
BlockRange().InsertAfter(arg, bitcast);

‎src/coreclr/jit/morph.cpp

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2159,10 +2159,6 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call
21592159
}
21602160
#endif
21612161
}
2162-
2163-
#if defined(UNIX_AMD64_ABI) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
2164-
INDEBUG(arg.CheckIsStruct());
2165-
#endif
21662162
}
21672163
else
21682164
{

0 commit comments

Comments
 (0)