Skip to content

Commit 7d75878

Browse files
hez2010jakobbotschAndyAyersMS
authored
JIT: Extend escape analysis to account for arrays with non-gcref elements (#104906)
Support stack allocation of small fixed-length arrays of non-GC types. * Size limit is currently 528 bytes (allows for a 512 byte array on 64 bit) per array. * No per-method limit (yet) on total amount of allocation * Bounds check elimination and such works normally * Codegen is similar to that for heap arrays * Limited ability to treat stack allocated array storage more aggressively than heap arrays Co-authored-by: Jakob Botsch Nielsen <[email protected]> Co-authored-by: Andy Ayers <[email protected]>
1 parent 4bb0bcd commit 7d75878

17 files changed

+766
-123
lines changed

src/coreclr/jit/compiler.cpp

+3
Original file line numberDiff line numberDiff line change
@@ -5127,6 +5127,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
51275127
// Expand thread local access
51285128
DoPhase(this, PHASE_EXPAND_TLS, &Compiler::fgExpandThreadLocalAccess);
51295129

5130+
// Expand stack allocated arrays
5131+
DoPhase(this, PHASE_EXPAND_STACK_ARR, &Compiler::fgExpandStackArrayAllocations);
5132+
51305133
// Insert GC Polls
51315134
DoPhase(this, PHASE_INSERT_GC_POLLS, &Compiler::fgInsertGCPolls);
51325135

src/coreclr/jit/compiler.h

+18-4
Original file line numberDiff line numberDiff line change
@@ -691,8 +691,8 @@ class LclVarDsc
691691
unsigned char lvSingleDefDisqualifyReason = 'H';
692692
#endif
693693

694-
unsigned char lvAllDefsAreNoGc : 1; // For pinned locals: true if all defs of this local are no-gc
695-
unsigned char lvStackAllocatedBox : 1; // Local is a stack allocated box
694+
unsigned char lvAllDefsAreNoGc : 1; // For pinned locals: true if all defs of this local are no-gc
695+
unsigned char lvStackAllocatedObject : 1; // Local is a stack allocated object (class, box, array, ...)
696696

697697
#if FEATURE_MULTIREG_ARGS
698698
regNumber lvRegNumForSlot(unsigned slotNum)
@@ -807,9 +807,9 @@ class LclVarDsc
807807
return lvIsMultiRegArg || lvIsMultiRegRet;
808808
}
809809

810-
bool IsStackAllocatedBox() const
810+
bool IsStackAllocatedObject() const
811811
{
812-
return lvStackAllocatedBox;
812+
return lvStackAllocatedObject;
813813
}
814814

815815
#if defined(DEBUG)
@@ -6106,6 +6106,9 @@ class Compiler
61066106
PhaseStatus fgExpandStaticInit();
61076107
bool fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
61086108

6109+
PhaseStatus fgExpandStackArrayAllocations();
6110+
bool fgExpandStackArrayAllocation(BasicBlock* pBlock, Statement* stmt, GenTreeCall* call);
6111+
61096112
PhaseStatus fgVNBasedIntrinsicExpansion();
61106113
bool fgVNBasedIntrinsicExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
61116114
bool fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
@@ -7537,6 +7540,7 @@ class Compiler
75377540
#define OMF_HAS_SPECIAL_INTRINSICS 0x00020000 // Method contains special intrinsics expanded in late phases
75387541
#define OMF_HAS_RECURSIVE_TAILCALL 0x00040000 // Method contains recursive tail call
75397542
#define OMF_HAS_EXPANDABLE_CAST 0x00080000 // Method contains casts eligible for late expansion
7543+
#define OMF_HAS_STACK_ARRAY 0x00100000 // Method contains stack allocated arrays
75407544

75417545
// clang-format on
75427546

@@ -7627,6 +7631,16 @@ class Compiler
76277631
optMethodFlags |= OMF_HAS_RECURSIVE_TAILCALL;
76287632
}
76297633

7634+
bool doesMethodHaveStackAllocatedArray()
7635+
{
7636+
return (optMethodFlags & OMF_HAS_STACK_ARRAY) != 0;
7637+
}
7638+
7639+
void setMethodHasStackAllocatedArray()
7640+
{
7641+
optMethodFlags |= OMF_HAS_STACK_ARRAY;
7642+
}
7643+
76307644
void pickGDV(GenTreeCall* call,
76317645
IL_OFFSET ilOffset,
76327646
bool isInterface,

src/coreclr/jit/compphases.h

+1
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ CompPhaseNameMacro(PHASE_EXPAND_RTLOOKUPS, "Expand runtime lookups",
109109
CompPhaseNameMacro(PHASE_EXPAND_STATIC_INIT, "Expand static init", false, -1, true)
110110
CompPhaseNameMacro(PHASE_EXPAND_CASTS, "Expand casts", false, -1, true)
111111
CompPhaseNameMacro(PHASE_EXPAND_TLS, "Expand TLS access", false, -1, true)
112+
CompPhaseNameMacro(PHASE_EXPAND_STACK_ARR, "Expand stack array allocation", false, -1, true)
112113
CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", false, -1, true)
113114
CompPhaseNameMacro(PHASE_CREATE_THROW_HELPERS, "Create throw helper blocks", false, -1, true)
114115
CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block", false, -1, true)

src/coreclr/jit/fgdiagnostic.cpp

+4
Original file line numberDiff line numberDiff line change
@@ -955,6 +955,10 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos)
955955
{
956956
fprintf(fgxFile, "\n hot=\"true\"");
957957
}
958+
if (block->HasFlag(BBF_HAS_NEWARR))
959+
{
960+
fprintf(fgxFile, "\n callsNewArr=\"true\"");
961+
}
958962
if (block->HasFlag(BBF_HAS_NEWOBJ))
959963
{
960964
fprintf(fgxFile, "\n callsNew=\"true\"");

src/coreclr/jit/gentree.cpp

+7-2
Original file line numberDiff line numberDiff line change
@@ -13192,6 +13192,8 @@ const char* Compiler::gtGetWellKnownArgNameForArgMsg(WellKnownArg arg)
1319213192
return "swift self";
1319313193
case WellKnownArg::X86TailCallSpecialArg:
1319413194
return "tail call";
13195+
case WellKnownArg::StackArrayLocal:
13196+
return "&lcl arr";
1319513197
default:
1319613198
return nullptr;
1319713199
}
@@ -19814,7 +19816,10 @@ void GenTreeArrAddr::ParseArrayAddress(Compiler* comp, GenTree** pArr, ValueNum*
1981419816
/* static */ void GenTreeArrAddr::ParseArrayAddressWork(
1981519817
GenTree* tree, Compiler* comp, target_ssize_t inputMul, GenTree** pArr, ValueNum* pInxVN, target_ssize_t* pOffset)
1981619818
{
19817-
if (tree->TypeIs(TYP_REF))
19819+
ValueNum vn = comp->GetValueNumStore()->VNLiberalNormalValue(tree->gtVNPair);
19820+
VNFuncApp vnf;
19821+
19822+
if (tree->TypeIs(TYP_REF) || comp->GetValueNumStore()->IsVNNewArr(vn, &vnf))
1981819823
{
1981919824
// This must be the array pointer.
1982019825
assert(*pArr == nullptr);
@@ -19917,7 +19922,7 @@ void GenTreeArrAddr::ParseArrayAddress(Compiler* comp, GenTree** pArr, ValueNum*
1991719922
// If we didn't return above, must be a contribution to the non-constant part of the index VN.
1991819923
// We don't get here for GT_CNS_INT, GT_ADD, or GT_SUB, or for GT_MUL by constant, or GT_LSH of
1991919924
// constant shift. Thus, the generated index VN does not include the parsed constant offset.
19920-
ValueNum vn = comp->GetValueNumStore()->VNLiberalNormalValue(tree->gtVNPair);
19925+
//
1992119926
if (inputMul != 1)
1992219927
{
1992319928
ValueNum mulVN = comp->GetValueNumStore()->VNForLongCon(inputMul);

src/coreclr/jit/gentree.h

+5-3
Original file line numberDiff line numberDiff line change
@@ -3586,7 +3586,7 @@ class SsaNumInfo final
35863586
}
35873587
};
35883588

3589-
// Common supertype of [STORE_]LCL_VAR, [STORE_]LCL_FLD, PHI_ARG, LCL_VAR_ADDR, LCL_FLD_ADDR.
3589+
// Common supertype of [STORE_]LCL_VAR, [STORE_]LCL_FLD, PHI_ARG, LCL_ADDR.
35903590
// This inherits from UnOp because lclvar stores are unary.
35913591
//
35923592
struct GenTreeLclVarCommon : public GenTreeUnOp
@@ -4227,6 +4227,7 @@ enum GenTreeCallFlags : unsigned int
42274227
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
42284228
GTF_CALL_M_CAST_OBJ_NONNULL = 0x08000000, // if we expand this specific cast we don't need to check the input object for null
42294229
// NOTE: if needed, this flag can be removed, and we can introduce new _NONNUL cast helpers
4230+
GTF_CALL_M_STACK_ARRAY = 0x10000000, // this call is a new array helper for a stack allocated array.
42304231
};
42314232

42324233
inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a)
@@ -4566,6 +4567,7 @@ enum class WellKnownArg : unsigned
45664567
SwiftError,
45674568
SwiftSelf,
45684569
X86TailCallSpecialArg,
4570+
StackArrayLocal,
45694571
};
45704572

45714573
#ifdef DEBUG
@@ -7586,12 +7588,12 @@ struct GenTreeArrAddr : GenTreeUnOp
75867588

75877589
public:
75887590
GenTreeArrAddr(GenTree* addr, var_types elemType, CORINFO_CLASS_HANDLE elemClassHandle, uint8_t firstElemOffset)
7589-
: GenTreeUnOp(GT_ARR_ADDR, TYP_BYREF, addr DEBUGARG(/* largeNode */ false))
7591+
: GenTreeUnOp(GT_ARR_ADDR, addr->TypeGet(), addr DEBUGARG(/* largeNode */ false))
75907592
, m_elemClassHandle(elemClassHandle)
75917593
, m_elemType(elemType)
75927594
, m_firstElemOffset(firstElemOffset)
75937595
{
7594-
assert(addr->TypeIs(TYP_BYREF));
7596+
assert(addr->TypeIs(TYP_BYREF, TYP_I_IMPL));
75957597
assert(((elemType == TYP_STRUCT) && (elemClassHandle != NO_CLASS_HANDLE)) ||
75967598
(elemClassHandle == NO_CLASS_HANDLE));
75977599
}

src/coreclr/jit/helperexpansion.cpp

+155
Original file line numberDiff line numberDiff line change
@@ -2733,3 +2733,158 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt,
27332733

27342734
return true;
27352735
}
2736+
2737+
//------------------------------------------------------------------------------
2738+
// fgExpandStackArrayAllocations : expand "new helpers" for stack arrays
2739+
//
2740+
// Returns:
2741+
// PhaseStatus indicating what, if anything, was changed.
2742+
//
2743+
PhaseStatus Compiler::fgExpandStackArrayAllocations()
2744+
{
2745+
PhaseStatus result = PhaseStatus::MODIFIED_NOTHING;
2746+
2747+
if (!doesMethodHaveStackAllocatedArray())
2748+
{
2749+
// The method being compiled doesn't have any stack allocated arrays.
2750+
return result;
2751+
}
2752+
2753+
// Find allocation sites, and transform them into initializations of the
2754+
// array method table and length, and replace the allocation call with
2755+
// the address of the local array.
2756+
//
2757+
bool modified = false;
2758+
2759+
for (BasicBlock* const block : Blocks())
2760+
{
2761+
for (Statement* const stmt : block->Statements())
2762+
{
2763+
if ((stmt->GetRootNode()->gtFlags & GTF_CALL) == 0)
2764+
{
2765+
continue;
2766+
}
2767+
2768+
for (GenTree* const tree : stmt->TreeList())
2769+
{
2770+
if (!tree->IsCall())
2771+
{
2772+
continue;
2773+
}
2774+
2775+
if (fgExpandStackArrayAllocation(block, stmt, tree->AsCall()))
2776+
{
2777+
// If we expand, we split the statement's tree
2778+
// so will be done with this statment.
2779+
//
2780+
modified = true;
2781+
break;
2782+
}
2783+
}
2784+
}
2785+
}
2786+
2787+
// we cant assert(modified) here as array allocation sites may
2788+
// have been unreachable or dead-coded.
2789+
//
2790+
return modified ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
2791+
}
2792+
2793+
//------------------------------------------------------------------------------
2794+
// fgExpandStackArrayAllocation: expand new array helpers for stack allocated arrays
2795+
//
2796+
// Arguments:
2797+
// block - block containing the helper call to expand
2798+
// stmt - Statement containing the helper call
2799+
// call - The helper call
2800+
//
2801+
// Returns:
2802+
// true if a runtime lookup was found and expanded.
2803+
//
2804+
bool Compiler::fgExpandStackArrayAllocation(BasicBlock* block, Statement* stmt, GenTreeCall* call)
2805+
{
2806+
if (!call->IsHelperCall())
2807+
{
2808+
return false;
2809+
}
2810+
2811+
const CorInfoHelpFunc helper = eeGetHelperNum(call->gtCallMethHnd);
2812+
int lengthArgIndex = -1;
2813+
2814+
switch (helper)
2815+
{
2816+
case CORINFO_HELP_NEWARR_1_DIRECT:
2817+
case CORINFO_HELP_NEWARR_1_VC:
2818+
case CORINFO_HELP_NEWARR_1_OBJ:
2819+
case CORINFO_HELP_NEWARR_1_ALIGN8:
2820+
lengthArgIndex = 1;
2821+
break;
2822+
2823+
case CORINFO_HELP_READYTORUN_NEWARR_1:
2824+
lengthArgIndex = 0;
2825+
break;
2826+
2827+
default:
2828+
return false;
2829+
}
2830+
2831+
// If this is a local array, the new helper will have an arg for the array's address
2832+
//
2833+
CallArg* const stackLocalAddressArg = call->gtArgs.FindWellKnownArg(WellKnownArg::StackArrayLocal);
2834+
2835+
if (stackLocalAddressArg == nullptr)
2836+
{
2837+
return false;
2838+
}
2839+
2840+
JITDUMP("Expanding new array helper for stack allocated array at [%06d] in " FMT_BB ":\n", dspTreeID(call),
2841+
block->bbNum);
2842+
DISPTREE(call);
2843+
JITDUMP("\n");
2844+
2845+
Statement* newStmt = nullptr;
2846+
GenTree** callUse = nullptr;
2847+
bool split = gtSplitTree(block, stmt, call, &newStmt, &callUse);
2848+
2849+
if (split)
2850+
{
2851+
while ((newStmt != nullptr) && (newStmt != stmt))
2852+
{
2853+
fgMorphStmtBlockOps(block, newStmt);
2854+
newStmt = newStmt->GetNextStmt();
2855+
}
2856+
}
2857+
2858+
GenTree* const stackLocalAddress = stackLocalAddressArg->GetNode();
2859+
2860+
// Initialize the array method table pointer.
2861+
//
2862+
CORINFO_CLASS_HANDLE arrayHnd = (CORINFO_CLASS_HANDLE)call->compileTimeHelperArgumentHandle;
2863+
2864+
GenTree* const mt = gtNewIconEmbClsHndNode(arrayHnd);
2865+
GenTree* const mtStore = gtNewStoreValueNode(TYP_I_IMPL, stackLocalAddress, mt);
2866+
Statement* const mtStmt = fgNewStmtFromTree(mtStore);
2867+
2868+
fgInsertStmtBefore(block, stmt, mtStmt);
2869+
2870+
// Initialize the array length.
2871+
//
2872+
GenTree* const lengthArg = call->gtArgs.GetArgByIndex(lengthArgIndex)->GetNode();
2873+
GenTree* const lengthArgInt = fgOptimizeCast(gtNewCastNode(TYP_INT, lengthArg, false, TYP_INT));
2874+
GenTree* const lengthAddress = gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(stackLocalAddress),
2875+
gtNewIconNode(OFFSETOF__CORINFO_Array__length, TYP_I_IMPL));
2876+
GenTree* const lengthStore = gtNewStoreValueNode(TYP_INT, lengthAddress, lengthArgInt);
2877+
Statement* const lenStmt = fgNewStmtFromTree(lengthStore);
2878+
2879+
fgInsertStmtBefore(block, stmt, lenStmt);
2880+
2881+
// Replace call with local address
2882+
//
2883+
*callUse = gtCloneExpr(stackLocalAddress);
2884+
DEBUG_DESTROY_NODE(call);
2885+
2886+
fgMorphStmtBlockOps(block, stmt);
2887+
gtUpdateStmtSideEffects(stmt);
2888+
2889+
return true;
2890+
}

src/coreclr/jit/jitconfigvalues.h

+2
Original file line numberDiff line numberDiff line change
@@ -668,6 +668,8 @@ CONFIG_STRING(JitObjectStackAllocationRange, "JitObjectStackAllocationRange")
668668
RELEASE_CONFIG_INTEGER(JitObjectStackAllocation, "JitObjectStackAllocation", 1)
669669
RELEASE_CONFIG_INTEGER(JitObjectStackAllocationRefClass, "JitObjectStackAllocationRefClass", 1)
670670
RELEASE_CONFIG_INTEGER(JitObjectStackAllocationBoxedValueClass, "JitObjectStackAllocationBoxedValueClass", 1)
671+
RELEASE_CONFIG_INTEGER(JitObjectStackAllocationArray, "JitObjectStackAllocationArray", 1)
672+
RELEASE_CONFIG_INTEGER(JitObjectStackAllocationSize, "JitObjectStackAllocationSize", 528)
671673

672674
RELEASE_CONFIG_INTEGER(JitEECallTimingInfo, "JitEECallTimingInfo", 0)
673675

src/coreclr/jit/jitmetadatalist.h

+2
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ JITMETADATAMETRIC(NewRefClassHelperCalls, int, 0)
8686
JITMETADATAMETRIC(StackAllocatedRefClasses, int, 0)
8787
JITMETADATAMETRIC(NewBoxedValueClassHelperCalls, int, 0)
8888
JITMETADATAMETRIC(StackAllocatedBoxedValueClasses, int, 0)
89+
JITMETADATAMETRIC(NewArrayHelperCalls, int, 0)
90+
JITMETADATAMETRIC(StackAllocatedArrays, int, 0)
8991
JITMETADATAMETRIC(LocalAssertionCount, int, 0)
9092
JITMETADATAMETRIC(LocalAssertionOverflow, int, 0)
9193
JITMETADATAMETRIC(MorphTrackedLocals, int, 0)

src/coreclr/jit/lclvars.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -2503,9 +2503,9 @@ bool Compiler::StructPromotionHelper::CanPromoteStructVar(unsigned lclNum)
25032503
return false;
25042504
}
25052505

2506-
if (varDsc->lvStackAllocatedBox)
2506+
if (varDsc->lvStackAllocatedObject)
25072507
{
2508-
JITDUMP(" struct promotion of V%02u is disabled because it is a stack allocated box\n", lclNum);
2508+
JITDUMP(" struct promotion of V%02u is disabled because it is a stack allocated object\n", lclNum);
25092509
return false;
25102510
}
25112511

src/coreclr/jit/morph.cpp

+8-1
Original file line numberDiff line numberDiff line change
@@ -780,6 +780,8 @@ const char* getWellKnownArgName(WellKnownArg arg)
780780
return "SwiftSelf";
781781
case WellKnownArg::X86TailCallSpecialArg:
782782
return "X86TailCallSpecialArg";
783+
case WellKnownArg::StackArrayLocal:
784+
return "StackArrayLocal";
783785
}
784786

785787
return "N/A";
@@ -3633,7 +3635,7 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr)
36333635
}
36343636

36353637
if (((index->gtFlags & (GTF_ASG | GTF_CALL | GTF_GLOB_REF)) != 0) ||
3636-
gtComplexityExceeds(index, MAX_ARR_COMPLEXITY) || index->OperIs(GT_LCL_FLD) ||
3638+
gtComplexityExceeds(index, MAX_INDEX_COMPLEXITY) || index->OperIs(GT_LCL_FLD) ||
36373639
(index->OperIs(GT_LCL_VAR) && lvaIsLocalImplicitlyAccessedByRef(index->AsLclVar()->GetLclNum())))
36383640
{
36393641
unsigned indexTmpNum = lvaGrabTemp(true DEBUGARG("index expr"));
@@ -5137,6 +5139,11 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call)
51375139
assert(lvaIsImplicitByRefLocal(lvaTable[varDsc->lvFieldLclStart].lvParentLcl));
51385140
assert(fgGlobalMorph);
51395141
}
5142+
else if (varDsc->IsStackAllocatedObject())
5143+
{
5144+
// Stack allocated objects currently cannot be passed to callees
5145+
// so won't be live at tail call sites.
5146+
}
51405147
#if FEATURE_FIXED_OUT_ARGS
51415148
else if (varNum == lvaOutgoingArgSpaceVar)
51425149
{

0 commit comments

Comments
 (0)