[Inliner] Put inline history into IR as !inline_history metadata (llvm#190700)

(Reland of llvm#190092 with verifier change to look through GlobalAliases)

So that it's preserved across all inline invocations rather than just
one inliner pass run.

This prevents cases where devirtualization in the simplification
pipeline uncovers inlining opportunities that should be discarded due to
inline history, but we dropped the inline history between inliner pass
runs, causing code size to blow up, sometimes exponentially.

For compile time reasons, we want to limit this to only call sites that
have the potential to inline through SCCs, potentially with the help of
devirtualization. This means that the callee is in a non-trivial
(Ref)SCC, or the call site was previously an indirect call, which can
potentially be devirtualized to call any function.

The CGSCCUpdater::InlinedInternalEdges logic still seems to be relevant
even with this change, as monster_scc.ll blows up if I remove that code.


http://llvm-compile-time-tracker.com/compare.php?from=e830d88e8ae5f44a97cc76136a0a4e83aa9157c0&to=ed535e732fc41b79ab8efda2417886cbd0812f7f&stat=instructions:u

Fixes llvm#186926.

Author: aeubanks

llvm/docs/LangRef.rst

@@ -8852,6 +8852,25 @@ Example:
     !0 = !{ptr @a}
     !1 = !{ptr @b}
 
+'``inline_history``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The ``inline_history`` metadata may be attached to a call instruction. It
+indicates that the call instruction has been inlined from the referenced
+functions. The call itself should not be inlined if it is a call to any of the
+referenced functions since that could result in infinite inlining as we
+continually inline through mutually recursive functions.
+
+This is intended to be added by and used by inliner passes.
+
+The metadata operands must all be function pointers or ``null``. ``null`` can
+appear when the referenced function is erased from the module, e.g. an internal
+function that has had all calls to it inlined.
+
+.. code-block:: text
+
+    call void @foo(), !inline_history !0
+
+    !0 = !{ptr @bar, null, ptr @baz}
 
 Module Flags Metadata
 =====================

llvm/include/llvm/Analysis/InlineOrder.h

@@ -17,26 +17,26 @@ namespace llvm {
 class CallBase;
 template <typename Fn> class function_ref;
 
-template <typename T> class InlineOrder {
+class InlineOrder {
 public:
   virtual ~InlineOrder() = default;
 
   virtual size_t size() = 0;
 
-  virtual void push(const T &Elt) = 0;
+  virtual void push(CallBase *Elt) = 0;
 
-  virtual T pop() = 0;
+  virtual CallBase *pop() = 0;
 
-  virtual void erase_if(function_ref<bool(T)> Pred) = 0;
+  virtual void erase_if(function_ref<bool(CallBase *)> Pred) = 0;
 
   bool empty() { return !size(); }
 };
 
-LLVM_ABI std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
+LLVM_ABI std::unique_ptr<InlineOrder>
 getDefaultInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
                       ModuleAnalysisManager &MAM, Module &M);
 
-LLVM_ABI std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
+LLVM_ABI std::unique_ptr<InlineOrder>
 getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
                ModuleAnalysisManager &MAM, Module &M);
 
@@ -54,10 +54,9 @@ class PluginInlineOrderAnalysis
 public:
   LLVM_ABI static AnalysisKey Key;
 
-  typedef std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>> (
-      *InlineOrderFactory)(FunctionAnalysisManager &FAM,
-                           const InlineParams &Params,
-                           ModuleAnalysisManager &MAM, Module &M);
+  typedef std::unique_ptr<InlineOrder> (*InlineOrderFactory)(
+      FunctionAnalysisManager &FAM, const InlineParams &Params,
+      ModuleAnalysisManager &MAM, Module &M);
 
   PluginInlineOrderAnalysis(InlineOrderFactory Factory) : Factory(Factory) {
     assert(Factory != nullptr &&

llvm/include/llvm/IR/FixedMetadataKinds.def

@@ -60,3 +60,4 @@ LLVM_FIXED_MD_KIND(MD_alloc_token, "alloc_token", 45)
 LLVM_FIXED_MD_KIND(MD_implicit_ref, "implicit.ref", 46)
 LLVM_FIXED_MD_KIND(MD_nofpclass, "nofpclass", 47)
 LLVM_FIXED_MD_KIND(MD_call_target, "call_target", 48)
+LLVM_FIXED_MD_KIND(MD_inline_history, "inline_history", 49)

llvm/include/llvm/Transforms/Utils/Cloning.h

@@ -18,6 +18,7 @@
 #define LLVM_TRANSFORMS_UTILS_CLONING_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/AssumptionCache.h"
@@ -88,6 +89,10 @@ struct ClonedCodeInfo {
   /// check whether the main VMap mapping involves simplification or not.
   DenseMap<const Value *, const Value *> OrigVMap;
 
+  // Cloned calls that were originally an indirect call. They may be direct or
+  // indirect after cloning.
+  SmallPtrSet<const Value *, 4> OriginallyIndirectCalls;
+
   ClonedCodeInfo() = default;
 
   bool isSimplified(const Value *From, const Value *To) const {
@@ -225,11 +230,12 @@ LLVM_ABI void CloneFunctionBodyInto(
     ValueMaterializer *Materializer = nullptr,
     const MetadataPredicate *IdentityMD = nullptr);
 
-LLVM_ABI void CloneAndPruneIntoFromInst(
-    Function *NewFunc, const Function *OldFunc, const Instruction *StartingInst,
-    ValueToValueMapTy &VMap, bool ModuleLevelChanges,
-    SmallVectorImpl<ReturnInst *> &Returns, const char *NameSuffix = "",
-    ClonedCodeInfo *CodeInfo = nullptr);
+LLVM_ABI void
+CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
+                          const Instruction *StartingInst,
+                          ValueToValueMapTy &VMap, bool ModuleLevelChanges,
+                          SmallVectorImpl<ReturnInst *> &Returns,
+                          const char *NameSuffix, ClonedCodeInfo &CodeInfo);
 
 /// This works exactly like CloneFunctionInto,
 /// except that it does some simple constant prop and DCE on the fly.  The
@@ -242,10 +248,11 @@ LLVM_ABI void CloneAndPruneIntoFromInst(
 /// If ModuleLevelChanges is false, VMap contains no non-identity GlobalValue
 /// mappings.
 ///
-LLVM_ABI void CloneAndPruneFunctionInto(
-    Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap,
-    bool ModuleLevelChanges, SmallVectorImpl<ReturnInst *> &Returns,
-    const char *NameSuffix = "", ClonedCodeInfo *CodeInfo = nullptr);
+LLVM_ABI void
+CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
+                          ValueToValueMapTy &VMap, bool ModuleLevelChanges,
+                          SmallVectorImpl<ReturnInst *> &Returns,
+                          const char *NameSuffix, ClonedCodeInfo &CodeInfo);
 
 /// This class captures the data input to the InlineFunction call, and records
 /// the auxiliary results produced by it.
@@ -311,6 +318,7 @@ LLVM_ABI void InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
                                  bool MergeAttributes = false,
                                  AAResults *CalleeAAR = nullptr,
                                  bool InsertLifetime = true,
+                                 bool TrackInlineHistory = false,
                                  Function *ForwardVarArgsTo = nullptr,
                                  OptimizationRemarkEmitter *ORE = nullptr);
 
@@ -340,6 +348,7 @@ LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
                                      bool MergeAttributes = false,
                                      AAResults *CalleeAAR = nullptr,
                                      bool InsertLifetime = true,
+                                     bool TrackInlineHistory = false,
                                      Function *ForwardVarArgsTo = nullptr,
                                      OptimizationRemarkEmitter *ORE = nullptr);
 
@@ -352,6 +361,7 @@ LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
                                      bool MergeAttributes = false,
                                      AAResults *CalleeAAR = nullptr,
                                      bool InsertLifetime = true,
+                                     bool TrackInlineHistory = false,
                                      Function *ForwardVarArgsTo = nullptr,
                                      OptimizationRemarkEmitter *ORE = nullptr);
 
@@ -433,14 +443,6 @@ LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
 LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
                                          Instruction *IStart, Instruction *IEnd,
                                          LLVMContext &Context, StringRef Ext);
-/// Check if Function F appears in the inline history chain.
-/// InlineHistory is a vector of (Function, ParentHistoryID) pairs.
-/// Returns true if F was already inlined in the chain leading to
-/// InlineHistoryID.
-LLVM_ABI bool
-inlineHistoryIncludes(Function *F, int InlineHistoryID,
-                      ArrayRef<std::pair<Function *, int>> InlineHistory);
-
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_UTILS_CLONING_H

llvm/lib/Analysis/InlineOrder.cpp

@@ -199,10 +199,7 @@ class MLPriority {
   int Cost = INT_MAX;
 };
 
-template <typename PriorityT>
-class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
-  using T = std::pair<CallBase *, int>;
-
+template <typename PriorityT> class PriorityInlineOrder : public InlineOrder {
   bool hasLowerPriority(const CallBase *L, const CallBase *R) const {
     const auto I1 = Priorities.find(L);
     const auto I2 = Priorities.find(R);
@@ -243,31 +240,21 @@ class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
 
   size_t size() override { return Heap.size(); }
 
-  void push(const T &Elt) override {
-    CallBase *CB = Elt.first;
-    const int InlineHistoryID = Elt.second;
-
+  void push(CallBase *CB) override {
     Heap.push_back(CB);
     Priorities[CB] = PriorityT(CB, FAM, Params);
     std::push_heap(Heap.begin(), Heap.end(), isLess);
-    InlineHistoryMap[CB] = InlineHistoryID;
   }
 
-  T pop() override {
+  CallBase *pop() override {
     assert(size() > 0);
     pop_heap_adjust();
 
-    CallBase *CB = Heap.pop_back_val();
-    T Result = std::make_pair(CB, InlineHistoryMap[CB]);
-    InlineHistoryMap.erase(CB);
-    return Result;
+    return Heap.pop_back_val();
   }
 
-  void erase_if(function_ref<bool(T)> Pred) override {
-    auto PredWrapper = [=](CallBase *CB) -> bool {
-      return Pred(std::make_pair(CB, InlineHistoryMap[CB]));
-    };
-    llvm::erase_if(Heap, PredWrapper);
+  void erase_if(function_ref<bool(CallBase *)> Pred) override {
+    llvm::erase_if(Heap, Pred);
     std::make_heap(Heap.begin(), Heap.end(), isLess);
   }
 
@@ -284,7 +271,7 @@ class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
 
 AnalysisKey llvm::PluginInlineOrderAnalysis::Key;
 
-std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
+std::unique_ptr<InlineOrder>
 llvm::getDefaultInlineOrder(FunctionAnalysisManager &FAM,
                             const InlineParams &Params,
                             ModuleAnalysisManager &MAM, Module &M) {
@@ -309,9 +296,10 @@ llvm::getDefaultInlineOrder(FunctionAnalysisManager &FAM,
   return nullptr;
 }
 
-std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
-llvm::getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
-                     ModuleAnalysisManager &MAM, Module &M) {
+std::unique_ptr<InlineOrder> llvm::getInlineOrder(FunctionAnalysisManager &FAM,
+                                                  const InlineParams &Params,
+                                                  ModuleAnalysisManager &MAM,
+                                                  Module &M) {
   if (MAM.isPassRegistered<PluginInlineOrderAnalysis>()) {
     LLVM_DEBUG(dbgs() << "    Current used priority: plugin ---- \n");
     return MAM.getResult<PluginInlineOrderAnalysis>(M).Factory(FAM, Params, MAM,

llvm/lib/IR/Verifier.cpp

@@ -549,6 +549,7 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   void visitAccessGroupMetadata(const MDNode *MD);
   void visitCapturesMetadata(Instruction &I, const MDNode *Captures);
   void visitAllocTokenMetadata(Instruction &I, MDNode *MD);
+  void visitInlineHistoryMetadata(Instruction &I, MDNode *MD);
 
   template <class Ty> bool isValidMetadataArray(const MDTuple &N);
 #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N);
@@ -5619,6 +5620,20 @@ void Verifier::visitAllocTokenMetadata(Instruction &I, MDNode *MD) {
         "expected integer constant", MD);
 }
 
+void Verifier::visitInlineHistoryMetadata(Instruction &I, MDNode *MD) {
+  Check(isa<CallBase>(I), "!inline_history should only exist on calls", &I);
+  for (Metadata *Op : MD->operands()) {
+    // Can be null when a function is erased.
+    if (!Op)
+      continue;
+    Check(isa<ValueAsMetadata>(Op) &&
+              isa<Function>(cast<ValueAsMetadata>(Op)
+                                ->getValue()
+                                ->stripPointerCastsAndAliases()),
+          "!inline_history operands must be functions or null", MD);
+  }
+}
+
 /// verifyInstruction - Verify that an instruction is well formed.
 ///
 void Verifier::visitInstruction(Instruction &I) {
@@ -5851,6 +5866,9 @@ void Verifier::visitInstruction(Instruction &I) {
   if (MDNode *MD = I.getMetadata(LLVMContext::MD_alloc_token))
     visitAllocTokenMetadata(I, MD);
 
+  if (MDNode *MD = I.getMetadata(LLVMContext::MD_inline_history))
+    visitInlineHistoryMetadata(I, MD);
+
   if (MDNode *N = I.getDebugLoc().getAsMDNode()) {
     CheckDI(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N);
     visitMDNode(*N, AreDebugLocsAllowed::Yes);

llvm/lib/Transforms/IPO/AlwaysInliner.cpp

@@ -53,8 +53,9 @@ bool AlwaysInlineImpl(
     BasicBlock *Block = CB.getParent();
 
     InlineFunctionInfo IFI(GetAssumptionCache, &PSI);
-    InlineResult Res = InlineFunction(CB, IFI, /*MergeAttributes=*/true,
-                                      &GetAAR(Callee), InsertLifetime);
+    InlineResult Res = InlineFunction(
+        CB, IFI, /*MergeAttributes=*/true, &GetAAR(Callee), InsertLifetime,
+        /*TrackInlineHistory=*/NewCallSites != nullptr);
     if (!Res.isSuccess()) {
       ORE.emit([&]() {
         return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
@@ -140,8 +141,7 @@ bool AlwaysInlineImpl(
         continue;
 
       // Detect recursion.
-      if (Callee == F ||
-          inlineHistoryIncludes(Callee, InlineHistoryID, InlineHistory)) {
+      if (Callee == F) {
         ORE.emit([&]() {
           return OptimizationRemarkMissed("inline", "NotInlined",
                                           CB->getDebugLoc(), CB->getParent())