[cling] Move generators to ProcessSymbols JITDylib

This makes them run after the <Platform> library: It provides the
__cxa_atexit function, which must not be found in the process.

Author: hahnjo

interpreter/cling/lib/Interpreter/IncrementalJIT.cpp

@@ -501,6 +501,34 @@ IncrementalJIT::IncrementalJIT(
     return std::make_unique<SimpleCompiler>(*m_TM);
   });
 
+  char LinkerPrefix = this->m_TM->createDataLayout().getGlobalPrefix();
+
+  Builder.setProcessSymbolsJITDylibSetup([&](LLJIT& J) -> Expected<JITDylibSP> {
+    auto& JD = J.getExecutionSession().createBareJITDylib("<Process Symbols>");
+    // Process symbol resolution
+    auto HostProcessLookup =
+        RTDynamicLibrarySearchGenerator::GetForCurrentProcess(
+            LinkerPrefix, [this] { return m_CurrentProcessRT; },
+            [this](const SymbolStringPtr& Sym) {
+              return !m_ForbidDlSymbols.contains(*Sym);
+            });
+    if (!HostProcessLookup) {
+      return HostProcessLookup.takeError();
+    }
+    JD.addGenerator(std::move(*HostProcessLookup));
+
+    // This must come after process resolution, to  consistently resolve global
+    // symbols (e.g. std::cout) to the same address.
+    auto LibLookup = std::make_unique<RTDynamicLibrarySearchGenerator>(
+        llvm::sys::DynamicLibrary(ExtraLibHandle), LinkerPrefix,
+        [this] { return m_CurrentProcessRT; },
+        [this](const SymbolStringPtr& Sym) {
+          return !m_ForbidDlSymbols.contains(*Sym);
+        });
+    JD.addGenerator(std::move(LibLookup));
+    return &JD;
+  });
+
   if (Expected<std::unique_ptr<LLJIT>> JitInstance = Builder.create()) {
     Jit = std::move(*JitInstance);
   } else {
@@ -518,32 +546,9 @@ IncrementalJIT::IncrementalJIT(
       m_CompiledModules[Unsafe] = std::move(TSM);
     });
 
-  char LinkerPrefix = this->m_TM->createDataLayout().getGlobalPrefix();
-
-  // Process symbol resolution
-  auto HostProcessLookup
-    = RTDynamicLibrarySearchGenerator::GetForCurrentProcess(LinkerPrefix,
-                                              [&]{ return m_CurrentRT; },
-                                              [&](const SymbolStringPtr &Sym) {
-                                  return !m_ForbidDlSymbols.contains(*Sym); });
-  if (!HostProcessLookup) {
-    Err = HostProcessLookup.takeError();
-    return;
-  }
-  Jit->getMainJITDylib().addGenerator(std::move(*HostProcessLookup));
-
-  // This must come after process resolution, to  consistently resolve global
-  // symbols (e.g. std::cout) to the same address.
-  auto LibLookup = std::make_unique<RTDynamicLibrarySearchGenerator>(
-                       llvm::sys::DynamicLibrary(ExtraLibHandle), LinkerPrefix,
-                                              [&]{ return m_CurrentRT; },
-                                              [&](const SymbolStringPtr &Sym) {
-                                  return !m_ForbidDlSymbols.contains(*Sym); });
-  Jit->getMainJITDylib().addGenerator(std::move(LibLookup));
-
 #if defined(__linux__) && defined(__GLIBC__)
   // See comment in ListOfLibcNonsharedSymbols.
-  cantFail(Jit->getMainJITDylib().define(
+  cantFail(Jit->getProcessSymbolsJITDylib()->define(
       absoluteSymbols(GetListOfLibcNonsharedSymbols(*Jit))));
 #endif
 
@@ -585,8 +590,11 @@ std::unique_ptr<llvm::orc::DefinitionGenerator> IncrementalJIT::getGenerator() {
 }
 
 void IncrementalJIT::addModule(Transaction& T) {
-  ResourceTrackerSP RT = Jit->getMainJITDylib().createResourceTracker();
-  m_ResourceTrackers[&T] = RT;
+  ResourceTrackerSP MainRT = Jit->getMainJITDylib().createResourceTracker();
+  m_MainResourceTrackers[&T] = MainRT;
+  ResourceTrackerSP ProcessRT =
+      Jit->getProcessSymbolsJITDylib()->createResourceTracker();
+  m_ProcessResourceTrackers[&T] = ProcessRT;
 
   std::unique_ptr<Module> module = T.takeModule();
 
@@ -609,22 +617,26 @@ void IncrementalJIT::addModule(Transaction& T) {
 
   const Module *Unsafe = TSM.getModuleUnlocked();
   T.m_CompiledModule = Unsafe;
-  m_CurrentRT = RT;
+  m_CurrentProcessRT = ProcessRT;
 
-  if (Error Err = Jit->addIRModule(RT, std::move(TSM))) {
+  if (Error Err = Jit->addIRModule(MainRT, std::move(TSM))) {
     logAllUnhandledErrors(std::move(Err), errs(),
                           "[IncrementalJIT] addModule() failed: ");
     return;
   }
 }
 
 llvm::Error IncrementalJIT::removeModule(const Transaction& T) {
-  ResourceTrackerSP RT = std::move(m_ResourceTrackers[&T]);
-  if (!RT)
+  ResourceTrackerSP MainRT = std::move(m_MainResourceTrackers[&T]);
+  if (!MainRT)
     return llvm::Error::success();
+  ResourceTrackerSP ProcessRT = std::move(m_ProcessResourceTrackers[&T]);
 
-  m_ResourceTrackers.erase(&T);
-  if (Error Err = RT->remove())
+  m_MainResourceTrackers.erase(&T);
+  m_ProcessResourceTrackers.erase(&T);
+  if (Error Err = MainRT->remove())
+    return Err;
+  if (Error Err = ProcessRT->remove())
     return Err;
   auto iMod = m_CompiledModules.find(T.m_CompiledModule);
   if (iMod != m_CompiledModules.end())
@@ -667,7 +679,14 @@ void* IncrementalJIT::getSymbolAddress(StringRef Name, bool IncludeHostSymbols){
   if (!IncludeHostSymbols)
     insertInfo = m_ForbidDlSymbols.insert(Name);
 
-  Expected<llvm::orc::ExecutorAddr> Symbol = Jit->lookup(Name);
+  Expected<llvm::orc::ExecutorAddr> Symbol =
+      Jit->lookup(Jit->getMainJITDylib(), Name);
+  if (!Symbol) {
+    // FIXME: We should take advantage of the fact that all process symbols
+    // are now in a separate JITDylib; see also the comments and ideas in
+    // IncrementalExecutor::getAddressOfGlobal().
+    Symbol = Jit->lookup(*Jit->getProcessSymbolsJITDylib(), Name);
+  }
 
   // If m_ForbidDlSymbols already contained Name before we tried to insert it
   // then some calling frame has added it and will remove it later because its

interpreter/cling/lib/Interpreter/IncrementalJIT.h

@@ -65,7 +65,7 @@ class IncrementalJIT {
   /// Register a DefinitionGenerator to dynamically provide symbols for
   /// generated code that are not already available within the process.
   void addGenerator(std::unique_ptr<llvm::orc::DefinitionGenerator> G) {
-    Jit->getMainJITDylib().addGenerator(std::move(G));
+    Jit->getProcessSymbolsJITDylib()->addGenerator(std::move(G));
   }
 
   /// Return a `DefinitionGenerator` that can provide addresses for symbols
@@ -109,13 +109,14 @@ class IncrementalJIT {
   llvm::orc::SymbolMap m_InjectedSymbols;
   SharedAtomicFlag SkipHostProcessLookup;
   llvm::StringSet<> m_ForbidDlSymbols;
-  llvm::orc::ResourceTrackerSP m_CurrentRT;
+  llvm::orc::ResourceTrackerSP m_CurrentProcessRT;
 
   /// FIXME: If the relation between modules and transactions is a bijection, the
   /// mapping via module pointers here is unnecessary. The transaction should
   /// store the resource tracker directly and pass it to `remove()` for
   /// unloading.
-  std::map<const Transaction*, llvm::orc::ResourceTrackerSP> m_ResourceTrackers;
+  std::map<const Transaction*, llvm::orc::ResourceTrackerSP> m_MainResourceTrackers;
+  std::map<const Transaction*, llvm::orc::ResourceTrackerSP> m_ProcessResourceTrackers;
   std::map<const llvm::Module *, llvm::orc::ThreadSafeModule> m_CompiledModules;
 
   bool m_JITLink;