[GR-77557] Pin initial layer singleton accesses to the initial layer in doRunInNewThread

PullRequest: graal/24742

Author: Zeavee

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/JavaMainWrapper.java

@@ -50,7 +50,6 @@
 import org.graalvm.word.impl.Word;
 
 import com.oracle.svm.core.jfr.events.ShutdownEvent;
-import com.oracle.svm.guest.staging.jdk.RuntimeSupport;
 import com.oracle.svm.core.jni.JNIJavaVMList;
 import com.oracle.svm.core.jni.functions.JNIFunctionTables;
 import com.oracle.svm.core.log.Log;
@@ -72,6 +71,7 @@
 import com.oracle.svm.guest.staging.core.UnmanagedMemoryUtil;
 import com.oracle.svm.guest.staging.core.thread.OSThreadHandle;
 import com.oracle.svm.guest.staging.jdk.InternalVMMethod;
+import com.oracle.svm.guest.staging.jdk.RuntimeSupport;
 import com.oracle.svm.sdk.staging.layeredimage.LayeredCompilationBehavior;
 import com.oracle.svm.sdk.staging.layeredimage.LayeredCompilationBehavior.Behavior;
 import com.oracle.svm.shared.Uninterruptible;
@@ -277,14 +277,13 @@ private static int doRun(int argc, CCharPointerPointer argv) {
     private static int doRunInNewThread(int argc, CCharPointerPointer argv) {
         try {
             Isolate isolate = createMainIsolate(argc, argv);
-            long javaStackSize = PlatformThreads.singleton().getMainThreadRunnerJavaStackSize();
-            PlatformThreads.singleton().releaseMainThreadFromCurrent();
+            long javaStackSize = getJavaStackSize();
             WordPointer threadExitStatus = StackValue.get(WordPointer.class);
             /*
              * The launcher blocks in a no-transition OS join, so detach it before the runner can
              * request VM operations.
              */
-            VMThreads.singleton().detachCurrentThread();
+            detachCurrentThread();
             int exitCode = startAndJoinMainRunner(isolate, javaStackSize, threadExitStatus);
 
             runShutdownOnInitialThread(isolate);
@@ -294,6 +293,20 @@ private static int doRunInNewThread(int argc, CCharPointerPointer argv) {
         }
     }
 
+    @Uninterruptible(reason = "Thread state not setup yet.")
+    @LayeredCompilationBehavior(Behavior.PINNED_TO_INITIAL_LAYER)
+    private static long getJavaStackSize() {
+        long javaStackSize = PlatformThreads.singleton().getMainThreadRunnerJavaStackSize();
+        PlatformThreads.singleton().releaseMainThreadFromCurrent();
+        return javaStackSize;
+    }
+
+    @Uninterruptible(reason = "Thread state not setup yet.")
+    @LayeredCompilationBehavior(Behavior.PINNED_TO_INITIAL_LAYER)
+    private static void detachCurrentThread() {
+        VMThreads.singleton().detachCurrentThread();
+    }
+
     /**
      * Starts and joins the runner while the current launcher thread is detached from the isolate.
      */

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java

@@ -29,23 +29,27 @@
 import java.util.function.Predicate;
 
 import org.graalvm.collections.EconomicMap;
+import org.graalvm.collections.EconomicSet;
 
 import com.oracle.graal.pointsto.api.HostVM;
 import com.oracle.graal.pointsto.meta.AnalysisType;
 import com.oracle.graal.pointsto.meta.AnalysisUniverse;
 import com.oracle.svm.core.c.BoxedRelocatedPointer;
 import com.oracle.svm.core.code.ImageCodeInfo;
-import com.oracle.svm.shared.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
 import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
-import com.oracle.svm.shared.util.VMError;
 import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl;
 import com.oracle.svm.hosted.FeatureImpl.BeforeCompilationAccessImpl;
 import com.oracle.svm.hosted.imagelayer.HostedImageLayerBuildingSupport;
 import com.oracle.svm.hosted.imagelayer.InitialLayerFeature;
 import com.oracle.svm.hosted.imagelayer.SVMImageLayerLoader;
 import com.oracle.svm.sdk.staging.layeredimage.LayeredCompilationBehavior;
+import com.oracle.svm.shared.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.shared.util.ReflectionUtil;
+import com.oracle.svm.shared.util.VMError;
+import com.oracle.svm.util.GuestAccess;
+
+import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
  * This feature contains some configs currently necessary to build an extension layer. We'll need
@@ -97,7 +101,17 @@ public void beforeCompilation(BeforeCompilationAccess a) {
         SVMImageLayerLoader imageLayerLoader = HostedImageLayerBuildingSupport.singleton().getLoader();
         EconomicMap<AnalysisType, Object> superCoreTypes = EconomicMap.create();
 
+        EconomicSet<ResolvedJavaType> extensionLayerCoreTypes = EconomicSet.create();
+        /*
+         * This type is meant to be only accessed in the application layer if the application needs
+         * a main routine, so it should not be checked here.
+         */
+        extensionLayerCoreTypes.add(GuestAccess.get().lookupType("com.oracle.svm.core.JavaRunMainRoutinePointerHolder"));
+
         for (var type : universe.getTypes()) {
+            if (extensionLayerCoreTypes.contains(type.getWrapped())) {
+                continue;
+            }
             /*
              * These checks allow to ensure that core types can be treated as closed (see
              * PointsToAnalysis.isClosed(AnalysisType)).