Signal low space semaphore on StackOverflowError

Related issue: #174

src/de.hpi.swa.trufflesqueak/src/de/hpi/swa/trufflesqueak/image/SqueakImageContext.java

@@ -70,6 +70,7 @@
 import de.hpi.swa.trufflesqueak.nodes.plugins.JPEGReader;
 import de.hpi.swa.trufflesqueak.nodes.plugins.Zip;
 import de.hpi.swa.trufflesqueak.nodes.plugins.ffi.InterpreterProxy;
+import de.hpi.swa.trufflesqueak.nodes.process.SignalSemaphoreNodeGen;
 import de.hpi.swa.trufflesqueak.shared.SqueakImageLocator;
 import de.hpi.swa.trufflesqueak.tools.SqueakMessageInterceptor;
 import de.hpi.swa.trufflesqueak.util.ArrayUtils;
@@ -168,6 +169,10 @@ public final class SqueakImageContext {
     private ContextObject interopExceptionThrowingContextPrototype;
     public ContextObject lastSeenContext;
 
+    /* Low space handling */
+    private static final int LOW_SPACE_NUM_SKIPPED_SENDS = 4;
+    private int lowSpaceSkippedSendsCount;
+
     @CompilationFinal private ClassObject fractionClass;
     private PointersObject parserSharedInstance;
     private AbstractSqueakObject requestorSharedInstanceOrNil;
@@ -679,6 +684,34 @@ public void setSemaphore(final int index, final AbstractSqueakObject semaphore)
         setSpecialObject(index, semaphore);
     }
 
+    /**
+     * Ensure the active process is saved and try to signal low space semaphore (see
+     * #setSignalLowSpaceFlagAndSaveProcess). The JVM has just thrown a {@link StackOverflowError},
+     * so thread stack space is limited. To avoid hitting the limit again, free up some space by
+     * unwinding a couple of sends before actually signaling the low space semaphore.
+     */
+    public StackOverflowError tryToSignalLowSpace(final VirtualFrame frame, final StackOverflowError stackOverflowError) {
+        CompilerAsserts.neverPartOfCompilation();
+        final Object lastSavedProcess = getSpecialObject(SPECIAL_OBJECT.PROCESS_SIGNALING_LOW_SPACE);
+        if (lastSavedProcess == NilObject.SINGLETON) {
+            setSpecialObject(SPECIAL_OBJECT.PROCESS_SIGNALING_LOW_SPACE, getActiveProcessSlow());
+        }
+        if (lowSpaceSkippedSendsCount < LOW_SPACE_NUM_SKIPPED_SENDS) {
+            lowSpaceSkippedSendsCount++;
+            throw stackOverflowError; // continue further up the sender chain
+        } else {
+            final Object lowSpaceSemaphoreOrNil = getSpecialObject(SPECIAL_OBJECT.THE_LOW_SPACE_SEMAPHORE);
+            try {
+                SignalSemaphoreNodeGen.executeUncached(frame, this, lowSpaceSemaphoreOrNil);
+            } catch (final ProcessSwitch ps) {
+                // success! reset counter and continue in new process
+                lowSpaceSkippedSendsCount = 0;
+                throw ps;
+            }
+            throw CompilerDirectives.shouldNotReachHere("Failed to signal low space semaphore.", stackOverflowError);
+        }
+    }
+
     public boolean hasDisplay() {
         return display != null;
     }

src/de.hpi.swa.trufflesqueak/src/de/hpi/swa/trufflesqueak/model/layout/ObjectLayouts.java

@@ -315,7 +315,7 @@ public static final class SPECIAL_OBJECT {
         public static final int CLASS_CHARACTER = 19;
         public static final int SELECTOR_DOES_NOT_UNDERSTAND = 20;
         public static final int SELECTOR_CANNOT_RETURN = 21;
-        public static final int THE_INPUT_SEMAPHORE = 22;
+        public static final int PROCESS_SIGNALING_LOW_SPACE = 22;
         public static final int SPECIAL_SELECTORS = 23;
         public static final int CHARACTER_TABLE = 24;
         public static final int SELECTOR_MUST_BE_BOOLEAN = 25;

src/de.hpi.swa.trufflesqueak/src/de/hpi/swa/trufflesqueak/nodes/ExecuteBytecodeNode.java

@@ -52,8 +52,11 @@ public Object execute(final VirtualFrame frame, final int startPC) {
         try {
             return interpretBytecode(frame, startPC);
         } catch (final NonLocalReturn nlr) {
-            /** {@link getHandleNonLocalReturnNode()} acts as {@link BranchProfile} */
+            /* {@link getHandleNonLocalReturnNode()} acts as {@link BranchProfile} */
             return getHandleNonLocalReturnNode().executeHandle(frame, nlr);
+        } catch (final StackOverflowError e) {
+            CompilerDirectives.transferToInterpreter();
+            throw getContext().tryToSignalLowSpace(frame, e);
         }
     }