[GR-73325] Template and Top of Stack optimization

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/HotSpotTruffleBytecodeHandlerStub.java

@@ -62,7 +62,7 @@ public HotSpotTruffleBytecodeHandlerStub(OptionValues options, HotSpotProviders
     protected StructuredGraph getGraph(DebugContext debug, CompilationIdentifier compilationId) {
         try {
             HotSpotGraphKit kit = new HotSpotGraphKit(debug, callsite.getEnclosingMethod(), providers, providers.getGraphBuilderPlugins(), compilationId, callsite.getStubName(), false, true);
-            return callsite.createStub(kit, callsite.getEnclosingMethod(), false, null, null);
+            return callsite.createStub(kit, callsite.getEnclosingMethod(), false, null, null, null);
         } catch (Exception e) {
             throw GraalError.shouldNotReachHere(e); // ExcludeFromJacocoGeneratedReport
         }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/host/OutlineBytecodeHandlerPhase.java

@@ -61,11 +61,12 @@ protected TruffleBytecodeHandlerTypes getTruffleBytecodeHandlerTypes(TruffleKnow
         return new TruffleBytecodeHandlerTypes(truffleKnownHostTypes.BytecodeInterpreterSwitch,
                         truffleKnownHostTypes.BytecodeInterpreterHandlerConfig,
                         truffleKnownHostTypes.BytecodeInterpreterHandler,
-                        truffleKnownHostTypes.BytecodeInterpreterFetchOpcode);
+                        truffleKnownHostTypes.BytecodeInterpreterFetchOpcode,
+                        truffleKnownHostTypes.BytecodeInterpreterDefaultHandler);
     }
 
     protected TruffleBytecodeHandlerCallsite getTruffleBytecodeHandlerCallsite(ResolvedJavaMethod enclosingMethod, int bci, ResolvedJavaMethod targetMethod, TruffleBytecodeHandlerTypes truffleTypes) {
-        return new TruffleBytecodeHandlerCallsite(enclosingMethod, bci, targetMethod, truffleTypes);
+        return new TruffleBytecodeHandlerCallsite(enclosingMethod, bci, targetMethod, 0, truffleTypes);
     }
 
     protected Function<ResolvedJavaField, ResolvedJavaField> getFieldMap(@SuppressWarnings("unused") MetaAccessProvider metaAccess) {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/host/TruffleKnownHostTypes.java

@@ -55,6 +55,7 @@ public final class TruffleKnownHostTypes extends AbstractKnownTruffleTypes {
     public final ResolvedJavaType BytecodeInterpreterHandler = lookupType("com.oracle.truffle.api.HostCompilerDirectives$BytecodeInterpreterHandler");
     public final ResolvedJavaType BytecodeInterpreterHandlerConfig = lookupType("com.oracle.truffle.api.HostCompilerDirectives$BytecodeInterpreterHandlerConfig");
     public final ResolvedJavaType BytecodeInterpreterFetchOpcode = lookupType("com.oracle.truffle.api.HostCompilerDirectives$BytecodeInterpreterFetchOpcode");
+    public final ResolvedJavaType BytecodeInterpreterDefaultHandler = lookupType("com.oracle.truffle.api.HostCompilerDirectives$BytecodeInterpreterDefaultHandler");
     public final ResolvedJavaType BytecodeInterpreterSwitchBoundary = lookupType("com.oracle.truffle.api.HostCompilerDirectives$BytecodeInterpreterSwitchBoundary");
     public final ResolvedJavaType InliningCutoff = lookupType("com.oracle.truffle.api.HostCompilerDirectives$InliningCutoff");
     public final ResolvedJavaType InliningRoot = lookupTypeOptional("com.oracle.truffle.api.HostCompilerDirectives$InliningRoot");

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/nodes/TruffleBytecodeHandlerDispatchAddressNode.java

@@ -26,10 +26,11 @@
 
 import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_4;
 
-import java.util.function.Supplier;
+import java.util.function.IntFunction;
 
 import jdk.graal.compiler.core.common.memory.BarrierType;
 import jdk.graal.compiler.core.common.type.StampFactory;
+import jdk.graal.compiler.debug.GraalError;
 import jdk.graal.compiler.graph.NodeClass;
 import jdk.graal.compiler.nodeinfo.NodeCycles;
 import jdk.graal.compiler.nodeinfo.NodeInfo;
@@ -60,21 +61,26 @@ public final class TruffleBytecodeHandlerDispatchAddressNode extends FixedWithNe
     public static final NodeClass<TruffleBytecodeHandlerDispatchAddressNode> TYPE = NodeClass.create(TruffleBytecodeHandlerDispatchAddressNode.class);
 
     @Input ValueNode opcode;
+    @Input ValueNode template;
 
-    private final Supplier<Object> bytecodeHandlerTableSupplier;
+    private final IntFunction<Object> bytecodeHandlerTableSupplier;
 
-    public TruffleBytecodeHandlerDispatchAddressNode(ValueNode opcode, Supplier<Object> bytecodeHandlerTableSupplier) {
+    public TruffleBytecodeHandlerDispatchAddressNode(ValueNode opcode, ValueNode template, IntFunction<Object> bytecodeHandlerTableSupplier) {
         super(TYPE, StampFactory.forKind(JavaKind.Long));
         this.opcode = opcode;
+        this.template = template;
         this.bytecodeHandlerTableSupplier = bytecodeHandlerTableSupplier;
     }
 
     @Override
     public void lower(LoweringTool tool) {
-        // Treat bytecodeHandlerTable as a long[] and return bytecodeHandlerTable[opcode]
         StructuredGraph graph = graph();
-        JavaConstant bytecodeHandlerTable = tool.getSnippetReflection().forObject(bytecodeHandlerTableSupplier.get());
-        ConstantNode base = ConstantNode.forConstant(bytecodeHandlerTable, tool.getMetaAccess(), graph);
+        GraalError.guarantee(template.isConstant(), "%s is not constant", template);
+        int templateIndex = template.asJavaConstant().asInt();
+        Object bytecodeHandlerTable = bytecodeHandlerTableSupplier.apply(templateIndex);
+        JavaConstant bytecodeHandlerTableConstant = tool.getSnippetReflection().forObject(bytecodeHandlerTable);
+
+        ConstantNode base = ConstantNode.forConstant(bytecodeHandlerTableConstant, tool.getMetaAccess(), graph);
         ConstantNode baseOffset = ConstantNode.forLong(tool.getMetaAccess().getArrayBaseOffset(JavaKind.Long), graph);
         ConstantNode indexShift = ConstantNode.forInt(CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(JavaKind.Long)), graph);
         ValueNode extendedOpcode = graph.addOrUnique(ZeroExtendNode.create(opcode, 64, NodeView.DEFAULT));

substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/SubstrateOutlineBytecodeHandlerPhase.java

@@ -70,9 +70,9 @@ public final class SubstrateOutlineBytecodeHandlerPhase extends OutlineBytecodeH
      * the stub method. This map is collected in {@link TruffleBytecodeHandlerInvokePlugin} and both
      * the keys and values are of {@link com.oracle.graal.pointsto.meta.AnalysisMethod}.
      */
-    private final EconomicMap<ResolvedJavaMethod, ResolvedJavaMethod> registeredBytecodeHandlers;
+    private final EconomicMap<ResolvedJavaMethod, ResolvedJavaMethod[]> registeredBytecodeHandlers;
 
-    public SubstrateOutlineBytecodeHandlerPhase(EconomicMap<ResolvedJavaMethod, ResolvedJavaMethod> registeredBytecodeHandlers) {
+    public SubstrateOutlineBytecodeHandlerPhase(EconomicMap<ResolvedJavaMethod, ResolvedJavaMethod[]> registeredBytecodeHandlers) {
         this.registeredBytecodeHandlers = registeredBytecodeHandlers;
     }
 
@@ -91,7 +91,7 @@ protected TruffleBytecodeHandlerTypes getTruffleBytecodeHandlerTypes(TruffleKnow
      * @param targetMethod a {@link HostedMethod}
      */
     private SubstrateTruffleBytecodeHandlerStub getStub(ResolvedJavaMethod targetMethod) {
-        return (SubstrateTruffleBytecodeHandlerStub) unwrap(registeredBytecodeHandlers.get(unwrap(targetMethod)));
+        return (SubstrateTruffleBytecodeHandlerStub) unwrap(registeredBytecodeHandlers.get(unwrap(targetMethod))[0]);
     }
 
     @Override
@@ -116,7 +116,7 @@ protected FixedNode replaceInvoke(HighTierContext context, TruffleBytecodeHandle
         StructuredGraph graph = invoke.asNode().graph();
         CallTargetNode oldCallTargetNode = invoke.callTarget();
         ResolvedJavaMethod targetMethod = oldCallTargetNode.targetMethod();
-        ResolvedJavaMethod analysisStub = registeredBytecodeHandlers.get(unwrap(targetMethod));
+        ResolvedJavaMethod analysisStub = registeredBytecodeHandlers.get(unwrap(targetMethod))[0];
         HostedMethod hostedStub = ((HostedMetaAccess) context.getMetaAccess()).getUniverse().optionalLookup(analysisStub);
 
         SubstrateMethodCallTargetNode newCallTargetNode = graph.add(new SubstrateMethodCallTargetNode(CallTargetNode.InvokeKind.Static, hostedStub, arguments,

substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/SubstrateTruffleBytecodeHandlerStub.java

@@ -62,6 +62,7 @@
 import jdk.graal.compiler.annotation.AnnotationValue;
 import jdk.graal.compiler.debug.DebugContext;
 import jdk.graal.compiler.debug.GraalError;
+import jdk.graal.compiler.java.FrameStateBuilder;
 import jdk.graal.compiler.nodes.MultiReturnNode;
 import jdk.graal.compiler.nodes.ParameterNode;
 import jdk.graal.compiler.nodes.ReturnNode;
@@ -99,28 +100,30 @@ public final class SubstrateTruffleBytecodeHandlerStub extends NonBytecodeMethod
     private final TruffleBytecodeHandlerCallsite callsite;
     private final boolean threading;
     private final boolean needSafepoint;
-    private final boolean isDefault;
+    private final boolean isDefaultHandler;
     private final ResolvedJavaMethod nextOpcodeMethod;
+    private final ResolvedJavaMethod defaultHandlerMethod;
 
     public SubstrateTruffleBytecodeHandlerStub(SubstrateTruffleBytecodeHandlerStubHelper stubHolder, ResolvedJavaType declaringClass, String stubName,
-                    TruffleBytecodeHandlerCallsite callsite, boolean threading, ResolvedJavaMethod nextOpcodeMethod, boolean needSafepoint, boolean isDefault) {
+                    TruffleBytecodeHandlerCallsite callsite, boolean threading, ResolvedJavaMethod nextOpcodeMethod, ResolvedJavaMethod defaultHandlerMethod, boolean needSafepoint, boolean isDefaultHandler) {
         super(stubName, true, declaringClass, ResolvedSignature.fromList(callsite.getArgumentTypes(),
                         callsite.getReturnType()), declaringClass.getDeclaredConstructors(false)[0].getConstantPool());
         this.stubHolder = stubHolder;
         this.callsite = callsite;
         this.threading = threading;
-        this.isDefault = isDefault;
         this.nextOpcodeMethod = nextOpcodeMethod;
+        this.defaultHandlerMethod = defaultHandlerMethod;
         this.needSafepoint = needSafepoint;
+        this.isDefaultHandler = isDefaultHandler;
     }
 
     @Override
     public StructuredGraph buildGraph(DebugContext debug, AnalysisMethod method, HostedProviders providers, Purpose purpose) {
         HostedGraphKit kit = new HostedGraphKit(debug, providers, method);
-        if (isDefault) {
-            return createEmptyStub(kit);
+        if (isDefaultHandler) {
+            return createDefaultHandlerStub(kit, method);
         }
-        return callsite.createStub(kit, method, threading, nextOpcodeMethod, () -> stubHolder.getBytecodeHandlers(callsite.getEnclosingMethod()));
+        return callsite.createStub(kit, method, threading, nextOpcodeMethod, defaultHandlerMethod, index -> stubHolder.getBytecodeHandlers(callsite.getEnclosingMethod(), index));
     }
 
     /**
@@ -131,7 +134,7 @@ public StructuredGraph buildGraph(DebugContext debug, AnalysisMethod method, Hos
      * parameters as additional return results. This stub effectively terminates the threading and
      * triggers a re-dispatch of the bytecode in the caller.
      */
-    private StructuredGraph createEmptyStub(GraphKit kit) {
+    private StructuredGraph createDefaultHandlerStub(GraphKit kit, AnalysisMethod frameOwner) {
         StructuredGraph graph = kit.getGraph();
         graph.getGraphState().forceDisableFrameStateVerification();
 
@@ -152,10 +155,20 @@ private StructuredGraph createEmptyStub(GraphKit kit) {
             }
         }
 
-        MultiReturnNode multiReturnNode = kit.unique(new MultiReturnNode(returnResult, null));
-        multiReturnNode.getAdditionalReturnResults().addAll(Arrays.asList(parameterNodes));
+        if (defaultHandlerMethod != null) {
+            FrameStateBuilder frameStateBuilder = new FrameStateBuilder(kit, frameOwner, graph);
+            graph.start().setStateAfter(frameStateBuilder.create(callsite.getBci(), graph.start()));
+            ValueNode[] argumentsToDefaultHandler = callsite.createCalleeArguments(kit, parameterNodes);
+            GraalError.guarantee(defaultHandlerMethod.getSignature().getReturnKind() == JavaKind.Void,
+                            "Expected @BytecodeInterpreterDefaultHandler to return void: %s", defaultHandlerMethod.format("%H.%n(%p)"));
+            callsite.appendInvoke(defaultHandlerMethod, argumentsToDefaultHandler, frameStateBuilder, kit);
+            kit.append(new ReturnNode(callsite.createStubReturn(kit, parameterNodes, returnResult, null, argumentsToDefaultHandler)));
+        } else {
+            MultiReturnNode multiReturnNode = kit.unique(new MultiReturnNode(returnResult, null));
+            multiReturnNode.getAdditionalReturnResults().addAll(Arrays.asList(parameterNodes));
+            kit.append(new ReturnNode(multiReturnNode));
+        }
 
-        kit.append(new ReturnNode(multiReturnNode));
         graph.getDebug().dump(DebugContext.VERBOSE_LEVEL, graph, "Initial graph for default bytecode handler stub");
         return graph;
     }
@@ -307,6 +320,7 @@ static TruffleBytecodeHandlerTypes asTruffleBytecodeHandlerTypes(TruffleKnownHos
         return new TruffleBytecodeHandlerTypes(unwrap(truffleKnownHostTypes.BytecodeInterpreterSwitch),
                         unwrap(truffleKnownHostTypes.BytecodeInterpreterHandlerConfig),
                         unwrap(truffleKnownHostTypes.BytecodeInterpreterHandler),
-                        unwrap(truffleKnownHostTypes.BytecodeInterpreterFetchOpcode));
+                        unwrap(truffleKnownHostTypes.BytecodeInterpreterFetchOpcode),
+                        unwrap(truffleKnownHostTypes.BytecodeInterpreterDefaultHandler));
     }
 }