From 854a212295ab7c871100ff68ad557f57df69d1d1 Mon Sep 17 00:00:00 2001 From: Tom Shull Date: Thu, 6 Apr 2023 16:50:06 +0200 Subject: [PATCH] Add support for inlinebeforeanalysis for runtime compiled methods. --- .../compiler/replacements/PEGraphDecoder.java | 12 +- .../TruffleGraphBuilderPlugins.java | 12 +- .../pointsto/AbstractAnalysisEngine.java | 28 ++ .../oracle/graal/pointsto/AnalysisPolicy.java | 3 + .../com/oracle/graal/pointsto/api/HostVM.java | 16 +- .../pointsto/flow/DirectInvokeTypeFlow.java | 3 +- .../graal/pointsto/flow/InvokeTypeFlow.java | 74 ++-- .../pointsto/flow/MethodTypeFlowBuilder.java | 44 +-- .../BytecodeSensitiveAnalysisPolicy.java | 7 + .../graal/pointsto/meta/AnalysisMethod.java | 15 +- .../graal/pointsto/meta/InvokeInfo.java | 4 + .../pointsto/phases/InlineBeforeAnalysis.java | 2 +- .../InlineBeforeAnalysisGraphDecoder.java | 26 +- ...nlineBeforeAnalysisInlineInvokePlugin.java | 6 +- .../phases/InlineBeforeAnalysisPolicy.java | 126 ++++--- .../pointsto/results/StrengthenGraphs.java | 69 +++- .../typestate/DefaultAnalysisPolicy.java | 10 + .../DefaultSpecialInvokeTypeFlow.java | 15 + .../DefaultStaticInvokeTypeFlow.java | 13 + .../oracle/svm/common/meta/MultiMethod.java | 11 +- .../svm/core/code/FrameInfoEncoder.java | 5 + .../oracle/svm/core/deopt/Deoptimizer.java | 31 +- .../nodes/InlinedInvokeArgumentsNode.java | 65 ++++ .../graal/replacements/SubstrateGraphKit.java | 7 +- .../com/oracle/svm/core/hub/DynamicHub.java | 2 +- .../hosted/GraalGraphObjectReplacer.java | 14 +- .../LegacyRuntimeCompilationFeature.java | 7 + .../ParseOnceRuntimeCompilationFeature.java | 228 +++++++++++- .../hosted/RuntimeCompilationFeature.java | 13 + .../src/com/oracle/svm/hosted/SVMHost.java | 72 +++- .../svm/hosted/SubstrateStrengthenGraphs.java | 40 ++- .../hosted/analysis/SVMParsingSupport.java | 14 + .../flow/SVMMethodTypeFlowBuilder.java | 22 +- .../svm/hosted/code/DeoptimizationUtils.java | 4 +- .../code/SubstrateCompilationDirectives.java | 7 + .../heap/PodFactorySubstitutionMethod.java | 28 +- .../svm/hosted/nodes/DeoptProxyNode.java | 2 +- .../svm/hosted/phases/EnumSwitchPlugin.java | 2 +- .../svm/hosted/phases/HostedGraphKit.java | 5 +- .../InlineBeforeAnalysisPolicyImpl.java | 249 ++----------- .../InlineBeforeAnalysisPolicyUtils.java | 331 ++++++++++++++++++ .../IntrinsificationPluginRegistry.java | 6 +- .../oracle/svm/truffle/TruffleFeature.java | 87 +++-- 43 files changed, 1289 insertions(+), 448 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/InlinedInvokeArgumentsNode.java create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java diff --git a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/replacements/PEGraphDecoder.java b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/replacements/PEGraphDecoder.java index 850788826759..7b0192c0e910 100644 --- a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/replacements/PEGraphDecoder.java +++ b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/replacements/PEGraphDecoder.java @@ -214,6 +214,10 @@ public boolean isInlinedMethod() { return caller != null; } + public ValueNode[] getArguments() { + return arguments; + } + /** * Gets the call stack representing this method scope and its callers. */ @@ -1208,6 +1212,8 @@ protected LoopScope doInline(PEMethodScope methodScope, LoopScope loopScope, Inv } } + predecessor = afterMethodScopeCreation(inlineScope, predecessor); + LoopScope inlineLoopScope = createInitialLoopScope(inlineScope, predecessor); /* @@ -1229,6 +1235,10 @@ protected LoopScope doInline(PEMethodScope methodScope, LoopScope loopScope, Inv return inlineLoopScope; } + protected FixedWithNextNode afterMethodScopeCreation(@SuppressWarnings("unused") PEMethodScope inlineScope, FixedWithNextNode predecessor) { + return predecessor; + } + @Override protected void afterMethodScope(MethodScope methodScope) { /* @@ -1349,7 +1359,7 @@ protected void finishInlining(MethodScope is) { } /* - * Usage the handles that we have on the return value and the exception to update the + * Use the handles that we have on the return value and the exception to update the * orderId->Node table. */ registerNode(loopScope, invokeData.invokeOrderId, returnValue, true, true); diff --git a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/truffle/compiler/substitutions/TruffleGraphBuilderPlugins.java b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/truffle/compiler/substitutions/TruffleGraphBuilderPlugins.java index 6ded60a4bc74..f76f4ff26e6f 100644 --- a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/truffle/compiler/substitutions/TruffleGraphBuilderPlugins.java +++ b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/truffle/compiler/substitutions/TruffleGraphBuilderPlugins.java @@ -173,8 +173,16 @@ private static void registerTruffleSafepointPlugins(InvocationPlugins plugins, K r.register(new RequiredInvocationPlugin("poll", com.oracle.truffle.api.nodes.Node.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg) { - if (arg.isConstant()) { - assert TruffleSafepointInsertionPhase.allowsSafepoints(b.getGraph()) : "TruffleSafepoint.poll only expected to be removed in Truffle compilations."; + if (!TruffleSafepointInsertionPhase.allowsSafepoints(b.getGraph())) { + if (!canDelayIntrinsification) { + /* + * TruffleSafepoint.poll only expected to be removed in Truffle + * compilations. + */ + throw failPEConstant(b, arg); + } + return false; + } else if (arg.isConstant()) { return true; } else if (canDelayIntrinsification) { return false; diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java index c58d997f9d80..248d7167a051 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java @@ -37,6 +37,7 @@ import org.graalvm.compiler.debug.Indent; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.nodes.DeoptBciSupplier; +import org.graalvm.compiler.nodes.StateSplit; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.printer.GraalDebugHandlersFactory; @@ -363,6 +364,33 @@ public static BytecodePosition syntheticSourcePosition(Node node, ResolvedJavaMe if (node instanceof DeoptBciSupplier) { bci = ((DeoptBciSupplier) node).bci(); } + + /* + * If the node is a state split, then we can read the framestate to get a better guess of + * the node's position + */ + if (node instanceof StateSplit stateSplit) { + var frameState = stateSplit.stateAfter(); + if (frameState != null) { + if (frameState.outerFrameState() != null) { + /* + * If the outer framestate is not null, then inlinebeforeanalysis has inlined this call. We + * store the position of the original call to prevent recursive flows. + */ + var current = frameState; + while (current.outerFrameState() != null) { + current = current.outerFrameState(); + } + assert method.equals(current.getMethod()); + bci = current.bci; + } else if (bci == BytecodeFrame.UNKNOWN_BCI) { + /* + * If there is a single framestate, then use its bci if nothing better is available + */ + bci = frameState.bci; + } + } + } return new BytecodePosition(null, method, bci); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AnalysisPolicy.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AnalysisPolicy.java index 8f61fccc902d..563bb3694dc2 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AnalysisPolicy.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AnalysisPolicy.java @@ -184,6 +184,9 @@ public abstract AbstractSpecialInvokeTypeFlow createSpecialInvokeTypeFlow(Byteco public abstract AbstractStaticInvokeTypeFlow createStaticInvokeTypeFlow(BytecodePosition invokeLocation, AnalysisType receiverType, PointsToAnalysisMethod targetMethod, TypeFlow[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethod.MultiMethodKey callerMultiMethodKey); + public abstract InvokeTypeFlow createDeoptInvokeTypeFlow(BytecodePosition invokeLocation, AnalysisType receiverType, PointsToAnalysisMethod targetMethod, + TypeFlow[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethod.MultiMethodKey callerMultiMethodKey); + public abstract MethodFlowsGraphInfo staticRootMethodGraph(PointsToAnalysis bb, PointsToAnalysisMethod method); public abstract AnalysisContext allocationContext(PointsToAnalysis bb, MethodFlowsGraph callerGraph); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java index e7c55ca74c7a..4f400fa007db 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiConsumer; +import java.util.function.Function; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; @@ -222,9 +223,9 @@ public boolean hasNeverInlineDirective(ResolvedJavaMethod method) { return true; } - public InlineBeforeAnalysisGraphDecoder createInlineBeforeAnalysisGraphDecoder(BigBang bb, AnalysisMethod method, StructuredGraph resultGraph) { + public InlineBeforeAnalysisGraphDecoder createInlineBeforeAnalysisGraphDecoder(BigBang bb, AnalysisMethod method, StructuredGraph resultGraph) { /* No inlining by the static analysis unless explicitly overwritten by the VM. */ - return new InlineBeforeAnalysisGraphDecoder<>(bb, InlineBeforeAnalysisPolicy.NO_INLINING, resultGraph, bb.getProviders(method), null); + return new InlineBeforeAnalysisGraphDecoder(bb, InlineBeforeAnalysisPolicy.NO_INLINING, resultGraph, bb.getProviders(method), null); } @SuppressWarnings("unused") @@ -316,7 +317,7 @@ public interface MultiMethodAnalysisPolicy { * called multiple times with the same parameters; hence, values returned by this * method are allowed to be cached */ - Collection determineCallees(BigBang bb, T implementation, T target, MultiMethod.MultiMethodKey callerMultiMethodKey, InvokeTypeFlow parsingReason); + Collection determineCallees(BigBang bb, T implementation, T target, MultiMethod.MultiMethodKey callerMultiMethodKey, InvokeTypeFlow invokeFlow); /** * Decides whether the caller's flows should be linked to callee's parameters flows. @@ -346,7 +347,7 @@ public interface MultiMethodAnalysisPolicy { protected static final MultiMethodAnalysisPolicy DEFAULT_MULTIMETHOD_ANALYSIS_POLICY = new MultiMethodAnalysisPolicy() { @Override - public Collection determineCallees(BigBang bb, T implementation, T target, MultiMethod.MultiMethodKey callerMultiMethodKey, InvokeTypeFlow parsingReason) { + public Collection determineCallees(BigBang bb, T implementation, T target, MultiMethod.MultiMethodKey callerMultiMethodKey, InvokeTypeFlow invokeFlow) { return List.of(implementation); } @@ -382,4 +383,11 @@ public boolean ignoreInstanceOfTypeDisallowed() { public boolean isUnknownValueField(@SuppressWarnings("unused") AnalysisField field) { return false; } + + /** + * Returns the function Strengthen Graphs should use to improve types based on analysis results. + */ + public Function getStrengthenGraphsToTargetFunction(@SuppressWarnings("unused") MultiMethod.MultiMethodKey key) { + return (t) -> t; + } } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/DirectInvokeTypeFlow.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/DirectInvokeTypeFlow.java index d23e905b1d67..d1852cc0db0f 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/DirectInvokeTypeFlow.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/DirectInvokeTypeFlow.java @@ -33,7 +33,6 @@ import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.meta.PointsToAnalysisMethod; import com.oracle.graal.pointsto.util.LightImmutableCollection; -import com.oracle.svm.common.meta.MultiMethod; import com.oracle.svm.common.meta.MultiMethod.MultiMethodKey; import jdk.vm.ci.code.BytecodePosition; @@ -81,7 +80,7 @@ public final Collection getAllCallees() { * callee to be set, but for it not to be linked. */ Collection result = LightImmutableCollection.toCollection(this, CALLEES_ACCESSOR); - assert result.stream().filter(MultiMethod::isOriginalMethod).allMatch(AnalysisMethod::isImplementationInvoked); + assert result.stream().filter(m -> m.isOriginalMethod()).allMatch(AnalysisMethod::isImplementationInvoked); return result; } return Collections.emptyList(); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/InvokeTypeFlow.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/InvokeTypeFlow.java index e997463cf23c..aeac64420dee 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/InvokeTypeFlow.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/InvokeTypeFlow.java @@ -148,6 +148,7 @@ public TypeFlow getActualReturn() { } public void setActualReturn(PointsToAnalysis bb, boolean isStatic, ActualReturnTypeFlow actualReturn) { + assert this.actualReturn == null; this.actualReturn = actualReturn; bb.analysisPolicy().linkActualReturn(bb, isStatic, this); } @@ -209,13 +210,20 @@ protected void linkCallee(PointsToAnalysis bb, boolean isStatic, MethodFlowsGrap * a non-state-transfer link. The link only exists for a proper iteration of type * flow graphs, but the state update of 'this' parameters is achieved through direct * state update in VirtualInvokeTypeFlow.update and SpecialInvokeTypeFlow.update by - * calling FormalReceiverTypeFlow.addReceiverState. This happens because the formal - * receiver , i.e., 'this' parameter, state must ONLY reflect those objects of the - * actual receiver that generated the context for the method clone which it belongs - * to. A direct link would instead transfer all the objects of compatible type from - * the actual receiver to the formal receiver. + * calling FormalReceiverTypeFlow.addReceiverState. + * + * In other words, while the receiver param (actualParameters[0] when !isStatic) is + * linked to the FormalReceiverTypeFlow of the callee, type information is not + * propagated along this edge. This is accomplished by overriding the addState + * method within FormalReceiverTypeFlow. + * + * This action is taken because the formal receiver (i.e., 'this' parameter) state + * must ONLY reflect those objects of the actual receiver that generated the context + * for the method clone which it belongs to. A direct link would instead transfer + * all the objects of compatible type from the actual receiver to the formal + * receiver. */ - if (actualParam != null && formalParam != null /* && (i != 0 || isStatic) */) { + if (actualParam != null && formalParam != null) { // create the use link: // (formalParam, callerContext) -> (actualParam, calleeContext) // Note: the callerContext is an implicit property of the current InvokeTypeFlow @@ -237,38 +245,26 @@ protected void linkCallee(PointsToAnalysis bb, boolean isStatic, MethodFlowsGrap } public void linkReturn(PointsToAnalysis bb, boolean isStatic, MethodFlowsGraphInfo calleeFlows) { - if (bb.getHostVM().getMultiMethodAnalysisPolicy().performReturnLinking(callerMultiMethodKey, calleeFlows.getMethod().getMultiMethodKey())) { - if (actualReturn != null) { - if (bb.optimizeReturnedParameter()) { - int paramNodeIndex = calleeFlows.getMethod().getTypeFlow().getReturnedParameterIndex(); - if (paramNodeIndex != -1) { - if (isStatic || paramNodeIndex != 0) { - TypeFlow actualParam = actualParameters[paramNodeIndex]; - actualParam.addUse(bb, actualReturn); - } else { - /* - * The callee returns `this`. The formal-receiver state is updated in - * InvokeTypeFlow#updateReceiver() for each linked callee and every time - * the formal-receiver is updated then the same update state is - * propagated to the actual-return. One may think that we could simply - * add a direct use link from the formal-receiver in the callee to the - * actual-return in the caller to get the state propagation - * automatically. But that would be wrong because then the actual-return - * would get the state from *all* the other places that callee may be - * called from, and that would defeat the purpose of this optimization: - * we want just the receiver state from the caller of current invoke to - * reach the actual-return. - */ - } + if (actualReturn != null && bb.getHostVM().getMultiMethodAnalysisPolicy().performReturnLinking(callerMultiMethodKey, calleeFlows.getMethod().getMultiMethodKey())) { + if (bb.optimizeReturnedParameter()) { + int paramNodeIndex = calleeFlows.getMethod().getTypeFlow().getReturnedParameterIndex(); + if (paramNodeIndex != -1) { + if (isStatic || paramNodeIndex != 0) { + TypeFlow actualParam = actualParameters[paramNodeIndex]; + actualParam.addUse(bb, actualReturn); } else { /* - * The callee may have a return type, hence the actualReturn is non-null, - * but it might throw an exception instead of returning, hence the formal - * return is null. + * The callee returns `this`. The formal-receiver state is updated in + * InvokeTypeFlow#updateReceiver() for each linked callee and every time the + * formal-receiver is updated then the same update state is propagated to + * the actual-return. One may think that we could simply add a direct use + * link from the formal-receiver in the callee to the actual-return in the + * caller to get the state propagation automatically. But that would be + * wrong because then the actual-return would get the state from *all* the + * other places that callee may be called from, and that would defeat the + * purpose of this optimization: we want just the receiver state from the + * caller of current invoke to reach the actual-return. */ - if (calleeFlows.getReturnFlow() != null) { - calleeFlows.getReturnFlow().addUse(bb, actualReturn); - } } } else { /* @@ -280,6 +276,14 @@ public void linkReturn(PointsToAnalysis bb, boolean isStatic, MethodFlowsGraphIn calleeFlows.getReturnFlow().addUse(bb, actualReturn); } } + } else { + /* + * The callee may have a return type, hence the actualReturn is non-null, but it + * might throw an exception instead of returning, hence the formal return is null. + */ + if (calleeFlows.getReturnFlow() != null) { + calleeFlows.getReturnFlow().addUse(bb, actualReturn); + } } } } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java index 75ea05432374..47dd8bca02cc 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java @@ -1432,7 +1432,7 @@ protected boolean delegateNodeProcessing(FixedNode n, TypeFlowsOfNodes state) { protected void processMacroInvokable(TypeFlowsOfNodes state, MacroInvokable macro, boolean installResult) { ValueNode macroNode = macro.asNode(); BytecodePosition invokePosition = getInvokePosition(macro, macroNode); - processMethodInvocation(state, macroNode, macro.getInvokeKind(), (PointsToAnalysisMethod) macro.getTargetMethod(), macro.getArguments(), installResult, invokePosition); + processMethodInvocation(state, macroNode, macro.getInvokeKind(), (PointsToAnalysisMethod) macro.getTargetMethod(), macro.getArguments(), installResult, invokePosition, false); } /* Reconstruct the macro node invoke position, avoiding cycles in the parsing backtrace. */ @@ -1457,7 +1457,7 @@ private BytecodePosition getInvokePosition(MacroInvokable macro, ValueNode macro protected void processMethodInvocation(TypeFlowsOfNodes state, Invoke invoke, InvokeKind invokeKind, PointsToAnalysisMethod targetMethod, NodeInputList arguments) { FixedNode invokeNode = invoke.asFixedNode(); BytecodePosition invokePosition = getInvokePosition(invokeNode); - processMethodInvocation(state, invokeNode, invokeKind, targetMethod, arguments, true, invokePosition); + processMethodInvocation(state, invokeNode, invokeKind, targetMethod, arguments, true, invokePosition, false); } /* Get a reasonable position for inlined invokes, avoiding cycles in the parsing backtrace. */ @@ -1487,8 +1487,9 @@ private BytecodePosition getInvokePosition(FixedNode invokeNode) { return invokePosition; } - protected void processMethodInvocation(TypeFlowsOfNodes state, ValueNode invoke, InvokeKind invokeKind, PointsToAnalysisMethod targetMethod, NodeInputList arguments, - boolean installResult, BytecodePosition invokeLocation) { + protected void processMethodInvocation(TypeFlowsOfNodes state, ValueNode invoke, InvokeKind invokeKind, PointsToAnalysisMethod targetMethod, + NodeInputList arguments, + boolean installResult, BytecodePosition invokeLocation, boolean createDeoptInvokeTypeFlow) { // check if the call is allowed bb.isCallAllowed(bb, method, targetMethod, invokeLocation); @@ -1496,7 +1497,6 @@ protected void processMethodInvocation(TypeFlowsOfNodes state, ValueNode invoke, * Collect the parameters builders into an array so that we don't capture the `state` * reference in the closure. */ - boolean targetIsStatic = Modifier.isStatic(targetMethod.getModifiers()); TypeFlowBuilder[] actualParametersBuilders = new TypeFlowBuilder[arguments.size()]; for (int i = 0; i < actualParametersBuilders.length; i++) { @@ -1533,19 +1533,23 @@ protected void processMethodInvocation(TypeFlowsOfNodes state, ValueNode invoke, MultiMethod.MultiMethodKey multiMethodKey = method.getMultiMethodKey(); InvokeTypeFlow invokeFlow; - switch (invokeKind) { - case Static: - invokeFlow = bb.analysisPolicy().createStaticInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, multiMethodKey); - break; - case Special: - invokeFlow = bb.analysisPolicy().createSpecialInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, multiMethodKey); - break; - case Virtual: - case Interface: - invokeFlow = bb.analysisPolicy().createVirtualInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, multiMethodKey); - break; - default: - throw shouldNotReachHere(); + if (createDeoptInvokeTypeFlow) { + invokeFlow = bb.analysisPolicy().createDeoptInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, multiMethodKey); + } else { + switch (invokeKind) { + case Static: + invokeFlow = bb.analysisPolicy().createStaticInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, multiMethodKey); + break; + case Special: + invokeFlow = bb.analysisPolicy().createSpecialInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, multiMethodKey); + break; + case Virtual: + case Interface: + invokeFlow = bb.analysisPolicy().createVirtualInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, multiMethodKey); + break; + default: + throw shouldNotReachHere(); + } } flowsGraph.addInvoke(StaticAnalysisResultsBuilder.uniqueKey(invoke), invokeFlow); @@ -1566,7 +1570,7 @@ protected void processMethodInvocation(TypeFlowsOfNodes state, ValueNode invoke, return invokeFlow; }); - if (invoke.asNode().getStackKind() == JavaKind.Object) { + if (!createDeoptInvokeTypeFlow && invoke.asNode().getStackKind() == JavaKind.Object) { /* Create the actual return builder. */ AnalysisType returnType = (AnalysisType) targetMethod.getSignature().getReturnType(null); TypeFlowBuilder actualReturnBuilder = TypeFlowBuilder.create(bb, invoke.asNode(), ActualReturnTypeFlow.class, () -> { @@ -1577,7 +1581,7 @@ protected void processMethodInvocation(TypeFlowsOfNodes state, ValueNode invoke, * Only set the actual return in the invoke when it is materialized, i.e., it is * used by other flows. */ - invokeFlow.setActualReturn(bb, targetIsStatic, actualReturn); + invokeFlow.setActualReturn(bb, targetMethod.isStatic(), actualReturn); actualReturn.setInvokeFlow(invokeFlow); return actualReturn; }); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/bytecode/BytecodeSensitiveAnalysisPolicy.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/bytecode/BytecodeSensitiveAnalysisPolicy.java index 9318ddb23e28..c724ac5adbac 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/bytecode/BytecodeSensitiveAnalysisPolicy.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/bytecode/BytecodeSensitiveAnalysisPolicy.java @@ -70,6 +70,7 @@ import com.oracle.graal.pointsto.typestore.SplitFieldTypeStore; import com.oracle.graal.pointsto.typestore.UnifiedArrayElementsTypeStore; import com.oracle.graal.pointsto.typestore.UnifiedFieldTypeStore; +import com.oracle.graal.pointsto.util.AnalysisError; import com.oracle.graal.pointsto.util.ListUtils; import com.oracle.graal.pointsto.util.ListUtils.UnsafeArrayList; import com.oracle.graal.pointsto.util.ListUtils.UnsafeArrayListClosable; @@ -340,6 +341,12 @@ public AbstractStaticInvokeTypeFlow createStaticInvokeTypeFlow(BytecodePosition return new BytecodeSensitiveStaticInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, callerMultiMethodKey); } + @Override + public InvokeTypeFlow createDeoptInvokeTypeFlow(BytecodePosition invokeLocation, AnalysisType receiverType, PointsToAnalysisMethod targetMethod, + TypeFlow[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethod.MultiMethodKey callerMultiMethodKey) { + throw AnalysisError.shouldNotReachHere("This has not been implemented yet"); + } + @Override public MethodFlowsGraphInfo staticRootMethodGraph(PointsToAnalysis bb, PointsToAnalysisMethod method) { return ((CallSiteSensitiveMethodTypeFlow) method.getTypeFlow()).addContext(bb, contextPolicy.emptyContext(), null); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java index a698b629e1f8..751ef58fcf5c 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java @@ -874,10 +874,23 @@ public StructuredGraph decodeAnalyzedGraph(DebugContext debug, Iterable { + assert analyzedGraph.getAssumptions().equals(result.getAssumptions()); + } + case NO -> { + assert analyzedGraph.getAssumptions() == null && result.getAssumptions() == null; + } + } return result; } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/InvokeInfo.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/InvokeInfo.java index 7328133d0596..2997aa12fd8c 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/InvokeInfo.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/InvokeInfo.java @@ -54,4 +54,8 @@ public interface InvokeInfo { BytecodePosition getPosition(); boolean isDirectInvoke(); + + default boolean isDeoptInvokeTypeFlow() { + return false; + } } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysis.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysis.java index 165fb3b9205a..52edf805ab50 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysis.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysis.java @@ -72,7 +72,7 @@ public static StructuredGraph decodeGraph(BigBang bb, AnalysisMethod method, Ana try (DebugContext.Scope s = debug.scope("InlineBeforeAnalysis", result)) { if (bb.strengthenGraalGraphs() && Options.InlineBeforeAnalysis.getValue(bb.getOptions())) { - InlineBeforeAnalysisGraphDecoder decoder = bb.getHostVM().createInlineBeforeAnalysisGraphDecoder(bb, method, result); + InlineBeforeAnalysisGraphDecoder decoder = bb.getHostVM().createInlineBeforeAnalysisGraphDecoder(bb, method, result); decoder.decode(method); } else { /* diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java index 3367e1569072..a04efd7a0acd 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java @@ -51,14 +51,15 @@ import com.oracle.graal.pointsto.flow.AnalysisParsedGraph; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.meta.HostedProviders; +import com.oracle.graal.pointsto.util.AnalysisError; import jdk.vm.ci.meta.ResolvedJavaMethod; -public class InlineBeforeAnalysisGraphDecoder extends PEGraphDecoder { +public class InlineBeforeAnalysisGraphDecoder extends PEGraphDecoder { public class InlineBeforeAnalysisMethodScope extends PEMethodScope { - public final S policyScope; + private final InlineBeforeAnalysisPolicy.AbstractPolicyScope policyScope; private boolean inliningAborted; @@ -76,7 +77,7 @@ public class InlineBeforeAnalysisMethodScope extends PEMethodScope { graph.getDebug().logv(" ".repeat(inliningDepth) + "createRootScope for " + method.format("%H.%n(%p)") + ": " + policyScope); } } else { - policyScope = policy.openCalleeScope((cast(caller)).policyScope); + policyScope = policy.openCalleeScope(method, (cast(caller)).policyScope); if (graph.getDebug().isLogEnabled()) { graph.getDebug().logv(" ".repeat(inliningDepth) + "openCalleeScope for " + method.format("%H.%n(%p)") + ": " + policyScope); } @@ -85,14 +86,14 @@ public class InlineBeforeAnalysisMethodScope extends PEMethodScope { } protected final BigBang bb; - protected final InlineBeforeAnalysisPolicy policy; + protected final InlineBeforeAnalysisPolicy policy; - public InlineBeforeAnalysisGraphDecoder(BigBang bb, InlineBeforeAnalysisPolicy policy, StructuredGraph graph, HostedProviders providers, LoopExplosionPlugin loopExplosionPlugin) { + public InlineBeforeAnalysisGraphDecoder(BigBang bb, InlineBeforeAnalysisPolicy policy, StructuredGraph graph, HostedProviders providers, LoopExplosionPlugin loopExplosionPlugin) { super(AnalysisParsedGraph.HOST_ARCHITECTURE, graph, providers, loopExplosionPlugin, providers.getGraphBuilderPlugins().getInvocationPlugins(), new InlineInvokePlugin[]{new InlineBeforeAnalysisInlineInvokePlugin(policy)}, null, policy.nodePlugins, null, null, - new ConcurrentHashMap<>(), new ConcurrentHashMap<>(), true, false); + new ConcurrentHashMap<>(), new ConcurrentHashMap<>(), policy.needsExplicitExceptions(), false); this.bb = bb; this.policy = policy; @@ -162,7 +163,7 @@ protected void maybeAbortInlining(MethodScope ms, @SuppressWarnings("unused") Lo if (graph.getDebug().isLogEnabled()) { graph.getDebug().logv(" ".repeat(methodScope.inliningDepth) + " node " + node + ": " + methodScope.policyScope); } - if (!policy.processNode(bb.getMetaAccess(), methodScope.method, methodScope.policyScope, node)) { + if (!methodScope.policyScope.processNode(bb.getMetaAccess(), methodScope.method, node)) { abortInlining(methodScope); } } @@ -204,8 +205,9 @@ protected void finishInlining(MethodScope is) { if (graph.getDebug().isLogEnabled()) { graph.getDebug().logv(" ".repeat(callerScope.inliningDepth) + " aborted " + invokeData.callTarget.targetMethod().format("%H.%n(%p)") + ": " + inlineScope.policyScope); } + AnalysisError.guarantee(inlineScope.policyScope.allowAbort(), "Unexpected abort: %s", inlineScope); if (callerScope.policyScope != null) { - policy.abortCalleeScope(callerScope.policyScope, inlineScope.policyScope); + callerScope.policyScope.abortCalleeScope(inlineScope.policyScope); } if (invokeData.invokePredecessor.next() != null) { killControlFlowNodes(inlineScope, invokeData.invokePredecessor.next()); @@ -229,7 +231,7 @@ protected void finishInlining(MethodScope is) { graph.getDebug().logv(" ".repeat(callerScope.inliningDepth) + " committed " + invokeData.callTarget.targetMethod().format("%H.%n(%p)") + ": " + inlineScope.policyScope); } if (callerScope.policyScope != null) { - policy.commitCalleeScope(callerScope.policyScope, inlineScope.policyScope); + callerScope.policyScope.commitCalleeScope(inlineScope.policyScope); } Object reason = graph.currentNodeSourcePosition() != null ? graph.currentNodeSourcePosition() : graph.method(); @@ -305,4 +307,10 @@ private void killControlFlowNodes(PEMethodScope inlineScope, FixedNode start) { protected InlineBeforeAnalysisMethodScope cast(MethodScope methodScope) { return (InlineBeforeAnalysisMethodScope) methodScope; } + + @Override + protected FixedWithNextNode afterMethodScopeCreation(PEMethodScope is, FixedWithNextNode predecessor) { + InlineBeforeAnalysisMethodScope inlineScope = cast(is); + return policy.processInvokeArgs(inlineScope.method, predecessor, inlineScope.getArguments()); + } } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java index 5b51a9d32b67..434dec5865ae 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java @@ -32,16 +32,16 @@ final class InlineBeforeAnalysisInlineInvokePlugin implements InlineInvokePlugin { - private final InlineBeforeAnalysisPolicy policy; + private final InlineBeforeAnalysisPolicy policy; - InlineBeforeAnalysisInlineInvokePlugin(InlineBeforeAnalysisPolicy policy) { + InlineBeforeAnalysisInlineInvokePlugin(InlineBeforeAnalysisPolicy policy) { this.policy = policy; } @Override public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { if (policy.shouldInlineInvoke(b, method, args)) { - return InlineInfo.createStandardInlineInfo(method); + return policy.createInvokeInfo(method); } else { return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java index c3c2436aa098..67b632ab65fe 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java @@ -24,13 +24,15 @@ */ package com.oracle.graal.pointsto.phases; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo; import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; import com.oracle.graal.pointsto.meta.AnalysisMetaAccess; +import com.oracle.graal.pointsto.util.AnalysisError; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -38,21 +40,43 @@ * Provides the policy which methods are inlined by {@link InlineBeforeAnalysis}. If * {@link #shouldInlineInvoke} returns true for an invocation, the graph decoding goes into callees * and starts decoding. A new {@link #openCalleeScope scope is opened} for each callee so that the - * policy implementation can track each inlined method. As long as {@link #processNode} returns - * true, inlining is continued. If {@link #processNode} returns false, the inlining is - * {@link #abortCalleeScope aborted}. If {@link #processNode} returns true for all nodes of the - * callee, the inlining is {@link #commitCalleeScope committed}. + * policy implementation can track each inlined method. As long as + * {@link AbstractPolicyScope#processNode} returns true, inlining is continued. If + * {@link AbstractPolicyScope#processNode} returns false, the inlining is + * {@link AbstractPolicyScope#abortCalleeScope aborted}. If {@link AbstractPolicyScope#processNode} + * returns true for all nodes of the callee, the inlining is + * {@link AbstractPolicyScope#commitCalleeScope committed}. */ @SuppressWarnings("unused") -public class InlineBeforeAnalysisPolicy { - - public static final InlineBeforeAnalysisPolicy NO_INLINING = new InlineBeforeAnalysisPolicy<>(new NodePlugin[0]); +public abstract class InlineBeforeAnalysisPolicy { /** * A place for policy implementations to store per-callee information like the number of nodes * seen in the callee. */ - public interface Scope { + public abstract static class AbstractPolicyScope { + public final int inliningDepth; + + protected AbstractPolicyScope(int inliningDepth) { + this.inliningDepth = inliningDepth; + } + + public abstract boolean allowAbort(); + + public abstract void commitCalleeScope(AbstractPolicyScope callee); + + public abstract void abortCalleeScope(AbstractPolicyScope callee); + + /** + * Invoked for each node of the callee during graph decoding. If the method returns true, + * inlining is continued. If the method returns false, inlining is aborted. + * + * This method is called during graph decoding. The provided node itself is already fully + * decoded and canonicalized, i.e., all properties and predecessors of the node are + * available. But usages have not been decoded yet, so the implementation must not base any + * decision on the current list of usages. The list of usages is often but not always empty. + */ + public abstract boolean processNode(AnalysisMetaAccess metaAccess, ResolvedJavaMethod method, Node node); } protected final NodePlugin[] nodePlugins; @@ -61,44 +85,60 @@ protected InlineBeforeAnalysisPolicy(NodePlugin[] nodePlugins) { this.nodePlugins = nodePlugins; } - protected boolean shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { - return false; - } + protected abstract boolean shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args); - protected boolean tryInvocationPlugins() { - /* - * If an invocation plugin was unable to be used during bytecode parsing, then it will be - * retried during graph decoding. In the default case this should not happen. - */ - return false; - } + protected abstract InlineInfo createInvokeInfo(ResolvedJavaMethod method); - protected S createRootScope() { - return null; - } + protected abstract boolean needsExplicitExceptions(); - protected S openCalleeScope(S outer) { - throw GraalError.unimplementedParent(); // ExcludeFromJacocoGeneratedReport - } + protected abstract boolean tryInvocationPlugins(); - protected void commitCalleeScope(S outer, S callee) { - throw GraalError.unimplementedParent(); // ExcludeFromJacocoGeneratedReport - } + protected abstract FixedWithNextNode processInvokeArgs(ResolvedJavaMethod targetMethod, FixedWithNextNode insertionPoint, ValueNode[] arguments); - protected void abortCalleeScope(S outer, S callee) { - throw GraalError.unimplementedParent(); // ExcludeFromJacocoGeneratedReport - } + protected abstract AbstractPolicyScope createRootScope(); - /** - * Invoked for each node of the callee during graph decoding. If the method returns true, - * inlining is continued. If the method returns false, inlining is aborted. - * - * This method is called during graph decoding. The provided node itself is already fully - * decoded and canonicalized, i.e., all properties and predecessors of the node are available. - * But usages have not been decoded yet, so the implementation must not base any decision on the - * current list of usages. The list of usages is often but not always empty. - */ - protected boolean processNode(AnalysisMetaAccess metaAccess, ResolvedJavaMethod method, S scope, Node node) { - throw GraalError.unimplementedParent(); // ExcludeFromJacocoGeneratedReport - } + protected abstract AbstractPolicyScope openCalleeScope(ResolvedJavaMethod method, AbstractPolicyScope outer); + + public static final InlineBeforeAnalysisPolicy NO_INLINING = new InlineBeforeAnalysisPolicy(new NodePlugin[0]) { + + @Override + protected boolean shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { + return false; + } + + @Override + protected InlineInfo createInvokeInfo(ResolvedJavaMethod method) { + throw AnalysisError.shouldNotReachHere("NO_INLINING policy should not try to inline"); + } + + @Override + protected boolean needsExplicitExceptions() { + return true; + } + + @Override + protected boolean tryInvocationPlugins() { + /* + * If an invocation plugin was unable to be used during bytecode parsing, then it will + * be retried during graph decoding. In the default case this should not happen. + */ + return false; + } + + @Override + protected FixedWithNextNode processInvokeArgs(ResolvedJavaMethod targetMethod, FixedWithNextNode insertionPoint, ValueNode[] arguments) { + throw AnalysisError.shouldNotReachHere("NO_INLINING policy should not try to inline"); + } + + @Override + protected AbstractPolicyScope createRootScope() { + // No inlining is performed; nothing to keep track of + return null; + } + + @Override + protected AbstractPolicyScope openCalleeScope(ResolvedJavaMethod method, AbstractPolicyScope outer) { + throw AnalysisError.shouldNotReachHere("NO_INLINING policy should not try to inline"); + } + }; } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/results/StrengthenGraphs.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/results/StrengthenGraphs.java index be62f7ae6948..23b3c5bde7ba 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/results/StrengthenGraphs.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/results/StrengthenGraphs.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; import java.util.function.Supplier; import org.graalvm.collections.EconomicSet; @@ -102,6 +103,7 @@ import com.oracle.graal.pointsto.typestate.TypeState; import com.oracle.svm.util.ImageBuildStatistics; +import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.JavaMethodProfile; @@ -227,10 +229,14 @@ public JavaTypeProfile makeTypeProfile(AnalysisField field) { protected abstract FixedNode createUnreachable(StructuredGraph graph, CoreProviders providers, Supplier message); + protected abstract FixedNode createInvokeWithNullReceiverReplacement(StructuredGraph graph); + protected abstract void setInvokeProfiles(Invoke invoke, JavaTypeProfile typeProfile, JavaMethodProfile methodProfile); protected abstract String getTypeName(AnalysisType type); + protected abstract boolean simplifyDelegate(Node n, SimplifierTool tool); + // Wrapper to clearly identify phase class AnalysisStrengthenGraphsPhase extends BasePhase { final CanonicalizerPhase phase; @@ -268,6 +274,12 @@ class StrengthenSimplifier implements CustomSimplification { private final boolean allowConstantFolding; private final EconomicSet unreachableValues = EconomicSet.create(); + /** + * For runtime compiled methods, we must be careful to ensure new SubstrateTypes are not + * created due to the optimizations performed during the AnalysisStrengthenGraphsPhase. + */ + private final Function toTargetFunction; + StrengthenSimplifier(PointsToAnalysisMethod method, StructuredGraph graph) { this.graph = graph; this.methodFlow = method.getTypeFlow(); @@ -288,6 +300,8 @@ class StrengthenSimplifier implements CustomSimplification { * to support it within deoptimization targets and runtime-compiled methods. */ this.allowConstantFolding = method.isOriginalMethod() && strengthenGraphWithConstants; + + this.toTargetFunction = bb.getHostVM().getStrengthenGraphsToTargetFunction(method.getMultiMethodKey()); } private TypeFlow getNodeFlow(Node node) { @@ -315,7 +329,9 @@ public void simplify(Node n, SimplifierTool tool) { updateStampInPlace(node, strengthenStamp(node.stamp(NodeView.DEFAULT)), tool); } - if (n instanceof ParameterNode) { + if (simplifyDelegate(n, tool)) { + // handled elsewhere + } else if (n instanceof ParameterNode) { ParameterNode node = (ParameterNode) n; StartNode anchorPoint = graph.start(); Object newStampOrConstant = strengthenStampFromTypeFlow(node, parameterFlows[node.index()], anchorPoint, tool); @@ -467,7 +483,30 @@ private void handleInvoke(Invoke invoke, SimplifierTool tool) { if (node.isDeleted()) { /* Parameter stamp was empty, so invoke is unreachable. */ return; - } else if (newStampOrConstant != null) { + } + if (i == 0 && invoke.getInvokeKind() != CallTargetNode.InvokeKind.Static) { + /* + * Check for null receiver. If so, the invoke is unreachable. + * + * Note it is not necessary to check for an empty stamp, as in that case + * strengthenStampFromTypeFlow will make the invoke unreachable. + */ + boolean nullReceiver = false; + if (argument instanceof ConstantNode constantNode) { + nullReceiver = constantNode.getValue().isDefaultForKind(); + } + if (!nullReceiver && newStampOrConstant instanceof ObjectStamp stamp) { + nullReceiver = stamp.alwaysNull(); + } + if (!nullReceiver && newStampOrConstant instanceof Constant constantValue) { + nullReceiver = constantValue.isDefaultForKind(); + } + if (nullReceiver) { + invokeWithNullReceiver(invoke); + return; + } + } + if (newStampOrConstant != null) { ValueNode pi = insertPi(argument, newStampOrConstant, beforeInvoke); if (pi != null && pi != argument) { callTarget.replaceAllInputs(argument, pi); @@ -548,6 +587,15 @@ private void optimizeReturnedParameter(Collection callees, NodeI invoke.replaceAtUsages(returnedActualParameter); } + /** + * The invoke always has a null receiver, so it can be removed. + */ + protected void invokeWithNullReceiver(Invoke invoke) { + FixedNode replacement = createInvokeWithNullReceiverReplacement(graph); + ((FixedWithNextNode) invoke.predecessor()).setNext(replacement); + GraphUtil.killCFG(invoke.asFixedNode()); + } + /** * The invoke has no callee, i.e., it is unreachable. */ @@ -726,7 +774,7 @@ assert getSingleImplementorType(exactType) == null || exactType.equals(getSingle assert exactType.equals(getStrengthenStampType(exactType)); if (!oldStamp.isExactType() || !exactType.equals(oldType)) { - ResolvedJavaType targetType = toTarget(exactType); + ResolvedJavaType targetType = toTargetFunction.apply(exactType); if (targetType != null) { TypeReference typeRef = TypeReference.createExactTrusted(targetType); return StampFactory.object(typeRef, nonNull); @@ -764,7 +812,7 @@ assert getSingleImplementorType(baseType) == null || baseType.equals(getSingleIm assert typeStateTypes.stream().map(typeStateType -> newType.isAssignableFrom(typeStateType)).reduce(Boolean::logicalAnd).get(); if (!newType.equals(oldType) && (oldType != null || !newType.isJavaLangObject())) { - ResolvedJavaType targetType = toTarget(newType); + ResolvedJavaType targetType = toTargetFunction.apply(newType); if (targetType != null) { TypeReference typeRef = TypeReference.createTrustedWithoutAssumptions(targetType); return StampFactory.object(typeRef, nonNull); @@ -786,15 +834,6 @@ private void makeUnreachable(FixedNode node, CoreProviders providers, Supplier[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethod.MultiMethodKey callerMultiMethodKey) { + if (targetMethod.isStatic()) { + return new DefaultStaticInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, callerMultiMethodKey, true); + } else { + return new DefaultSpecialInvokeTypeFlow(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, callerMultiMethodKey, true); + } + } + @Override public MethodFlowsGraphInfo staticRootMethodGraph(PointsToAnalysis bb, PointsToAnalysisMethod method) { return method.getTypeFlow().getOrCreateMethodFlowsGraphInfo(bb, null); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultSpecialInvokeTypeFlow.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultSpecialInvokeTypeFlow.java index 7e4bafc906de..3863d4558d13 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultSpecialInvokeTypeFlow.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultSpecialInvokeTypeFlow.java @@ -42,10 +42,17 @@ final class DefaultSpecialInvokeTypeFlow extends AbstractSpecialInvokeTypeFlow { private volatile boolean calleesLinked = false; + private final boolean isDeoptInvokeTypeFlow; DefaultSpecialInvokeTypeFlow(BytecodePosition invokeLocation, AnalysisType receiverType, PointsToAnalysisMethod targetMethod, TypeFlow[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethodKey callerMultiMethodKey) { + this(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, callerMultiMethodKey, false); + } + + DefaultSpecialInvokeTypeFlow(BytecodePosition invokeLocation, AnalysisType receiverType, PointsToAnalysisMethod targetMethod, + TypeFlow[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethodKey callerMultiMethodKey, boolean isDeoptInvokeTypeFlow) { super(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, callerMultiMethodKey); + this.isDeoptInvokeTypeFlow = isDeoptInvokeTypeFlow; } @Override @@ -75,6 +82,9 @@ public void onObservedUpdate(PointsToAnalysis bb) { * Every time the actual receiver state changes in the caller the formal receiver state * needs to be updated as there is no direct update link between actual and formal * receivers. + * + * See InvokeTypeFlow#linkCallee for a more thorough explanation of the receiver + * linking. */ TypeState invokeState = filterReceiverState(bb, getReceiver().getState()); updateReceiver(bb, calleeFlows, invokeState); @@ -86,4 +96,9 @@ public void onObservedUpdate(PointsToAnalysis bb) { protected Collection getAllCalleesFlows(PointsToAnalysis bb) { return DefaultInvokeTypeFlowUtil.getAllCalleesFlows(this); } + + @Override + public boolean isDeoptInvokeTypeFlow() { + return isDeoptInvokeTypeFlow; + } } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultStaticInvokeTypeFlow.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultStaticInvokeTypeFlow.java index 83e8cc5e3054..51b661433fad 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultStaticInvokeTypeFlow.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultStaticInvokeTypeFlow.java @@ -42,9 +42,17 @@ import jdk.vm.ci.code.BytecodePosition; final class DefaultStaticInvokeTypeFlow extends AbstractStaticInvokeTypeFlow { + private final boolean isDeoptInvokeTypeFlow; + DefaultStaticInvokeTypeFlow(BytecodePosition invokeLocation, AnalysisType receiverType, PointsToAnalysisMethod targetMethod, TypeFlow[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethodKey callerMultiMethodKey) { + this(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, callerMultiMethodKey, false); + } + + DefaultStaticInvokeTypeFlow(BytecodePosition invokeLocation, AnalysisType receiverType, PointsToAnalysisMethod targetMethod, + TypeFlow[] actualParameters, ActualReturnTypeFlow actualReturn, MultiMethodKey callerMultiMethodKey, boolean isDeoptInvokeTypeFlow) { super(invokeLocation, receiverType, targetMethod, actualParameters, actualReturn, callerMultiMethodKey); + this.isDeoptInvokeTypeFlow = isDeoptInvokeTypeFlow; } @Override @@ -73,4 +81,9 @@ public void update(PointsToAnalysis bb) { protected Collection getAllCalleesFlows(PointsToAnalysis bb) { return DefaultInvokeTypeFlowUtil.getAllCalleesFlows(this); } + + @Override + public boolean isDeoptInvokeTypeFlow() { + return isDeoptInvokeTypeFlow; + } } diff --git a/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/meta/MultiMethod.java b/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/meta/MultiMethod.java index 5868e8db835c..1bc1ed09cff1 100644 --- a/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/meta/MultiMethod.java +++ b/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/meta/MultiMethod.java @@ -34,9 +34,16 @@ */ public interface MultiMethod { + static boolean isOriginalMethod(ResolvedJavaMethod method) { + if (method instanceof MultiMethod multiMethod) { + return multiMethod.isOriginalMethod(); + } + return false; + } + static boolean isDeoptTarget(ResolvedJavaMethod method) { - if (method instanceof MultiMethod) { - return ((MultiMethod) method).isDeoptTarget(); + if (method instanceof MultiMethod multiMethod) { + return multiMethod.isDeoptTarget(); } return false; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java index aa4a4194caba..7173b4d6d600 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java @@ -163,6 +163,11 @@ protected void fillSourceFields(BytecodeFrame bytecodeFrame, FrameInfoQueryResul resultFrameInfo.sourceLineNumber = targetFrameInfo.sourceLineNumber; resultFrameInfo.methodId = targetFrameInfo.methodId; } + } else { + /* + * GR-6000: Make this a hard error once all frame states are covered by + * DeoptEntries. + */ } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java index add89c356355..086c2e1245c8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java @@ -711,23 +711,23 @@ private DeoptimizedFrame deoptSourceFrameOperation(CodePointer pc, boolean ignor if (ignoreNonDeoptimizable) { return null; } else { - throw fatalDeoptimizationError("Deoptimization: cannot deoptimize a method that has no deoptimization entry point", deoptInfo); + throw fatalDeoptimizationError("Deoptimization: cannot deoptimize a method that has no deoptimization entry point", deoptInfo, frameInfo); } } CodeInfoQueryResult targetInfo = CodeInfoTable.lookupDeoptimizationEntrypoint(deoptInfo.getDeoptMethodOffset(), deoptInfo.getEncodedBci()); if (targetInfo == null || targetInfo.getFrameInfo() == null) { throw fatalDeoptimizationError( - "Deoptimization: no matching target bytecode frame found for deopt target method", deoptInfo); + "Deoptimization: no matching target bytecode frame found for deopt target method", deoptInfo, frameInfo); } else if (!targetInfo.getFrameInfo().isDeoptEntry()) { throw fatalDeoptimizationError( - "Deoptimization: target frame information not marked as deoptimization entry point", deoptInfo); + "Deoptimization: target frame information not marked as deoptimization entry point", deoptInfo, frameInfo); } else if (targetInfo.getFrameInfo().getDeoptMethod() != null && targetInfo.getFrameInfo().getDeoptMethod().hasCalleeSavedRegisters()) { /* * The deoptMethod is not guaranteed to be available, but this is only a last check, * to have a better error than the probable segfault. */ - throw fatalDeoptimizationError("Deoptimization: target method has callee saved registers, which are not properly restored by the deoptimization runtime", deoptInfo); + throw fatalDeoptimizationError("Deoptimization: target method has callee saved registers, which are not properly restored by the deoptimization runtime", deoptInfo, frameInfo); } VirtualFrame virtualFrame = constructTargetFrame(targetInfo, deoptInfo); if (previousVirtualFrame != null) { @@ -744,7 +744,7 @@ private DeoptimizedFrame deoptSourceFrameOperation(CodePointer pc, boolean ignor if (sourceChunk.getTotalFrameSize() < FrameAccess.wordSize()) { throw fatalDeoptimizationError( String.format("Insufficient space in frame for pointer to DeoptimizedFrame sourceChunkSize: %s, word size: %s", sourceChunk.getTotalFrameSize(), FrameAccess.wordSize()), - frameInfo); + frameInfo, frameInfo); } RelockObjectData[] relockObjectData = relockedObjects == null ? null : relockedObjects.toArray(new RelockObjectData[relockedObjects.size()]); @@ -1298,8 +1298,27 @@ private Pointer addressOfFrameArray0() { } static RuntimeException fatalDeoptimizationError(String originalMessage, FrameInfoQueryResult frameInfo) { + throw fatalDeoptimizationError0(originalMessage, frameInfo, frameInfo, false); + } + + static RuntimeException fatalDeoptimizationError(String originalMessage, FrameInfoQueryResult frameInfo, FrameInfoQueryResult topFrame) { + throw fatalDeoptimizationError0(originalMessage, frameInfo, topFrame, true); + } + + private static RuntimeException fatalDeoptimizationError0(String originalMessage, FrameInfoQueryResult frameInfo, FrameInfoQueryResult topFrame, boolean fullStack) { long encodedBci = frameInfo.getEncodedBci(); String message = String.format("%s%nencodedBci: %s (bci %s)%nMethod info: %s", originalMessage, encodedBci, FrameInfoDecoder.readableBci(encodedBci), frameInfo.getSourceReference()); - throw VMError.shouldNotReachHere(message); + StringBuilder sb = new StringBuilder(message); + if (fullStack) { + sb.append("%nFull Deoptimized Stack%n"); + } else { + sb.append("%nPartial Deoptimized Stack%n"); + } + FrameInfoQueryResult current = topFrame; + while (current != null) { + sb.append(current.getSourceReference()).append("%n"); + current = current.getCaller(); + } + throw VMError.shouldNotReachHere(sb.toString()); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/InlinedInvokeArgumentsNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/InlinedInvokeArgumentsNode.java new file mode 100644 index 000000000000..a4e290cb9ae6 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/InlinedInvokeArgumentsNode.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.graal.nodes; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_0; + +import org.graalvm.compiler.core.common.type.StampFactory; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.NodeInputList; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodeinfo.NodeSize; +import org.graalvm.compiler.nodes.FixedWithNextNode; +import org.graalvm.compiler.nodes.ValueNode; + +import jdk.vm.ci.meta.ResolvedJavaMethod; + +/** + * This node is used to temporarily track the arguments to a call which was inlined before analysis + * so that its parameter flows can be passed to other methods. The node itself will be deleted + * during AnalysisStrengthenGraphsPhase. + */ +@NodeInfo(cycles = CYCLES_0, size = NodeSize.SIZE_0) +public class InlinedInvokeArgumentsNode extends FixedWithNextNode { + public static final NodeClass TYPE = NodeClass.create(InlinedInvokeArgumentsNode.class); + + @Input protected NodeInputList arguments; + private final ResolvedJavaMethod targetMethod; + + @SuppressWarnings("this-escape") + public InlinedInvokeArgumentsNode(ResolvedJavaMethod targetMethod, ValueNode[] arguments) { + super(TYPE, StampFactory.forVoid()); + this.targetMethod = targetMethod; + this.arguments = new NodeInputList<>(this, arguments); + } + + public ResolvedJavaMethod getInvokeTarget() { + return targetMethod; + } + + public NodeInputList getArguments() { + return arguments; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java index 60ac0f50e26c..29b98383c613 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java @@ -96,10 +96,15 @@ public class SubstrateGraphKit extends GraphKit { private final FrameStateBuilder frameState; private int nextBCI; + // For GR-45916 this should be unconditionally true when parseOnce is enabled. + private static boolean trackNodeSourcePosition(boolean forceTrackNodeSourcePosition) { + return forceTrackNodeSourcePosition || (SubstrateOptions.parseOnce() && !SubstrateOptions.ParseOnceJIT.getValue()); + } + @SuppressWarnings("this-escape") public SubstrateGraphKit(DebugContext debug, ResolvedJavaMethod stubMethod, Providers providers, WordTypes wordTypes, GraphBuilderConfiguration.Plugins graphBuilderPlugins, CompilationIdentifier compilationId, boolean forceTrackNodeSourcePosition) { - super(debug, stubMethod, providers, wordTypes, graphBuilderPlugins, compilationId, null, forceTrackNodeSourcePosition || SubstrateOptions.parseOnce(), false); + super(debug, stubMethod, providers, wordTypes, graphBuilderPlugins, compilationId, null, trackNodeSourcePosition(forceTrackNodeSourcePosition), false); assert wordTypes != null : "Support for Word types is mandatory"; frameState = new FrameStateBuilder(this, stubMethod, graph); frameState.disableKindVerification(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index 7879c57e7810..535129683df5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -137,7 +137,7 @@ public final class DynamicHub implements AnnotatedElement, java.lang.reflect.Typ /** * The name of the class this hub is representing, as defined in {@link Class#getName()}. * - * Even though the field is only assinged in the constructor, it cannot be final: The + * Even though the field is only assigned in the constructor, it cannot be final: The * substitution system does not allow a final field when the target class has a field with the * same name. And using a different field name fails for various other reasons that are too * complicated to fix. Therefore, we ensure early constant folding using an invocation plugin diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java index 31ecad114125..bd853c8a5420 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java @@ -112,6 +112,11 @@ public class GraalGraphObjectReplacer implements Function { private final Field substrateTypeRawAllInstanceFieldsField; private final Field substrateMethodImplementationsField; + /** + * Tracks whether it is legal to create new types. + */ + private boolean forbidNewTypes = false; + public GraalGraphObjectReplacer(AnalysisUniverse aUniverse, SubstrateProviders sProviders, SubstrateUniverseFactory universeFactory) { this.aUniverse = aUniverse; this.sProviders = sProviders; @@ -284,6 +289,13 @@ public boolean typeCreated(JavaType original) { return types.containsKey(toAnalysisType(original)); } + /** + * After this is called no new types can be created. + */ + public void forbidNewTypes() { + forbidNewTypes = true; + } + public synchronized SubstrateType createType(JavaType original) { assert !(original instanceof SubstrateType) : original; if (original == null) { @@ -295,7 +307,7 @@ public synchronized SubstrateType createType(JavaType original) { SubstrateType sType = types.get(aType); if (sType == null) { - assert !(original instanceof HostedType) : "too late to create new type"; + VMError.guarantee(!(forbidNewTypes || (original instanceof HostedType)), "Too late to create a new type: %s", aType); aType.registerAsReachable("type reachable from Graal graphs"); DynamicHub hub = ((SVMHost) aUniverse.hostVM()).dynamicHub(aType); sType = new SubstrateType(aType.getJavaKind(), hub); diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/LegacyRuntimeCompilationFeature.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/LegacyRuntimeCompilationFeature.java index 4a68fc045678..b6b06e0754f1 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/LegacyRuntimeCompilationFeature.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/LegacyRuntimeCompilationFeature.java @@ -551,4 +551,11 @@ public void initializeAnalysisProviders(BigBang bb, Function RemoveUnneededDeoptSupport = new HostedOptionKey<>(false); + + @Option(help = "Perform InlineBeforeAnalysis on runtime compiled methods")// + public static final HostedOptionKey RuntimeCompilationInlineBeforeAnalysis = new HostedOptionKey<>(true); } public static final class CallTreeNode extends AbstractCallTreeNode { @@ -288,6 +307,7 @@ protected boolean shouldVerifyFrameStates() { private Map runtimeCandidateCallTree = null; private Map runtimeCompiledMethodCallTree = null; private HostedProviders analysisProviders = null; + private AllowInliningPredicate allowInliningPredicate = null; @Override public List> getRequiredFeatures() { @@ -312,6 +332,12 @@ public void beforeAnalysis(BeforeAnalysisAccess c) { beforeAnalysisHelper(c); } + @Override + public void registerAllowInliningPredicate(AllowInliningPredicate predicate) { + assert allowInliningPredicate == null; + allowInliningPredicate = predicate; + } + @Override public void initializeAnalysisProviders(BigBang bb, Function generator) { HostedProviders defaultProviders = bb.getProviders(ORIGINAL_METHOD); @@ -374,6 +400,11 @@ public void afterAnalysis(AfterAnalysisAccess access) { // call super after afterAnalysisHelper(); + + // after analysis has completed we must ensure no new SubstrateTypes are introduced + objectReplacer.forbidNewTypes(); + + System.out.println("Number of runtime compiled methods: " + getRuntimeCompiledMethods().size()); } @Override @@ -455,9 +486,15 @@ private void buildCallTrees() { assert runtimeMethod != null; for (InvokeInfo invokeInfo : runtimeMethod.getInvokes()) { - AnalysisMethod target = invokeInfo.getTargetMethod(); + AnalysisMethod invokeTarget = invokeInfo.getTargetMethod(); + if (invokeInfo.isDeoptInvokeTypeFlow()) { + assert SubstrateCompilationDirectives.isRuntimeCompiledMethod(invokeTarget); + invokeTarget = invokeTarget.getMultiMethod(ORIGINAL_METHOD); + } + AnalysisMethod target = invokeTarget; + assert target.isOriginalMethod(); for (AnalysisMethod implementation : invokeInfo.getAllCallees()) { - if (implementation.getMultiMethodKey() == RUNTIME_COMPILED_METHOD) { + if (SubstrateCompilationDirectives.isRuntimeCompiledMethod(implementation)) { var origImpl = implementation.getMultiMethod(ORIGINAL_METHOD); assert origImpl != null; runtimeCandidateCallTree.computeIfAbsent(new RuntimeCompilationCandidateImpl(origImpl, target), (candidate) -> { @@ -549,7 +586,12 @@ private void compileRuntimeCompiledMethod(DebugContext debug, HostedMethod metho * subset of the deopt entrypoints seen during evaluation. */ AnalysisMethod origMethod = method.getMultiMethod(ORIGINAL_METHOD).getWrapped(); - DeoptimizationUtils.registerDeoptEntries(graph, registeredRuntimeCompilations.contains(origMethod), ParseOnceRuntimeCompilationFeature::getDeoptTargetMethod); + DeoptimizationUtils.registerDeoptEntries(graph, registeredRuntimeCompilations.contains(origMethod), + (deoptEntryMethod -> { + PointsToAnalysisMethod deoptMethod = (PointsToAnalysisMethod) ((PointsToAnalysisMethod) deoptEntryMethod).getMultiMethod(DEOPT_TARGET_METHOD); + VMError.guarantee(deoptMethod != null, "New deopt target method seen: %s", deoptEntryMethod); + return deoptMethod; + })); assert RuntimeCompilationFeature.verifyNodes(graph); var previous = runtimeGraphs.put(method, graph); @@ -593,6 +635,8 @@ private void encodeRuntimeCompiledMethods() { @Override public void beforeCompilation(BeforeCompilationAccess c) { + System.out.println("Number of runtime compiled methods: " + getRuntimeCompiledMethods().size()); + beforeCompilationHelper(); System.out.println("Num runtime parsed methods " + parsedRuntimeMethods.size()); @@ -687,12 +731,6 @@ public SubstrateMethod prepareMethodForRuntimeCompilation(ResolvedJavaMethod met return sMethod; } - private static ResolvedJavaMethod getDeoptTargetMethod(ResolvedJavaMethod method) { - PointsToAnalysisMethod deoptMethod = (PointsToAnalysisMethod) ((PointsToAnalysisMethod) method).getMultiMethod(DEOPT_TARGET_METHOD); - VMError.guarantee(deoptMethod != null, "I need to implement this"); - return deoptMethod; - } - @Override protected void requireFrameInformationForMethodHelper(AnalysisMethod aMethod) { /* @@ -705,6 +743,7 @@ protected void requireFrameInformationForMethodHelper(AnalysisMethod aMethod) { } private class RuntimeCompilationParsingSupport implements SVMParsingSupport { + RuntimeCompilationInlineBeforeAnalysisPolicy runtimeInlineBeforeAnalysisPolicy = null; @Override public HostedProviders getHostedProviders(MultiMethod.MultiMethodKey key) { @@ -820,8 +859,13 @@ public boolean validateGraph(PointsToAnalysis bb, StructuredGraph graph) { * Register all FrameStates as DeoptEntries. */ AnalysisMethod origMethod = aMethod.getMultiMethod(ORIGINAL_METHOD); + + /* + * Because this graph will have its flowgraph immediately updated after this, there + * is no reason to make this method's flowgraph a stub on creation. + */ Collection recomputeMethods = DeoptimizationUtils.registerDeoptEntries(graph, registeredRuntimeCompilations.contains(origMethod), - ParseOnceRuntimeCompilationFeature::getDeoptTargetMethod); + (deoptEntryMethod -> ((PointsToAnalysisMethod) deoptEntryMethod).getOrCreateMultiMethod(DEOPT_TARGET_METHOD))); /* * If new frame states are found, then redo the type flow @@ -848,12 +892,168 @@ public boolean validateGraph(PointsToAnalysis bb, StructuredGraph graph) { return true; } + + @Override + public void initializeInlineBeforeAnalysisPolicy(SVMHost svmHost, InlineBeforeAnalysisPolicyUtils inliningUtils) { + if (Options.RuntimeCompilationInlineBeforeAnalysis.getValue()) { + assert runtimeInlineBeforeAnalysisPolicy == null; + runtimeInlineBeforeAnalysisPolicy = new RuntimeCompilationInlineBeforeAnalysisPolicy(svmHost, inliningUtils); + } + } + + @Override + public InlineBeforeAnalysisPolicy inlineBeforeAnalysisPolicy(MultiMethod.MultiMethodKey multiMethodKey, InlineBeforeAnalysisPolicy defaultPolicy) { + if (multiMethodKey == ORIGINAL_METHOD) { + return defaultPolicy; + } else if (multiMethodKey == DEOPT_TARGET_METHOD) { + return InlineBeforeAnalysisPolicy.NO_INLINING; + } else if (multiMethodKey == RUNTIME_COMPILED_METHOD) { + if (Options.RuntimeCompilationInlineBeforeAnalysis.getValue()) { + assert runtimeInlineBeforeAnalysisPolicy != null; + return runtimeInlineBeforeAnalysisPolicy; + } + return InlineBeforeAnalysisPolicy.NO_INLINING; + } else { + throw VMError.shouldNotReachHere("Unexpected method key: %s", multiMethodKey); + } + } + + @Override + public Function getStrengthenGraphsToTargetFunction(MultiMethod.MultiMethodKey key) { + if (key == RUNTIME_COMPILED_METHOD) { + /* + * For runtime compiled methods, we must be careful to ensure new SubstrateTypes are + * not created during the AnalysisStrengthenGraphsPhase. If the type does not + * already exist at this point (which is after the analysis phase), then we must + * return null. + */ + return (t) -> objectReplacer.typeCreated(t) ? t : null; + } + return null; + } + } + + /** + * This policy is a combination of the default InliningBeforeAnalysisImpl and the Trivial + * Inlining Policy. When the depth is less than + * {@code BytecodeParserOptions#InlineDuringParsingMaxDepth} (and all inlined parents were also + * trivial inlined), then it will continue to try to trivial inline methods (i.e., inline + * methods with less than a given number of nodes). Then, up to + * {@code InlineBeforeAnalysisPolicyUtils.Options#InlineBeforeAnalysisAllowedDepth}, inlining is + * enabled as long as the cumulative number of nodes inlined stays within the specified limits. + * + * Note that this policy is used exclusively by the runtime compiled methods, so there is no + * need to check multi-method keys; all callers (and callees) should be + * {@code RUNTIME_COMPILED_METHOD}s. + */ + private class RuntimeCompilationInlineBeforeAnalysisPolicy extends InlineBeforeAnalysisPolicy { + private final int accumulativeAllowedInliningDepth = InlineBeforeAnalysisAllowedDepth.getValue(); + private final int trivialAllowingInliningDepth = InlineDuringParsingMaxDepth.getValue(HostedOptionValues.singleton()); + + final SVMHost hostVM; + final InlineBeforeAnalysisPolicyUtils inliningUtils; + + protected RuntimeCompilationInlineBeforeAnalysisPolicy(SVMHost hostVM, InlineBeforeAnalysisPolicyUtils inliningUtils) { + super(new NodePlugin[]{new ConstantFoldLoadFieldPlugin(ParsingReason.PointsToAnalysis)}); + this.hostVM = hostVM; + this.inliningUtils = inliningUtils; + } + + @Override + protected boolean tryInvocationPlugins() { + return true; + } + + @Override + protected boolean needsExplicitExceptions() { + return false; + } + + @Override + protected FixedWithNextNode processInvokeArgs(ResolvedJavaMethod targetMethod, FixedWithNextNode insertionPoint, ValueNode[] arguments) { + StructuredGraph graph = insertionPoint.graph(); + InlinedInvokeArgumentsNode newNode = graph.add(new InlinedInvokeArgumentsNode(targetMethod, arguments)); + graph.addAfterFixed(insertionPoint, newNode); + return newNode; + } + + @Override + protected boolean shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { + if (inliningUtils.alwaysInlineInvoke((AnalysisMetaAccess) b.getMetaAccess(), method)) { + return true; + } + // worse case depth is max trivial, and then max accumulative + if (b.getDepth() > trivialAllowingInliningDepth + accumulativeAllowedInliningDepth) { + return false; + } + if (b.recursiveInliningDepth(method) > 0) { + /* Prevent recursive inlining. */ + return false; + } + + if (!InlineBeforeAnalysisPolicyUtils.inliningAllowed(hostVM, b, method)) { + return false; + } + + AllowInliningPredicate.InlineDecision result = allowInliningPredicate.allowInlining(b, method); + + return result == AllowInliningPredicate.InlineDecision.INLINE; + } + + @Override + protected InlineInvokePlugin.InlineInfo createInvokeInfo(ResolvedJavaMethod method) { + /* + * Set this graph initially to a stub. If there are no explicit calls to this method + * (i.e., all calls to this method are inlined), then the method's full flow will not + * need to be created. + */ + AnalysisMethod runtimeMethod = ((AnalysisMethod) method).getOrCreateMultiMethod(RUNTIME_COMPILED_METHOD, (newMethod) -> ((PointsToAnalysisMethod) newMethod).getTypeFlow().setAsStubFlow()); + return InlineInvokePlugin.InlineInfo.createStandardInlineInfo(runtimeMethod); + } + + @Override + protected AbstractPolicyScope createRootScope() { + /* We do not need a scope for the root method. */ + return null; + } + + @Override + protected AbstractPolicyScope openCalleeScope(ResolvedJavaMethod method, AbstractPolicyScope outer) { + if (outer instanceof AccumulativeInlineScope accOuter) { + // once the accumulative policy is activated, then we cannot return to the trivial policy + return InlineBeforeAnalysisPolicyUtils.createAccumulativeInlineScope(accOuter, inliningUtils); + } + + assert outer == null || outer instanceof AlwaysInlineScope : "unexpected outer scope: " + outer; + + // check if trivial is possible + boolean trivialInlineAllowed = hostVM.isAnalysisTrivialMethod((AnalysisMethod) method); + int inliningDepth = outer == null ? 1 : outer.inliningDepth + 1; + if (trivialInlineAllowed && inliningDepth <= trivialAllowingInliningDepth) { + return new AlwaysInlineScope(inliningDepth); + } else { + // start with a new accumulative inline scope + return InlineBeforeAnalysisPolicyUtils.createAccumulativeInlineScope(null, inliningUtils); + } + } } private class RuntimeCompilationAnalysisPolicy implements HostVM.MultiMethodAnalysisPolicy { @Override - public Collection determineCallees(BigBang bb, T implementation, T target, MultiMethod.MultiMethodKey callerMultiMethodKey, InvokeTypeFlow parsingReason) { + public Collection determineCallees(BigBang bb, T implementation, T target, MultiMethod.MultiMethodKey callerMultiMethodKey, InvokeTypeFlow invokeFlow) { + if (invokeFlow.isDeoptInvokeTypeFlow()) { + /* + * When the type flow represents a deopt invoke, then the arguments only need to be + * linked to the deopt target. As there is not a call here (the call is inlined), no + * other linking is necessary. + */ + assert SubstrateCompilationDirectives.isRuntimeCompiledMethod(implementation); + var originalTarget = implementation.getMultiMethod(ORIGINAL_METHOD); + assert originalTarget != null; + runtimeCompilationCandidates.add(new RuntimeCompilationCandidateImpl(originalTarget, originalTarget)); + return List.of(getDeoptVersion(implementation)); + } assert implementation.isOriginalMethod() && target.isOriginalMethod(); // recording compilation candidate @@ -877,7 +1077,7 @@ public Collection determineCallees(BigBang bb, T i * variant). */ if (registeredRuntimeCompilations.contains(implementation)) { - return List.of(implementation, getDeoptVersion(implementation), getRuntimeVersion(bb, implementation, true, parsingReason)); + return List.of(implementation, getDeoptVersion(implementation), getRuntimeVersion(bb, implementation, true, invokeFlow)); } else { return List.of(implementation); } @@ -887,7 +1087,7 @@ public Collection determineCallees(BigBang bb, T i * evaluated), runtime (if it is partial evaluated), and deoptimized (if the runtime * deoptimizes). */ - return List.of(implementation, getDeoptVersion(implementation), getRuntimeVersion(bb, implementation, true, parsingReason)); + return List.of(implementation, getDeoptVersion(implementation), getRuntimeVersion(bb, implementation, true, invokeFlow)); } else { assert callerMultiMethodKey == DEOPT_TARGET_METHOD; /* @@ -898,7 +1098,7 @@ public Collection determineCallees(BigBang bb, T i * only register the runtime variant as a stub though because its flow only needs to * be made upon it being reachable from a runtime compiled method's invoke. */ - return List.of(implementation, getDeoptVersion(implementation), getRuntimeVersion(bb, implementation, false, parsingReason)); + return List.of(implementation, getDeoptVersion(implementation), getRuntimeVersion(bb, implementation, false, invokeFlow)); } } diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/RuntimeCompilationFeature.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/RuntimeCompilationFeature.java index 3e5389959bdd..d5775f5a69fe 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/RuntimeCompilationFeature.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/RuntimeCompilationFeature.java @@ -51,6 +51,7 @@ import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.BytecodeExceptionMode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.tiers.Suites; @@ -183,6 +184,16 @@ public interface RuntimeCompilationCandidatePredicate { boolean allowRuntimeCompilation(ResolvedJavaMethod method); } + public interface AllowInliningPredicate { + enum InlineDecision { + INLINE, + INLINING_DISALLOWED, + NO_DECISION + } + + InlineDecision allowInlining(GraphBuilderContext builder, ResolvedJavaMethod target); + } + public abstract static class AbstractCallTreeNode implements Comparable { private final AnalysisMethod implementationMethod; private final AnalysisMethod targetMethod; @@ -516,6 +527,8 @@ public SubstrateMethod prepareMethodForRuntimeCompilation(Executable method, Bef public abstract void initializeAnalysisProviders(BigBang bb, Function generator); + public abstract void registerAllowInliningPredicate(AllowInliningPredicate predicate); + public abstract SubstrateMethod prepareMethodForRuntimeCompilation(ResolvedJavaMethod method, BeforeAnalysisAccessImpl config); protected final void afterAnalysisHelper() { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java index 6dde347cecd4..97a6c639cabc 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java @@ -45,6 +45,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiPredicate; +import java.util.function.Function; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; @@ -127,6 +128,7 @@ import com.oracle.svm.hosted.phases.AnalysisGraphBuilderPhase; import com.oracle.svm.hosted.phases.ImplicitAssertionsPhase; import com.oracle.svm.hosted.phases.InlineBeforeAnalysisPolicyImpl; +import com.oracle.svm.hosted.phases.InlineBeforeAnalysisPolicyUtils; import com.oracle.svm.hosted.substitute.UnsafeAutomaticSubstitutionProcessor; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ReflectionUtil; @@ -166,7 +168,9 @@ public class SVMHost extends HostVM { private final Set finalFieldsInitializedOutsideOfConstructor = ConcurrentHashMap.newKeySet(); private final MultiMethodAnalysisPolicy multiMethodAnalysisPolicy; private final SVMParsingSupport parsingSupport; + private final InlineBeforeAnalysisPolicy inlineBeforeAnalysisPolicy; + @SuppressWarnings("this-escape") public SVMHost(OptionValues options, ClassLoader classLoader, ClassInitializationSupport classInitializationSupport, UnsafeAutomaticSubstitutionProcessor automaticSubstitutions, Platform platform) { super(options, classLoader); @@ -183,7 +187,18 @@ public SVMHost(OptionValues options, ClassLoader classLoader, ClassInitializatio ImageSingletons.add(HostVM.MultiMethodAnalysisPolicy.class, DEFAULT_MULTIMETHOD_ANALYSIS_POLICY); multiMethodAnalysisPolicy = DEFAULT_MULTIMETHOD_ANALYSIS_POLICY; } - parsingSupport = ImageSingletons.contains(SVMParsingSupport.class) ? ImageSingletons.lookup(SVMParsingSupport.class) : null; + InlineBeforeAnalysisPolicyUtils inliningUtils = getInlineBeforeAnalysisPolicyUtils(); + inlineBeforeAnalysisPolicy = new InlineBeforeAnalysisPolicyImpl(this, inliningUtils); + if (ImageSingletons.contains(SVMParsingSupport.class)) { + parsingSupport = ImageSingletons.lookup(SVMParsingSupport.class); + parsingSupport.initializeInlineBeforeAnalysisPolicy(this, inliningUtils); + } else { + parsingSupport = null; + } + } + + protected InlineBeforeAnalysisPolicyUtils getInlineBeforeAnalysisPolicyUtils() { + return new InlineBeforeAnalysisPolicyUtils(); } private static Map> setupForbiddenTypes(OptionValues options) { @@ -551,7 +566,14 @@ public void methodAfterParsingHook(BigBang bb, AnalysisMethod method, Structured } if (parseOnce) { - new ImplicitAssertionsPhase().apply(graph, getProviders(method.getMultiMethodKey())); + if (!SubstrateCompilationDirectives.isRuntimeCompiledMethod(method)) { + /* + * Runtime compiled methods should not have assertions. If they do, then they + * should be caught via the blocklist instead of being converted to bytecode + * exceptions. + */ + new ImplicitAssertionsPhase().apply(graph, getProviders(method.getMultiMethodKey())); + } UninterruptibleAnnotationChecker.checkAfterParsing(method, graph); optimizeAfterParsing(bb, method, graph); @@ -560,6 +582,14 @@ public void methodAfterParsingHook(BigBang bb, AnalysisMethod method, Structured * leftover uncanonicalized nodes. */ CanonicalizerPhase.create().apply(graph, getProviders(method.getMultiMethodKey())); + /* + * To avoid keeping the whole Graal graphs alive in production use cases, we extract + * the necessary bits of information and store them in secondary storage maps. + */ + if (InliningUtilities.isTrivialMethod(graph)) { + analysisTrivialMethods.put(method, true); + } + } super.methodAfterParsingHook(bb, method, graph); @@ -620,10 +650,12 @@ public void methodBeforeTypeFlowCreationHook(BigBang bb, AnalysisMethod method, } /* * To avoid keeping the whole Graal graphs alive in production use cases, we extract the - * necessary bits of information here and store them in secondary storage maps. + * necessary bits of information and store them in secondary storage maps. */ - if (InliningUtilities.isTrivialMethod(graph)) { - analysisTrivialMethods.put(method, true); + if (!parseOnce) { + if (InliningUtilities.isTrivialMethod(graph)) { + analysisTrivialMethods.put(method, true); + } } for (Node n : graph.getNodes()) { if (n instanceof StackValueNode) { @@ -724,23 +756,16 @@ public boolean hasNeverInlineDirective(ResolvedJavaMethod method) { return SubstrateOptions.NeverInline.getValue().values().stream().anyMatch(re -> MethodFilter.parse(re).matches(method)); } - @SuppressWarnings("this-escape")// - private final InlineBeforeAnalysisPolicy inlineBeforeAnalysisPolicy = new InlineBeforeAnalysisPolicyImpl(this); - - protected InlineBeforeAnalysisPolicy inlineBeforeAnalysisPolicy(MultiMethod.MultiMethodKey multiMethodKey) { - /* - * Currently we only allow inlining before analysis in the original methods. - */ - if (multiMethodKey == MultiMethod.ORIGINAL_METHOD) { - return inlineBeforeAnalysisPolicy; - } else { - return InlineBeforeAnalysisPolicy.NO_INLINING; + private InlineBeforeAnalysisPolicy inlineBeforeAnalysisPolicy(MultiMethod.MultiMethodKey multiMethodKey) { + if (parsingSupport != null) { + return parsingSupport.inlineBeforeAnalysisPolicy(multiMethodKey, inlineBeforeAnalysisPolicy); } + return inlineBeforeAnalysisPolicy; } @Override - public InlineBeforeAnalysisGraphDecoder createInlineBeforeAnalysisGraphDecoder(BigBang bb, AnalysisMethod method, StructuredGraph resultGraph) { - return new InlineBeforeAnalysisGraphDecoder<>(bb, inlineBeforeAnalysisPolicy(method.getMultiMethodKey()), resultGraph, bb.getProviders(method), null); + public InlineBeforeAnalysisGraphDecoder createInlineBeforeAnalysisGraphDecoder(BigBang bb, AnalysisMethod method, StructuredGraph resultGraph) { + return new InlineBeforeAnalysisGraphDecoder(bb, inlineBeforeAnalysisPolicy(method.getMultiMethodKey()), resultGraph, bb.getProviders(method), null); } public static class Options { @@ -920,4 +945,15 @@ public boolean ignoreInstanceOfTypeDisallowed() { } return super.ignoreInstanceOfTypeDisallowed(); } + + @Override + public Function getStrengthenGraphsToTargetFunction(MultiMethod.MultiMethodKey key) { + if (parsingSupport != null) { + var result = parsingSupport.getStrengthenGraphsToTargetFunction(key); + if (result != null) { + return result; + } + } + return super.getStrengthenGraphsToTargetFunction(key); + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SubstrateStrengthenGraphs.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SubstrateStrengthenGraphs.java index 9d3d97442846..fa56b353e89e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SubstrateStrengthenGraphs.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SubstrateStrengthenGraphs.java @@ -26,12 +26,15 @@ import java.util.function.Supplier; +import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.DeoptimizeNode; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.spi.CoreProviders; +import org.graalvm.compiler.nodes.spi.SimplifierTool; import com.oracle.graal.pointsto.PointsToAnalysis; import com.oracle.graal.pointsto.infrastructure.Universe; @@ -39,12 +42,17 @@ import com.oracle.graal.pointsto.results.StrengthenGraphs; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.graal.nodes.InlinedInvokeArgumentsNode; import com.oracle.svm.core.graal.nodes.LoweredDeadEndNode; import com.oracle.svm.core.nodes.SubstrateMethodCallTargetNode; import com.oracle.svm.core.snippets.SnippetRuntime; import com.oracle.svm.core.util.HostedStringDeduplication; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.code.SubstrateCompilationDirectives; import com.oracle.svm.hosted.meta.HostedType; +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; import jdk.vm.ci.meta.JavaMethodProfile; import jdk.vm.ci.meta.JavaTypeProfile; @@ -66,6 +74,17 @@ protected AnalysisType getStrengthenStampType(AnalysisType originalType) { return strengthenStampType == null ? null : strengthenStampType.getWrapped(); } + @Override + protected FixedNode createInvokeWithNullReceiverReplacement(StructuredGraph graph) { + /* + * Since this only should happen in a runtime compiled method, we can directly insert a + * deoptimize node. + */ + VMError.guarantee(SubstrateCompilationDirectives.isRuntimeCompiledMethod(graph.method()), "Creating null check deoptimize in non-runtime compiled method: %s", graph.method()); + DeoptimizeNode deopt = graph.add(new DeoptimizeNode(DeoptimizationAction.None, DeoptimizationReason.NullCheckException)); + return deopt; + } + @Override protected FixedNode createUnreachable(StructuredGraph graph, CoreProviders providers, Supplier message) { FixedNode unreachableNode = graph.add(new LoweredDeadEndNode()); @@ -76,8 +95,14 @@ protected FixedNode createUnreachable(StructuredGraph graph, CoreProviders provi * non-VM developers, we only do it when assertions are enabled for the image builder. And * Uninterruptible methods might not be able to access the heap yet for the error message * constant, so we skip it for such methods too. + * + * We also do not print out this message for runtime compiled methods because it would + * require us to preserve additional graph state. */ - if (SubstrateUtil.assertionsEnabled() && !Uninterruptible.Utils.isUninterruptible(graph.method())) { + boolean insertMessage = SubstrateUtil.assertionsEnabled() && + !Uninterruptible.Utils.isUninterruptible(graph.method()) && + !SubstrateCompilationDirectives.isRuntimeCompiledMethod(graph.method()); + if (insertMessage) { ConstantNode messageNode = ConstantNode.forConstant(providers.getConstantReflection().forString(message.get()), providers.getMetaAccess(), graph); ForeignCallNode foreignCallNode = graph.add(new ForeignCallNode(SnippetRuntime.UNSUPPORTED_FEATURE, messageNode)); foreignCallNode.setNext(unreachableNode); @@ -100,4 +125,17 @@ protected void setInvokeProfiles(Invoke invoke, JavaTypeProfile typeProfile, Jav protected String getTypeName(AnalysisType type) { return HostedStringDeduplication.singleton().deduplicate(type.toJavaName(true), false); } + + @Override + protected boolean simplifyDelegate(Node n, SimplifierTool tool) { + /* + * This node is only necessary for analysis and can be removed once StrengthenGraphs is + * reached. + */ + if (n instanceof InlinedInvokeArgumentsNode nodeToRemove) { + nodeToRemove.graph().removeFixed(nodeToRemove); + return true; + } + return false; + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/SVMParsingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/SVMParsingSupport.java index cc58ecdf9537..c1a596bde201 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/SVMParsingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/SVMParsingSupport.java @@ -24,14 +24,22 @@ */ package com.oracle.svm.hosted.analysis; +import java.util.function.Function; + import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.nodes.StructuredGraph; import com.oracle.graal.pointsto.BigBang; import com.oracle.graal.pointsto.PointsToAnalysis; import com.oracle.graal.pointsto.meta.AnalysisMethod; +import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.meta.HostedProviders; +import com.oracle.graal.pointsto.phases.InlineBeforeAnalysisPolicy; import com.oracle.svm.common.meta.MultiMethod; +import com.oracle.svm.hosted.SVMHost; +import com.oracle.svm.hosted.phases.InlineBeforeAnalysisPolicyUtils; + +import jdk.vm.ci.meta.ResolvedJavaType; /** * {@link com.oracle.graal.pointsto.api.HostVM} methods which may be overwritten by substratevm @@ -46,4 +54,10 @@ public interface SVMParsingSupport { boolean allowAssumptions(AnalysisMethod method); HostedProviders getHostedProviders(MultiMethod.MultiMethodKey key); + + void initializeInlineBeforeAnalysisPolicy(SVMHost svmHost, InlineBeforeAnalysisPolicyUtils inliningUtils); + + InlineBeforeAnalysisPolicy inlineBeforeAnalysisPolicy(MultiMethod.MultiMethodKey multiMethodKey, InlineBeforeAnalysisPolicy defaultPolicy); + + Function getStrengthenGraphsToTargetFunction(MultiMethod.MultiMethodKey key); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/flow/SVMMethodTypeFlowBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/flow/SVMMethodTypeFlowBuilder.java index d72156716b39..6c10981646c2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/flow/SVMMethodTypeFlowBuilder.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/flow/SVMMethodTypeFlowBuilder.java @@ -29,6 +29,7 @@ import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.NodeView; @@ -46,6 +47,7 @@ import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.meta.PointsToAnalysisMethod; +import com.oracle.svm.core.graal.nodes.InlinedInvokeArgumentsNode; import com.oracle.svm.core.graal.thread.CompareAndSetVMThreadLocalNode; import com.oracle.svm.core.graal.thread.StoreVMThreadLocalNode; import com.oracle.svm.core.util.UserError.UserException; @@ -176,14 +178,15 @@ protected boolean delegateNodeProcessing(FixedNode n, TypeFlowsOfNodes state) { * the node has an exact type. This works with allocation site sensitivity because the * StoreVMThreadLocal is modeled by writing the objects to the all-instantiated. */ - if (n instanceof StoreVMThreadLocalNode) { - StoreVMThreadLocalNode node = (StoreVMThreadLocalNode) n; + if (n instanceof StoreVMThreadLocalNode node) { storeVMThreadLocal(state, node, node.getValue()); return true; - } else if (n instanceof CompareAndSetVMThreadLocalNode) { - CompareAndSetVMThreadLocalNode node = (CompareAndSetVMThreadLocalNode) n; + } else if (n instanceof CompareAndSetVMThreadLocalNode node) { storeVMThreadLocal(state, node, node.getUpdate()); return true; + } else if (n instanceof InlinedInvokeArgumentsNode node) { + processInlinedInvokeArgumentsNode(state, node); + return true; } return super.delegateNodeProcessing(n, state); } @@ -205,4 +208,15 @@ private void storeVMThreadLocal(TypeFlowsOfNodes state, ValueNode storeNode, Val typeFlowGraphBuilder.registerSinkBuilder(storeBuilder); } } + + private void processInlinedInvokeArgumentsNode(TypeFlowsOfNodes state, InlinedInvokeArgumentsNode node) { + /* + * Create a proxy invoke type flow for the inlined method. + */ + PointsToAnalysisMethod targetMethod = (PointsToAnalysisMethod) node.getInvokeTarget(); + // GR-45916 add proper source position information. + BytecodePosition position = AbstractAnalysisEngine.syntheticSourcePosition(node, method); + InvokeKind invokeKind = targetMethod.isStatic() ? InvokeKind.Static : InvokeKind.Special; + processMethodInvocation(state, node, invokeKind, targetMethod, node.getArguments(), true, position, true); + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/DeoptimizationUtils.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/DeoptimizationUtils.java index 82754da739fa..8915348c51b2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/DeoptimizationUtils.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/DeoptimizationUtils.java @@ -399,9 +399,7 @@ public static Collection registerDeoptEntries(StructuredGrap * graph.getInvokes() only iterates invokes that have a MethodCallTarget, so by using it * we would miss invocations that are already intrinsified to an indirect call. */ - if (n instanceof Invoke) { - Invoke invoke = (Invoke) n; - + if (n instanceof Invoke invoke) { /* * The FrameState for the invoke (which is visited by the above loop) is the state * after the call (where deoptimization that happens after the call has returned diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/SubstrateCompilationDirectives.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/SubstrateCompilationDirectives.java index 2893233a1433..306caf86ae00 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/SubstrateCompilationDirectives.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/SubstrateCompilationDirectives.java @@ -47,6 +47,13 @@ @AutomaticallyRegisteredImageSingleton public class SubstrateCompilationDirectives { + public static boolean isRuntimeCompiledMethod(ResolvedJavaMethod method) { + if (method instanceof MultiMethod multiMethod) { + return multiMethod.getMultiMethodKey() == RUNTIME_COMPILED_METHOD; + } + return false; + } + public static final MultiMethod.MultiMethodKey RUNTIME_COMPILED_METHOD = new MultiMethod.MultiMethodKey() { @Override public String toString() { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java index 04e473bed936..67bfba4ea2e7 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java @@ -53,12 +53,12 @@ import com.oracle.svm.core.deopt.DeoptTest; import com.oracle.svm.core.graal.nodes.DeoptEntryBeginNode; import com.oracle.svm.core.graal.nodes.DeoptEntryNode; +import com.oracle.svm.core.graal.nodes.DeoptProxyAnchorNode; import com.oracle.svm.core.graal.nodes.LoweredDeadEndNode; import com.oracle.svm.core.graal.nodes.NewPodInstanceNode; import com.oracle.svm.core.graal.nodes.TestDeoptimizeNode; import com.oracle.svm.core.heap.Pod; import com.oracle.svm.core.heap.Pod.RuntimeSupport.PodFactory; -import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.annotation.CustomSubstitutionMethod; import com.oracle.svm.hosted.code.SubstrateCompilationDirectives; import com.oracle.svm.hosted.nodes.DeoptProxyNode; @@ -240,13 +240,17 @@ private static int invokeWithDeoptAndExceptionUnwind(HostedGraphKit kit, DeoptIn kit.append(new UnwindNode(exception)); } - if (shouldInsertDeoptEntry(deoptInfo, invoke.stateAfter().bci, false, false)) { - // Deopt entry after invoke without exception - + boolean needDeoptEntry = shouldInsertDeoptEntry(deoptInfo, invoke.stateAfter().bci, false, false); + boolean needDeoptProxy = shouldInsertDeoptEntry(deoptInfo, bci, true, false); + if (needDeoptEntry || needDeoptProxy) { kit.noExceptionPart(); - nextDeoptIndex = appendDeoptWithExceptionUnwind(kit, invoke.stateAfter(), invoke.stateAfter().bci, nextDeoptIndex); - } else { - VMError.guarantee(!shouldInsertDeoptEntry(deoptInfo, bci, true, false), "need to add support for inserting DeoptProxyAnchorNode"); + if (needDeoptEntry) { + // Deopt entry after invoke without exception + nextDeoptIndex = appendDeoptWithExceptionUnwind(kit, invoke.stateAfter(), invoke.stateAfter().bci, nextDeoptIndex); + } else { + // Only a proxy is needed + nextDeoptIndex = appendDeoptProxyAnchorNode(kit, invoke.stateAfter(), nextDeoptIndex); + } } kit.endInvokeWithException(); @@ -275,6 +279,16 @@ private static int appendDeoptWithExceptionUnwind(HostedGraphKit kit, FrameState return nextDeoptIndex + 1; } + /** @see com.oracle.svm.hosted.phases.SharedGraphBuilderPhase */ + private static int appendDeoptProxyAnchorNode(HostedGraphKit kit, FrameState state, int nextDeoptIndex) { + var anchor = kit.append(new DeoptProxyAnchorNode()); + anchor.setStateAfter(state.duplicate()); + + // Ensure later nodes see values from potential deoptimization + kit.getFrameState().insertProxies(value -> createDeoptProxy(kit, nextDeoptIndex, anchor, value)); + return nextDeoptIndex + 1; + } + private static ValueNode createDeoptProxy(HostedGraphKit kit, int nextDeoptIndex, FixedNode deoptTarget, ValueNode value) { return kit.getGraph().addOrUniqueWithInputs(DeoptProxyNode.create(value, deoptTarget, nextDeoptIndex)); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/nodes/DeoptProxyNode.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/nodes/DeoptProxyNode.java index 70984d0fa1c9..58076e389f40 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/nodes/DeoptProxyNode.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/nodes/DeoptProxyNode.java @@ -54,7 +54,7 @@ * variable (and stack) accesses across deoptimization points. *

* This is needed to ensure that the values, which are set by the {@link Deoptimizer} at the - * deoptimization point, are really read from their locations (and not hold in a temporary register, + * deoptimization point, are really read from their locations (and not held in a temporary register, * etc.) */ @NodeInfo(cycles = NodeCycles.CYCLES_0, size = NodeSize.SIZE_0) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/EnumSwitchPlugin.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/EnumSwitchPlugin.java index 54177774c442..3a2ea7494142 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/EnumSwitchPlugin.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/EnumSwitchPlugin.java @@ -148,7 +148,7 @@ private void onMethodParsed(AnalysisMethod method, StructuredGraph graph) { boolean methodSafeForExecution = graph.getNodes().filter(node -> node instanceof EnsureClassInitializedNode).isEmpty(); Boolean existingValue = methodsSafeForExecution.put(method, methodSafeForExecution); - assert existingValue == null || !method.isOriginalMethod() : "Method parsed twice: " + method.format("%H.%n(%p)"); + assert existingValue == null || method.isDeoptTarget() : "Method parsed twice: " + method.format("%H.%n(%p)"); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java index 9298ee15b593..932c1efdac44 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.hosted.phases; -import com.oracle.graal.pointsto.infrastructure.GraphProvider; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.StampPair; import org.graalvm.compiler.debug.DebugContext; @@ -47,6 +46,7 @@ import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.type.StampTool; +import com.oracle.graal.pointsto.infrastructure.GraphProvider; import com.oracle.graal.pointsto.meta.HostedProviders; import com.oracle.graal.pointsto.results.StaticAnalysisResults; import com.oracle.svm.core.c.BoxedRelocatedPointer; @@ -65,10 +65,7 @@ public class HostedGraphKit extends SubstrateGraphKit { public HostedGraphKit(DebugContext debug, HostedProviders providers, ResolvedJavaMethod method, GraphProvider.Purpose purpose) { - // Needed to match type flows to invokes so invoked methods can be inlined in runtime - // compilations, see GraalFeature.processMethod() and MethodTypeFlowBuilder.uniqueKey() super(debug, method, providers, providers.getWordTypes(), providers.getGraphBuilderPlugins(), new SubstrateCompilationIdentifier(), purpose == GraphProvider.Purpose.ANALYSIS); - graph.getGraphState().configureExplicitExceptionsNoDeopt(); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java index c7bceacbaf5f..5c07008a77e2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java @@ -24,44 +24,15 @@ */ package com.oracle.svm.hosted.phases; -import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.compiler.graph.Node; -import org.graalvm.compiler.graph.Node.NodeIntrinsic; -import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.AbstractEndNode; -import org.graalvm.compiler.nodes.CallTargetNode; -import org.graalvm.compiler.nodes.ConstantNode; -import org.graalvm.compiler.nodes.FrameState; -import org.graalvm.compiler.nodes.FullInfopointNode; -import org.graalvm.compiler.nodes.Invoke; -import org.graalvm.compiler.nodes.LogicConstantNode; -import org.graalvm.compiler.nodes.ParameterNode; -import org.graalvm.compiler.nodes.ReturnNode; -import org.graalvm.compiler.nodes.StartNode; -import org.graalvm.compiler.nodes.UnwindNode; +import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.extended.ValueAnchorNode; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo; import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; -import org.graalvm.compiler.nodes.java.AbstractNewObjectNode; -import org.graalvm.compiler.nodes.java.NewArrayNode; -import org.graalvm.compiler.nodes.spi.ValueProxy; -import org.graalvm.compiler.nodes.virtual.AllocatedObjectNode; -import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; -import org.graalvm.compiler.nodes.virtual.VirtualArrayNode; -import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; -import org.graalvm.compiler.options.Option; -import org.graalvm.nativeimage.AnnotationAccess; import com.oracle.graal.pointsto.meta.AnalysisMetaAccess; -import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.phases.InlineBeforeAnalysisPolicy; import com.oracle.svm.core.ParsingReason; -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.heap.RestrictHeapAccess; -import com.oracle.svm.core.option.HostedOptionKey; -import com.oracle.svm.core.util.VMError; -import com.oracle.svm.hosted.ReachabilityRegistrationNode; import com.oracle.svm.hosted.SVMHost; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -81,56 +52,24 @@ * amount of inlining in a future version without breaking compatibility. This also means that we * must be conservative and only inline what is necessary for known use cases. */ -public class InlineBeforeAnalysisPolicyImpl extends InlineBeforeAnalysisPolicy { +public class InlineBeforeAnalysisPolicyImpl extends InlineBeforeAnalysisPolicy { + private final int maxInliningDepth = InlineBeforeAnalysisPolicyUtils.Options.InlineBeforeAnalysisAllowedDepth.getValue(); - public static class Options { - @Option(help = "Maximum number of computation nodes for method inlined before static analysis")// - public static final HostedOptionKey InlineBeforeAnalysisAllowedNodes = new HostedOptionKey<>(1); + private final SVMHost hostVM; + private final InlineBeforeAnalysisPolicyUtils inliningUtils; - @Option(help = "Maximum number of invokes for method inlined before static analysis")// - public static final HostedOptionKey InlineBeforeAnalysisAllowedInvokes = new HostedOptionKey<>(1); - - @Option(help = "Maximum number of invokes for method inlined before static analysis")// - public static final HostedOptionKey InlineBeforeAnalysisAllowedDepth = new HostedOptionKey<>(20); - } - - protected final SVMHost hostVM; - - private final int allowedNodes = Options.InlineBeforeAnalysisAllowedNodes.getValue(); - private final int allowedInvokes = Options.InlineBeforeAnalysisAllowedInvokes.getValue(); - private final int allowedDepth = Options.InlineBeforeAnalysisAllowedDepth.getValue(); - - protected static final class CountersScope implements InlineBeforeAnalysisPolicy.Scope { - final CountersScope accumulated; - - int numNodes; - int numInvokes; - - CountersScope(CountersScope accumulated) { - this.accumulated = accumulated; - } - - @Override - public String toString() { - return numNodes + "/" + numInvokes + " (" + accumulated.numNodes + "/" + accumulated.numInvokes + ")"; - } - } - - public InlineBeforeAnalysisPolicyImpl(SVMHost hostVM) { + public InlineBeforeAnalysisPolicyImpl(SVMHost hostVM, InlineBeforeAnalysisPolicyUtils inliningUtils) { super(new NodePlugin[]{new ConstantFoldLoadFieldPlugin(ParsingReason.PointsToAnalysis)}); this.hostVM = hostVM; - } - - protected boolean alwaysInlineInvoke(@SuppressWarnings("unused") AnalysisMetaAccess metaAccess, @SuppressWarnings("unused") ResolvedJavaMethod method) { - return false; + this.inliningUtils = inliningUtils; } @Override protected boolean shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { - if (alwaysInlineInvoke((AnalysisMetaAccess) b.getMetaAccess(), method)) { + if (inliningUtils.alwaysInlineInvoke((AnalysisMetaAccess) b.getMetaAccess(), method)) { return true; } - if (b.getDepth() > allowedDepth) { + if (b.getDepth() > maxInliningDepth) { return false; } if (b.recursiveInliningDepth(method) > 0) { @@ -138,7 +77,17 @@ protected boolean shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod m return false; } - return inliningAllowed(hostVM, b, method); + return InlineBeforeAnalysisPolicyUtils.inliningAllowed(hostVM, b, method); + } + + @Override + protected InlineInfo createInvokeInfo(ResolvedJavaMethod method) { + return InlineInfo.createStandardInlineInfo(method); + } + + @Override + protected boolean needsExplicitExceptions() { + return true; } @Override @@ -150,164 +99,20 @@ protected boolean tryInvocationPlugins() { return true; } - public static boolean inliningAllowed(SVMHost hostVM, GraphBuilderContext b, ResolvedJavaMethod method) { - AnalysisMethod caller = (AnalysisMethod) b.getMethod(); - AnalysisMethod callee = (AnalysisMethod) method; - if (hostVM.neverInlineTrivial(caller, callee)) { - return false; - } - if (AnnotationAccess.isAnnotationPresent(callee, Fold.class) || AnnotationAccess.isAnnotationPresent(callee, NodeIntrinsic.class)) { - /* - * We should never see a call to such a method. But if we do, do not inline them - * otherwise we miss the opportunity later to report it as an error. - */ - return false; - } - if (AnnotationAccess.isAnnotationPresent(callee, RestrictHeapAccess.class)) { - /* - * This is conservative. We do not know the caller's heap restriction state yet because - * that can only be computed after static analysis (it relies on the call graph produced - * by the static analysis). - */ - return false; - } - if (!Uninterruptible.Utils.inliningAllowed(caller, callee)) { - return false; - } - if (callee.getReturnsAllInstantiatedTypes()) { - /* - * When a callee returns all instantiated types then it cannot be inlined. Inlining the - * method would expose the method's return values instead of treating it as an - * AllInstantiatedTypeFlow. - */ - return false; - } - return true; - } - @Override - protected CountersScope createRootScope() { + protected AbstractPolicyScope createRootScope() { /* We do not need a scope for the root method. */ return null; } @Override - protected CountersScope openCalleeScope(CountersScope outer) { - CountersScope accumulated; - if (outer == null) { - /* - * The first level of method inlining, i.e., the top scope from the inlining policy - * point of view. - */ - accumulated = new CountersScope(null); - } else { - /* Nested inlining. */ - accumulated = outer.accumulated; - } - return new CountersScope(accumulated); - } - - @Override - protected void commitCalleeScope(CountersScope outer, CountersScope callee) { - assert outer.accumulated == callee.accumulated; - outer.numNodes += callee.numNodes; - outer.numInvokes += callee.numInvokes; - } - - @Override - protected void abortCalleeScope(CountersScope outer, CountersScope callee) { - assert outer.accumulated == callee.accumulated; - outer.accumulated.numNodes -= callee.numNodes; - outer.accumulated.numInvokes -= callee.numInvokes; + protected AbstractPolicyScope openCalleeScope(ResolvedJavaMethod method, AbstractPolicyScope outer) { + return InlineBeforeAnalysisPolicyUtils.createAccumulativeInlineScope((InlineBeforeAnalysisPolicyUtils.AccumulativeInlineScope) outer, inliningUtils); } @Override - protected boolean processNode(AnalysisMetaAccess metaAccess, ResolvedJavaMethod method, CountersScope scope, Node node) { - if (node instanceof StartNode || node instanceof ParameterNode || node instanceof ReturnNode || node instanceof UnwindNode) { - /* Infrastructure nodes that are not even visible to the policy. */ - throw VMError.shouldNotReachHere("Node must not be visible to policy: " + node.getClass().getTypeName()); - } - if (node instanceof FullInfopointNode || node instanceof ValueProxy || node instanceof ValueAnchorNode || node instanceof FrameState || - node instanceof AbstractBeginNode || node instanceof AbstractEndNode) { - /* - * Infrastructure nodes that are never counted. We could look at the NodeSize annotation - * of a node, but that is a bit unreliable. For example, FrameState and - * ExceptionObjectNode have size != 0 but we do not want to count them; CallTargetNode - * has size 0 but we need to count it. - */ - return true; - } - - if (node instanceof ConstantNode || node instanceof LogicConstantNode) { - /* An unlimited number of constants is allowed. We like constants. */ - return true; - } - - if (node instanceof ReachabilityRegistrationNode) { - /* - * These nodes do not affect compilation and are only used to execute handlers depending - * on their reachability. - */ - return true; - } - - if (alwaysInlineInvoke(metaAccess, method)) { - return true; - } - - if (node instanceof AbstractNewObjectNode) { - /* - * We never allow to inline any kind of allocations, because the machine code size is - * large. - * - * With one important exception: we allow (and do not even count) arrays allocated with - * length 0. Such allocations occur when a method has a Java vararg parameter but the - * caller does not provide any vararg. Without this exception, important vararg usages - * like Class.getDeclaredConstructor would not be considered for inlining. - * - * Note that we are during graph decoding, so usages of the node are not decoded yet. So - * we cannot base the decision on a certain usage pattern of the allocation. - */ - if (node instanceof NewArrayNode) { - ValueNode newArrayLength = ((NewArrayNode) node).length(); - if (newArrayLength.isJavaConstant() && newArrayLength.asJavaConstant().asInt() == 0) { - return true; - } - } - return false; - } else if (node instanceof VirtualObjectNode) { - /* - * Same as the explicit allocation nodes above, but this time for the virtualized - * allocations created when escape analysis runs immediately after bytecode parsing. - */ - if (node instanceof VirtualArrayNode) { - int newArrayLength = ((VirtualArrayNode) node).entryCount(); - if (newArrayLength == 0) { - return true; - } - } - return false; - } else if (node instanceof CommitAllocationNode || node instanceof AllocatedObjectNode) { - /* Ignore nodes created by escape analysis in addition to the VirtualInstanceNode. */ - return true; - } - - if (node instanceof CallTargetNode) { - throw VMError.shouldNotReachHere("Node must not be visible to policy: " + node.getClass().getTypeName()); - } else if (node instanceof Invoke) { - if (scope.accumulated.numInvokes >= allowedInvokes) { - return false; - } - scope.numInvokes++; - scope.accumulated.numInvokes++; - } - - if (scope.accumulated.numNodes >= allowedNodes) { - return false; - } - scope.numNodes++; - scope.accumulated.numNodes++; - - return true; + protected FixedWithNextNode processInvokeArgs(ResolvedJavaMethod targetMethod, FixedWithNextNode insertionPoint, ValueNode[] arguments) { + // No action is needed + return insertionPoint; } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java new file mode 100644 index 000000000000..915ff63f2a76 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.phases; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.AbstractEndNode; +import org.graalvm.compiler.nodes.CallTargetNode; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.FullInfopointNode; +import org.graalvm.compiler.nodes.Invoke; +import org.graalvm.compiler.nodes.LogicConstantNode; +import org.graalvm.compiler.nodes.ParameterNode; +import org.graalvm.compiler.nodes.ReturnNode; +import org.graalvm.compiler.nodes.StartNode; +import org.graalvm.compiler.nodes.UnwindNode; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.extended.ValueAnchorNode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.java.AbstractNewObjectNode; +import org.graalvm.compiler.nodes.java.NewArrayNode; +import org.graalvm.compiler.nodes.spi.ValueProxy; +import org.graalvm.compiler.nodes.virtual.AllocatedObjectNode; +import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; +import org.graalvm.compiler.nodes.virtual.VirtualArrayNode; +import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; +import org.graalvm.compiler.options.Option; +import org.graalvm.nativeimage.AnnotationAccess; + +import com.oracle.graal.pointsto.meta.AnalysisMetaAccess; +import com.oracle.graal.pointsto.meta.AnalysisMethod; +import com.oracle.graal.pointsto.phases.InlineBeforeAnalysisPolicy; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.ReachabilityRegistrationNode; +import com.oracle.svm.hosted.SVMHost; + +import jdk.vm.ci.meta.ResolvedJavaMethod; + +public class InlineBeforeAnalysisPolicyUtils { + public static class Options { + @Option(help = "Maximum number of computation nodes for method inlined before static analysis")// + public static final HostedOptionKey InlineBeforeAnalysisAllowedNodes = new HostedOptionKey<>(1); + + @Option(help = "Maximum number of invokes for method inlined before static analysis")// + public static final HostedOptionKey InlineBeforeAnalysisAllowedInvokes = new HostedOptionKey<>(1); + + @Option(help = "Maximum number of invokes for method inlined before static analysis")// + public static final HostedOptionKey InlineBeforeAnalysisAllowedDepth = new HostedOptionKey<>(20); + } + + public static boolean inliningAllowed(SVMHost hostVM, GraphBuilderContext b, ResolvedJavaMethod method) { + AnalysisMethod caller = (AnalysisMethod) b.getMethod(); + AnalysisMethod callee = (AnalysisMethod) method; + if (hostVM.neverInlineTrivial(caller, callee)) { + return false; + } + if (AnnotationAccess.isAnnotationPresent(callee, Fold.class) || AnnotationAccess.isAnnotationPresent(callee, Node.NodeIntrinsic.class)) { + /* + * We should never see a call to such a method. But if we do, do not inline them + * otherwise we miss the opportunity later to report it as an error. + */ + return false; + } + if (AnnotationAccess.isAnnotationPresent(callee, RestrictHeapAccess.class)) { + /* + * This is conservative. We do not know the caller's heap restriction state yet because + * that can only be computed after static analysis (it relies on the call graph produced + * by the static analysis). + */ + return false; + } + if (!Uninterruptible.Utils.inliningAllowed(caller, callee)) { + return false; + } + if (callee.getReturnsAllInstantiatedTypes()) { + /* + * When a callee returns all instantiated types then it cannot be inlined. Inlining the + * method would expose the method's return values instead of treating it as an + * AllInstantiatedTypeFlow. + */ + return false; + } + return true; + } + + public boolean alwaysInlineInvoke(@SuppressWarnings("unused") AnalysisMetaAccess metaAccess, @SuppressWarnings("unused") ResolvedJavaMethod method) { + return false; + } + + /** + * This scope will always allow nodes to be inlined. + */ + public static class AlwaysInlineScope extends InlineBeforeAnalysisPolicy.AbstractPolicyScope { + + public AlwaysInlineScope(int inliningDepth) { + super(inliningDepth); + } + + @Override + public boolean allowAbort() { + // should not be able to abort + return false; + } + + @Override + public void commitCalleeScope(InlineBeforeAnalysisPolicy.AbstractPolicyScope callee) { + // nothing to do + } + + @Override + public void abortCalleeScope(InlineBeforeAnalysisPolicy.AbstractPolicyScope callee) { + // nothing to do + } + + @Override + public boolean processNode(AnalysisMetaAccess metaAccess, ResolvedJavaMethod method, Node node) { + // always inlining + return true; + } + + @Override + public String toString() { + return "AlwaysInlineScope"; + } + } + + static class AccumulativeCounters { + final int maxNodes = Options.InlineBeforeAnalysisAllowedNodes.getValue(); + final int maxInvokes = Options.InlineBeforeAnalysisAllowedInvokes.getValue(); + final int maxInliningDepth = Options.InlineBeforeAnalysisAllowedDepth.getValue(); + + int numNodes = 0; + int numInvokes = 0; + } + + /** + * This scope tracks the total number of nodes inlined. Once the total number of inlined nodes + * has exceeded a specified count, or an illegal node is inlined, then the process will be + * aborted. + */ + public static AccumulativeInlineScope createAccumulativeInlineScope(AccumulativeInlineScope outer, InlineBeforeAnalysisPolicyUtils inliningUtils) { + AccumulativeCounters accumulativeCounters; + int depth; + if (outer == null) { + /* + * The first level of method inlining, i.e., the top scope from the inlining policy + * point of view. + */ + accumulativeCounters = new AccumulativeCounters(); + depth = 1; + } else { + /* Nested inlining. */ + accumulativeCounters = outer.accumulativeCounters; + depth = outer.inliningDepth + 1; + } + return new AccumulativeInlineScope(accumulativeCounters, depth, inliningUtils); + } + + public static class AccumulativeInlineScope extends InlineBeforeAnalysisPolicy.AbstractPolicyScope { + final AccumulativeCounters accumulativeCounters; + final InlineBeforeAnalysisPolicyUtils inliningUtils; + + /** + * The number of nodes and invokes which have been inlined from this method (and also + * committed child methods). This must be kept track of to ensure correct accounting when + * aborts occur. + */ + int numNodes = 0; + int numInvokes = 0; + + AccumulativeInlineScope(AccumulativeCounters accumulativeCounters, int inliningDepth, InlineBeforeAnalysisPolicyUtils inliningUtils) { + super(inliningDepth); + this.accumulativeCounters = accumulativeCounters; + this.inliningUtils = inliningUtils; + } + + @Override + public boolean allowAbort() { + // when too many nodes are inlined any abort can occur + return true; + } + + @Override + public void commitCalleeScope(InlineBeforeAnalysisPolicy.AbstractPolicyScope callee) { + AccumulativeInlineScope calleeScope = (AccumulativeInlineScope) callee; + assert accumulativeCounters == calleeScope.accumulativeCounters; + numNodes += calleeScope.numNodes; + numInvokes += calleeScope.numInvokes; + } + + @Override + public void abortCalleeScope(InlineBeforeAnalysisPolicy.AbstractPolicyScope callee) { + AccumulativeInlineScope calleeScope = (AccumulativeInlineScope) callee; + assert accumulativeCounters == calleeScope.accumulativeCounters; + accumulativeCounters.numNodes -= calleeScope.numNodes; + accumulativeCounters.numInvokes -= calleeScope.numInvokes; + } + + @Override + public boolean processNode(AnalysisMetaAccess metaAccess, ResolvedJavaMethod method, Node node) { + if (inliningUtils.alwaysInlineInvoke(metaAccess, method)) { + return true; + } + + if (inliningDepth > accumulativeCounters.maxInliningDepth) { + // too deep to continue inlining + return false; + } + + if (node instanceof StartNode || node instanceof ParameterNode || node instanceof ReturnNode || node instanceof UnwindNode) { + /* Infrastructure nodes that are not even visible to the policy. */ + throw VMError.shouldNotReachHere("Node must not be visible to policy: " + node.getClass().getTypeName()); + } + if (node instanceof FullInfopointNode || node instanceof ValueProxy || node instanceof ValueAnchorNode || node instanceof FrameState || + node instanceof AbstractBeginNode || node instanceof AbstractEndNode) { + /* + * Infrastructure nodes that are never counted. We could look at the NodeSize + * annotation of a node, but that is a bit unreliable. For example, FrameState and + * ExceptionObjectNode have size != 0 but we do not want to count them; + * CallTargetNode has size 0 but we need to count it. + */ + return true; + } + + if (node instanceof ConstantNode || node instanceof LogicConstantNode) { + /* An unlimited number of constants is allowed. We like constants. */ + return true; + } + + if (node instanceof ReachabilityRegistrationNode) { + /* + * These nodes do not affect compilation and are only used to execute handlers + * depending on their reachability. + */ + return true; + } + + if (node instanceof AbstractNewObjectNode) { + /* + * We never allow to inline any kind of allocations, because the machine code size + * is large. + * + * With one important exception: we allow (and do not even count) arrays allocated + * with length 0. Such allocations occur when a method has a Java vararg parameter + * but the caller does not provide any vararg. Without this exception, important + * vararg usages like Class.getDeclaredConstructor would not be considered for + * inlining. + * + * Note that we are during graph decoding, so usages of the node are not decoded + * yet. So we cannot base the decision on a certain usage pattern of the allocation. + */ + if (node instanceof NewArrayNode) { + ValueNode newArrayLength = ((NewArrayNode) node).length(); + if (newArrayLength.isJavaConstant() && newArrayLength.asJavaConstant().asInt() == 0) { + return true; + } + } + return false; + } else if (node instanceof VirtualObjectNode) { + /* + * Same as the explicit allocation nodes above, but this time for the virtualized + * allocations created when escape analysis runs immediately after bytecode parsing. + */ + if (node instanceof VirtualArrayNode) { + int newArrayLength = ((VirtualArrayNode) node).entryCount(); + if (newArrayLength == 0) { + return true; + } + } + return false; + } else if (node instanceof CommitAllocationNode || node instanceof AllocatedObjectNode) { + /* + * Ignore nodes created by escape analysis in addition to the VirtualInstanceNode. + */ + return true; + } + + if (node instanceof CallTargetNode) { + throw VMError.shouldNotReachHere("Node must not be visible to policy: " + node.getClass().getTypeName()); + } + + if (node instanceof Invoke) { + if (accumulativeCounters.numInvokes >= accumulativeCounters.maxInvokes) { + return false; + } + numInvokes++; + accumulativeCounters.numInvokes++; + } + + if (accumulativeCounters.numNodes >= accumulativeCounters.maxNodes) { + return false; + } + numNodes++; + accumulativeCounters.numNodes++; + + return true; + } + + @Override + public String toString() { + return "AccumulativeInlineScope: " + numNodes + "/" + numInvokes + " (" + accumulativeCounters.numNodes + "/" + accumulativeCounters.numInvokes + ")"; + } + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/IntrinsificationPluginRegistry.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/IntrinsificationPluginRegistry.java index 21816faa1317..58adc4e4cb5a 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/IntrinsificationPluginRegistry.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/IntrinsificationPluginRegistry.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; import com.oracle.graal.pointsto.meta.AnalysisMethod; +import com.oracle.svm.common.meta.MultiMethod; import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.meta.HostedMethod; @@ -89,8 +90,11 @@ public void add(ResolvedJavaMethod method, int bci, Object element) { /* * New elements can only be added when the intrinsification is executed during the analysis. * If an intrinsified element was already registered that's an error. + * + * The only exception to this is deoptimization target methods; as these methods can be + * reparsed during analysis, the same element may be registered multiple times. */ - VMError.guarantee(previous == null, "Detected previously intrinsified element"); + VMError.guarantee(previous == null || MultiMethod.isDeoptTarget(method), "Detected previously intrinsified element"); } @SuppressWarnings("unchecked") diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleFeature.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleFeature.java index e4305f798c7f..5d84eb17c242 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleFeature.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleFeature.java @@ -25,7 +25,11 @@ package com.oracle.svm.truffle; +import static com.oracle.svm.graal.hosted.RuntimeCompilationFeature.AllowInliningPredicate.InlineDecision.INLINE; +import static com.oracle.svm.graal.hosted.RuntimeCompilationFeature.AllowInliningPredicate.InlineDecision.INLINING_DISALLOWED; +import static com.oracle.svm.graal.hosted.RuntimeCompilationFeature.AllowInliningPredicate.InlineDecision.NO_DECISION; import static org.graalvm.compiler.java.BytecodeParserOptions.InlineDuringParsingMaxDepth; +import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createStandardInlineInfo; import java.lang.reflect.Executable; @@ -357,12 +361,13 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { GraphBuilderConfiguration graphBuilderConfig = partialEvaluator.getGraphBuilderConfigPrototype(); + TruffleAllowInliningPredicate allowInliningPredicate = new TruffleAllowInliningPredicate(runtimeCompilationFeature.getHostedProviders().getReplacements(), + graphBuilderConfig.getPlugins().getInvocationPlugins(), + partialEvaluator, this::allowRuntimeCompilation); if (Options.TruffleInlineDuringParsing.getValue() && !SubstrateOptions.parseOnce()) { - graphBuilderConfig.getPlugins().appendInlineInvokePlugin( - new TruffleParsingInlineInvokePlugin(config.getHostVM(), runtimeCompilationFeature.getHostedProviders().getReplacements(), - graphBuilderConfig.getPlugins().getInvocationPlugins(), - partialEvaluator, this::allowRuntimeCompilation)); + graphBuilderConfig.getPlugins().appendInlineInvokePlugin(new TruffleParsingInlineInvokePlugin(config.getHostVM(), allowInliningPredicate)); } + runtimeCompilationFeature.registerAllowInliningPredicate(allowInliningPredicate::allowInlining); registerNeverPartOfCompilation(graphBuilderConfig.getPlugins().getInvocationPlugins()); graphBuilderConfig.getPlugins().getInvocationPlugins().closeRegistration(); @@ -443,54 +448,81 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { }, BytecodeOSRNode.class); } - static class TruffleParsingInlineInvokePlugin implements InlineInvokePlugin { - - private final SVMHost hostVM; + static class TruffleAllowInliningPredicate { private final Replacements replacements; private final InvocationPlugins invocationPlugins; private final PartialEvaluator partialEvaluator; private final Predicate allowRuntimeCompilationPredicate; - TruffleParsingInlineInvokePlugin(SVMHost hostVM, Replacements replacements, InvocationPlugins invocationPlugins, PartialEvaluator partialEvaluator, + TruffleAllowInliningPredicate(Replacements replacements, InvocationPlugins invocationPlugins, PartialEvaluator partialEvaluator, Predicate allowRuntimeCompilationPredicate) { - this.hostVM = hostVM; this.replacements = replacements; this.invocationPlugins = invocationPlugins; this.partialEvaluator = partialEvaluator; this.allowRuntimeCompilationPredicate = allowRuntimeCompilationPredicate; } - @Override - public InlineInfo shouldInlineInvoke(GraphBuilderContext builder, ResolvedJavaMethod original, ValueNode[] arguments) { - assert !SubstrateOptions.parseOnce() : "inlining during parsing is not enabled with parse once"; + public RuntimeCompilationFeature.AllowInliningPredicate.InlineDecision allowInlining(GraphBuilderContext builder, ResolvedJavaMethod target) { - if (original.hasNeverInlineDirective()) { - return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; - } else if (invocationPlugins.lookupInvocation(original, builder.getOptions()) != null) { - return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; - } else if (original.getAnnotation(ExplodeLoop.class) != null) { + if (target.hasNeverInlineDirective()) { + return INLINING_DISALLOWED; + } else if (invocationPlugins.lookupInvocation(target, builder.getOptions()) != null) { + return INLINING_DISALLOWED; + } else if (target.getAnnotation(ExplodeLoop.class) != null) { /* * We cannot inline a method annotated with @ExplodeLoop, because then loops are no * longer exploded. */ - return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; + return INLINING_DISALLOWED; } else if (builder.getMethod().getAnnotation(ExplodeLoop.class) != null) { /* * We cannot inline anything into a method annotated with @ExplodeLoop, because then * loops of the inlined callee are exploded too. */ - return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; - } else if (replacements.hasSubstitution(original, builder.getOptions())) { - return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; + return INLINING_DISALLOWED; + } else if (replacements.hasSubstitution(target, builder.getOptions())) { + return INLINING_DISALLOWED; } for (ResolvedJavaMethod m : partialEvaluator.getNeverInlineMethods()) { - if (original.equals(m)) { - return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; + if (target.equals(m)) { + return INLINING_DISALLOWED; } } - if (original.getCode() != null && allowRuntimeCompilationPredicate.test(original) && hostVM.isAnalysisTrivialMethod((AnalysisMethod) original) && + if (target.getCode() != null && allowRuntimeCompilationPredicate.test(target)) { + return INLINE; + } + + return NO_DECISION; + } + } + + static class TruffleParsingInlineInvokePlugin implements InlineInvokePlugin { + + private final SVMHost hostVM; + TruffleAllowInliningPredicate inlineCriteria; + + TruffleParsingInlineInvokePlugin(SVMHost hostVM, TruffleAllowInliningPredicate inlineCriteria) { + this.hostVM = hostVM; + this.inlineCriteria = inlineCriteria; + } + + @Override + public InlineInfo shouldInlineInvoke(GraphBuilderContext builder, ResolvedJavaMethod original, ValueNode[] arguments) { + assert !SubstrateOptions.parseOnce() : "inlining during parsing is not enabled with parse once"; + + var result = inlineCriteria.allowInlining(builder, original); + switch (result) { + case NO_DECISION -> { + return null; + } + case INLINING_DISALLOWED -> { + return DO_NOT_INLINE_WITH_EXCEPTION; + } + } + assert result == INLINE; + if (hostVM.isAnalysisTrivialMethod((AnalysisMethod) original) && builder.getDepth() < InlineDuringParsingMaxDepth.getValue(HostedOptionValues.singleton())) { return createStandardInlineInfo(original); } @@ -786,7 +818,12 @@ private void warnAllMethods(MetaAccessProvider metaAccess, Class clazz) { public void beforeCompilation(BeforeCompilationAccess config) { FeatureImpl.BeforeCompilationAccessImpl access = (FeatureImpl.BeforeCompilationAccessImpl) config; - boolean failBlockListViolations = Options.TruffleCheckBlockListMethods.getValue() || Options.TruffleCheckBlackListedMethods.getValue(); + boolean failBlockListViolations; + if (Options.TruffleCheckBlackListedMethods.hasBeenSet()) { + failBlockListViolations = Options.TruffleCheckBlackListedMethods.getValue(); + } else { + failBlockListViolations = Options.TruffleCheckBlockListMethods.getValue(); + } boolean printBlockListViolations = RuntimeCompilationFeature.Options.PrintRuntimeCompileMethods.getValue() || failBlockListViolations; if (printBlockListViolations) { Set blocklistViolations = new TreeSet<>(RuntimeCompilationFeature.singleton().getRuntimeCompilationComparator());