diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionEpilogueNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionEpilogueNode.java index e799aa550e44..8c9ac2358e72 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionEpilogueNode.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionEpilogueNode.java @@ -55,16 +55,18 @@ public final class CFunctionEpilogueNode extends AbstractStateSplit implements L public CFunctionEpilogueNode(int oldThreadStatus) { super(TYPE, StampFactory.forVoid()); this.oldThreadStatus = oldThreadStatus; - marker = new CFunctionEpilogueMarker(); } @Override protected void afterClone(Node other) { super.afterClone(other); - marker = new CFunctionEpilogueMarker(); + assert marker == null : "Marker must be unique"; } public CFunctionEpilogueMarker getMarker() { + if (marker == null) { + marker = new CFunctionEpilogueMarker(); + } return marker; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionPrologueNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionPrologueNode.java index 381018d0a9a6..4038c759729d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionPrologueNode.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionPrologueNode.java @@ -62,28 +62,32 @@ public final class CFunctionPrologueNode extends FixedWithNextNode implements Lo private final int newThreadStatus; /** - * The marker object prevents value numbering of the node. This means that the marker must be a - * unique object per node, even after node cloning (e.g., because of method inlining). - * Therefore, {@link #afterClone} properly re-initializes the field to a new marker instance. - * - * The marker is also used for LIR frame state verification, to ensure we have a proper matching - * of prologue and epilogue and no unexpected machine code while the thread is in Native state. + * The marker is used for LIR frame state verification, to ensure we have a proper matching of + * prologue and epilogue and no unexpected machine code while the thread is in Native state. */ private CFunctionPrologueMarker marker; public CFunctionPrologueNode(int newThreadStatus) { super(TYPE, StampFactory.forVoid()); this.newThreadStatus = newThreadStatus; - marker = new CFunctionPrologueMarker(newThreadStatus); } @Override protected void afterClone(Node other) { super.afterClone(other); - marker = new CFunctionPrologueMarker(newThreadStatus); + /* + * Note that this method is invoked by the regular method inlining, but not by the + * PEGraphDecoder. So the method inlining before analysis, as well as the trivial method + * inlining before compilation, do not invoke this method. So it is only suitable for + * assertion checking. + */ + assert marker == null : "Marker must be unique"; } public CFunctionPrologueMarker getMarker() { + if (marker == null) { + marker = new CFunctionPrologueMarker(newThreadStatus); + } return marker; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationGraph.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationGraph.java new file mode 100644 index 000000000000..2736b88fe549 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationGraph.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.code; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.graalvm.compiler.graph.NodeSourcePosition; +import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; +import org.graalvm.compiler.nodes.EncodedGraph; +import org.graalvm.compiler.nodes.GraphEncoder; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.java.MethodCallTargetNode; + +import com.oracle.graal.pointsto.flow.AnalysisParsedGraph; +import com.oracle.svm.hosted.meta.HostedMethod; + +/** + * A holder for an {@link EncodedGraph} while doing the AOT compilation in {@link CompileQueue}. + * Encoding the graph is important to reduce the memory footprint of the image generator. But a few + * properties of the graph need to remain accessible, for example to guide inlining decisions. All + * such information is stored in separate fields of this class. This ensures that an encoded graph + * and its extra information are always in sync. + */ +public final class CompilationGraph { + + /** Information about an invocation in the encoded graph. */ + public static class InvokeInfo { + private final InvokeKind invokeKind; + private final HostedMethod targetMethod; + private final HostedMethod directCaller; + private final NodeSourcePosition nodeSourcePosition; + + InvokeInfo(InvokeKind invokeKind, HostedMethod targetMethod, HostedMethod directCaller, NodeSourcePosition nodeSourcePosition) { + this.invokeKind = invokeKind; + this.targetMethod = targetMethod; + this.directCaller = directCaller; + this.nodeSourcePosition = nodeSourcePosition; + } + + public InvokeKind getInvokeKind() { + return invokeKind; + } + + public HostedMethod getTargetMethod() { + return targetMethod; + } + + public HostedMethod getDirectCaller() { + return directCaller; + } + + public NodeSourcePosition getNodeSourcePosition() { + return nodeSourcePosition; + } + } + + /** Information about an allocation in the encoded graph. */ + public static class AllocationInfo { + private final NodeSourcePosition nodeSourcePosition; + + AllocationInfo(NodeSourcePosition nodeSourcePosition) { + this.nodeSourcePosition = nodeSourcePosition; + } + + public NodeSourcePosition getNodeSourcePosition() { + return nodeSourcePosition; + } + } + + private final EncodedGraph encodedGraph; + private final int nodeCount; + private final Set invokeInfos; + private final Set allocationInfos; + + private CompilationGraph(EncodedGraph encodedGraph, int nodeCount, Set invokeInfos, Set allocationInfos) { + this.encodedGraph = encodedGraph; + this.nodeCount = nodeCount; + this.invokeInfos = invokeInfos; + this.allocationInfos = allocationInfos; + } + + static CompilationGraph encode(StructuredGraph graph) { + Set invokeInfos = new HashSet<>(); + Set allocationInfos = new HashSet<>(); + for (var n : graph.getNodes()) { + if (n instanceof MethodCallTargetNode) { + MethodCallTargetNode node = (MethodCallTargetNode) n; + invokeInfos.add(new InvokeInfo( + node.invokeKind(), + (HostedMethod) node.targetMethod(), + (HostedMethod) node.invoke().stateAfter().getMethod(), + node.getNodeSourcePosition())); + } + if (UninterruptibleAnnotationChecker.isAllocationNode(n)) { + allocationInfos.add(new AllocationInfo(n.getNodeSourcePosition())); + } + } + + return new CompilationGraph( + GraphEncoder.encodeSingleGraph(graph, AnalysisParsedGraph.HOST_ARCHITECTURE), + graph.getNodeCount(), + invokeInfos.isEmpty() ? Collections.emptySet() : invokeInfos, + allocationInfos.isEmpty() ? Collections.emptySet() : allocationInfos); + } + + public EncodedGraph getEncodedGraph() { + return encodedGraph; + } + + public int getNodeCount() { + return nodeCount; + } + + public Set getInvokeInfos() { + return invokeInfos; + } + + public Set getAllocationInfos() { + return allocationInfos; + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationInfo.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationInfo.java index 62a1f80a229d..553c91bf94c6 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationInfo.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationInfo.java @@ -27,10 +27,14 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.GraphDecoder; import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.options.OptionValues; -import com.oracle.svm.core.SubstrateOptions; +import com.oracle.graal.pointsto.flow.AnalysisParsedGraph; import com.oracle.svm.core.annotate.DeoptTest; import com.oracle.svm.core.annotate.Specialize; import com.oracle.svm.hosted.code.CompileQueue.CompileFunction; @@ -48,14 +52,13 @@ public class CompilationInfo { */ protected boolean inCompileQueue; - protected volatile StructuredGraph graph; + private volatile CompilationGraph compilationGraph; + private OptionValues compileOptions; protected boolean isTrivialMethod; protected boolean canDeoptForTesting; - protected boolean modifiesSpecialRegisters; - /** * The constant arguments for a {@link DeoptTest} method called by a {@link Specialize} method. * Note: this is only used for testing. @@ -126,22 +129,41 @@ public HostedMethod getDeoptTargetMethod() { return deoptTarget; } - public void setGraph(StructuredGraph graph) { - this.graph = graph; + public CompilationGraph getCompilationGraph() { + return compilationGraph; } - public void clear() { - graph = null; - specializedArguments = null; - } + @SuppressWarnings("try") + public StructuredGraph createGraph(DebugContext debug, CompilationIdentifier compilationId, boolean decode) { + var graph = new StructuredGraph.Builder(compileOptions, debug) + .method(method) + .recordInlinedMethods(false) + .trackNodeSourcePosition(getCompilationGraph().getEncodedGraph().trackNodeSourcePosition()) + .compilationId(compilationId) + .build(); - public StructuredGraph getGraph() { + if (decode) { + try (var s = debug.scope("CreateGraph", graph, method)) { + var decoder = new GraphDecoder(AnalysisParsedGraph.HOST_ARCHITECTURE, graph); + decoder.decode(getCompilationGraph().getEncodedGraph()); + } catch (Throwable ex) { + throw debug.handle(ex); + } + } return graph; } - public boolean modifiesSpecialRegisters() { - assert SubstrateOptions.useLLVMBackend(); - return modifiesSpecialRegisters; + void encodeGraph(StructuredGraph graph) { + compilationGraph = CompilationGraph.encode(graph); + } + + public void setCompileOptions(OptionValues compileOptions) { + this.compileOptions = compileOptions; + } + + public void clear() { + compilationGraph = null; + specializedArguments = null; } public boolean isTrivialMethod() { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java index a198061451b9..3a44b49461c3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java @@ -47,6 +47,7 @@ import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.asm.Assembler; import org.graalvm.compiler.bytecode.Bytecode; +import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.bytecode.ResolvedJavaMethodBytecode; import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.code.DataSection; @@ -81,6 +82,7 @@ import org.graalvm.compiler.lir.phases.PostAllocationOptimizationPhase.PostAllocationOptimizationContext; import org.graalvm.compiler.nodes.CallTargetNode; import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.EncodedGraph; import org.graalvm.compiler.nodes.FieldLocationIdentity; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.FixedWithNextNode; @@ -99,6 +101,8 @@ import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedFoldInvocationPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.BytecodeExceptionMode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.util.GraphUtil; @@ -114,18 +118,19 @@ import org.graalvm.compiler.phases.common.CanonicalizerPhase; import org.graalvm.compiler.phases.common.FixReadsPhase; import org.graalvm.compiler.phases.common.FloatingReadPhase; -import org.graalvm.compiler.phases.common.inlining.InliningUtil; import org.graalvm.compiler.phases.tiers.HighTierContext; import org.graalvm.compiler.phases.tiers.LowTierContext; import org.graalvm.compiler.phases.tiers.MidTierContext; import org.graalvm.compiler.phases.tiers.Suites; import org.graalvm.compiler.phases.util.GraphOrder; import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.compiler.replacements.PEGraphDecoder; import org.graalvm.compiler.replacements.nodes.MacroInvokable; import org.graalvm.compiler.virtual.phases.ea.PartialEscapePhase; import org.graalvm.compiler.virtual.phases.ea.ReadEliminationPhase; import org.graalvm.nativeimage.ImageSingletons; +import com.oracle.graal.pointsto.flow.AnalysisParsedGraph; import com.oracle.graal.pointsto.infrastructure.GraphProvider.Purpose; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; @@ -166,6 +171,7 @@ import com.oracle.svm.hosted.NativeImageGenerator; import com.oracle.svm.hosted.NativeImageOptions; import com.oracle.svm.hosted.ProgressReporter; +import com.oracle.svm.hosted.diagnostic.HostedHeapDumpFeature; import com.oracle.svm.hosted.meta.HostedField; import com.oracle.svm.hosted.meta.HostedMethod; import com.oracle.svm.hosted.meta.HostedType; @@ -219,7 +225,7 @@ public interface CompileFunction { private SnippetReflectionProvider snippetReflection; private final FeatureHandler featureHandler; - private final OptionValues compileOptions; + protected final OptionValues compileOptions; private volatile boolean inliningProgress; @@ -321,9 +327,6 @@ public CompileTask(HostedMethod method, CompileReason reason) { @Override public void run(DebugContext debug) { - if (method.compilationInfo.graph != null) { - method.compilationInfo.graph.resetDebug(debug); - } result = doCompile(debug, method, compilationIdentifier, reason); } @@ -425,9 +428,15 @@ public void finish(DebugContext debug) { method.wrapped.setAnalyzedGraph(null); } + if (ImageSingletons.contains(HostedHeapDumpFeature.class)) { + ImageSingletons.lookup(HostedHeapDumpFeature.class).beforeInlining(); + } try (ProgressReporter.ReporterClosable ac = reporter.printInlining()) { inlineTrivialMethods(debug); } + if (ImageSingletons.contains(HostedHeapDumpFeature.class)) { + ImageSingletons.lookup(HostedHeapDumpFeature.class).afterInlining(); + } assert suitesNotCreated(); createSuites(); @@ -440,6 +449,9 @@ public void finish(DebugContext debug) { if (printMethodHistogram) { printMethodHistogram(); } + if (ImageSingletons.contains(HostedHeapDumpFeature.class)) { + ImageSingletons.lookup(HostedHeapDumpFeature.class).compileQueueAfterCompilation(); + } } private boolean suitesNotCreated() { @@ -624,24 +636,17 @@ private void ensureParsedForDeoptTesting(HostedMethod method) { ensureParsed(universe.createDeoptTarget(method), null, new EntryPointReason()); } - protected void checkTrivial(HostedMethod method) { - if (!method.compilationInfo.isTrivialMethod() && method.canBeInlined() && InliningUtilities.isTrivialMethod(method.compilationInfo.getGraph())) { + private static boolean checkTrivial(HostedMethod method, StructuredGraph graph) { + if (!method.compilationInfo.isTrivialMethod() && method.canBeInlined() && InliningUtilities.isTrivialMethod(graph)) { method.compilationInfo.setTrivialMethod(true); + return true; + } else { + return false; } } @SuppressWarnings("try") protected void inlineTrivialMethods(DebugContext debug) throws InterruptedException { - for (HostedMethod method : universe.getMethods()) { - try (DebugContext.Scope s = debug.scope("InlineTrivial", method.compilationInfo.getGraph(), method, this)) { - if (method.compilationInfo.getGraph() != null) { - checkTrivial(method); - } - } catch (Throwable e) { - throw debug.handle(e); - } - } - int round = 0; do { ProgressReporter.singleton().reportStageProgress(); @@ -650,10 +655,12 @@ protected void inlineTrivialMethods(DebugContext debug) throws InterruptedExcept try (Indent ignored = debug.logAndIndent("==== Trivial Inlining round %d\n", round)) { executor.init(); - universe.getMethods().stream().filter(method -> method.compilationInfo.getGraph() != null).forEach(method -> executor.execute(new TrivialInlineTask(method))); - - universe.getMethods().stream().map(method -> method.compilationInfo.getDeoptTargetMethod()).filter(Objects::nonNull).forEach( - deoptTargetMethod -> executor.execute(new TrivialInlineTask(deoptTargetMethod))); + universe.getMethods().stream() + .filter(method -> method.compilationInfo.getCompilationGraph() != null) + .forEach(method -> executor.execute(new TrivialInlineTask(method))); + universe.getMethods().stream() + .map(method -> method.compilationInfo.getDeoptTargetMethod()).filter(Objects::nonNull) + .forEach(deoptTargetMethod -> executor.execute(new TrivialInlineTask(deoptTargetMethod))); executor.start(); executor.complete(); executor.shutdown(); @@ -661,71 +668,79 @@ protected void inlineTrivialMethods(DebugContext debug) throws InterruptedExcept } while (inliningProgress); } - @SuppressWarnings("try") - private void doInlineTrivial(DebugContext debug, final HostedMethod method) { - /* - * Make a copy of the graph to avoid concurrency problems. Graph manipulations are not - * thread safe, and another thread can concurrently inline this method. - */ - final StructuredGraph graph = (StructuredGraph) method.compilationInfo.getGraph().copy(debug); + class TrivialInliningPlugin implements InlineInvokePlugin { - try (DebugContext.Scope s = debug.scope("InlineTrivial", graph, method, this)) { + boolean inlinedDuringDecoding; - try { - - try (Indent in = debug.logAndIndent("do inline trivial in %s", method)) { + @Override + public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { + if (makeInlineDecision((HostedMethod) method) && b.recursiveInliningDepth(method) == 0) { + inlinedDuringDecoding = true; + return InlineInfo.createStandardInlineInfo(method); + } else { + return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; + } + } + } - boolean inlined = false; - for (Invoke invoke : graph.getInvokes()) { - if (invoke instanceof InvokeNode) { - throw VMError.shouldNotReachHere("Found InvokeNode without exception edge: invocation of " + - invoke.callTarget().targetMethod().format("%H.%n(%p)") + " in " + (graph.method() == null ? graph.toString() : graph.method().format("%H.%n(%p)"))); - } + class InliningGraphDecoder extends PEGraphDecoder { - if (invoke.getInlineControl() == Invoke.InlineControl.Normal) { - inlined |= tryInlineTrivial(graph, invoke, !inlined); - } - } + InliningGraphDecoder(StructuredGraph graph, Providers providers, TrivialInliningPlugin inliningPlugin) { + super(AnalysisParsedGraph.HOST_ARCHITECTURE, graph, providers, null, + null, + new InlineInvokePlugin[]{inliningPlugin}, + null, null, null, null, + new ConcurrentHashMap<>(), new ConcurrentHashMap<>(), true); + } - if (inlined) { - Providers providers = runtimeConfig.lookupBackend(method).getProviders(); - CanonicalizerPhase.create().apply(graph, providers); + @Override + protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution, boolean trackNodeSourcePosition) { + return ((HostedMethod) method).compilationInfo.getCompilationGraph().getEncodedGraph(); + } + } - /* - * Publish the new graph, it can be picked up immediately by other threads - * trying to inline this method. - */ - graph.minimizeSize(); - method.compilationInfo.setGraph(graph); - checkTrivial(method); - inliningProgress = true; - } - } - } catch (Throwable ex) { - GraalError error = ex instanceof GraalError ? (GraalError) ex : new GraalError(ex); - error.addContext("method: " + method.format("%r %H.%n(%p)")); - throw error; + @SuppressWarnings("try") + private void doInlineTrivial(DebugContext debug, HostedMethod method) { + /* + * Before doing any work, check if there is any potential for inlining. + * + * Note that we do not have information about the recursive inlining depth, but that is OK + * because in that case we just over-estimate the inlining potential, i.e., we do the + * decoding just to find out that nothing could be inlined. + */ + boolean inliningPotential = false; + for (var invokeInfo : method.compilationInfo.getCompilationGraph().getInvokeInfos()) { + if (invokeInfo.getInvokeKind().isDirect() && makeInlineDecision(invokeInfo.getTargetMethod())) { + inliningPotential = true; + break; } - - } catch (Throwable e) { - throw debug.handle(e); } - } + if (!inliningPotential) { + return; + } - private boolean tryInlineTrivial(StructuredGraph graph, Invoke invoke, boolean firstInline) { - if (invoke.getInvokeKind().isDirect()) { - HostedMethod singleCallee = (HostedMethod) invoke.callTarget().targetMethod(); - if (makeInlineDecision(singleCallee) && InliningUtilities.recursionDepth(invoke, singleCallee) == 0) { - if (firstInline) { - graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "Before inlining"); - } - InliningUtil.inline(invoke, singleCallee.compilationInfo.getGraph(), true, singleCallee); + var providers = runtimeConfig.lookupBackend(method).getProviders(); + var graph = method.compilationInfo.createGraph(debug, CompilationIdentifier.INVALID_COMPILATION_ID, false); + try (var s = debug.scope("InlineTrivial", graph, method, this)) { + var inliningPlugin = new TrivialInliningPlugin(); + var decoder = new InliningGraphDecoder(graph, providers, inliningPlugin); + decoder.decode(method, false, graph.trackNodeSourcePosition()); - graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "After inlining %s with trivial callee %s", invoke, singleCallee.getQualifiedName()); - return true; + if (inliningPlugin.inlinedDuringDecoding) { + CanonicalizerPhase.create().apply(graph, providers); + /* + * Publish the new graph, it can be picked up immediately by other threads trying to + * inline this method. This can be a minor source of non-determinism in inlining + * decisions. + */ + method.compilationInfo.encodeGraph(graph); + if (checkTrivial(method, graph)) { + inliningProgress = true; + } } + } catch (Throwable ex) { + throw debug.handle(ex); } - return false; } private boolean makeInlineDecision(HostedMethod callee) { @@ -1109,13 +1124,11 @@ private void defaultParseFunction(DebugContext debug, HostedMethod method, Compi graph.setGuardsStage(GuardsStage.FIXED_DEOPTS); } - method.compilationInfo.setGraph(graph); - afterParse(method); + afterParse(method, graph); PhaseSuite afterParseSuite = afterParseCanonicalization(); - afterParseSuite.apply(method.compilationInfo.graph, new HighTierContext(providers, afterParseSuite, getOptimisticOpts())); - assert GraphOrder.assertSchedulableGraph(method.compilationInfo.getGraph()); + afterParseSuite.apply(graph, new HighTierContext(providers, afterParseSuite, getOptimisticOpts())); + assert GraphOrder.assertSchedulableGraph(graph); - graph.minimizeSize(); method.compilationInfo.numNodesAfterParsing = graph.getNodeCount(); if (!parseOnce) { UninterruptibleAnnotationChecker.checkAfterParsing(method, graph); @@ -1140,6 +1153,10 @@ private void defaultParseFunction(DebugContext debug, HostedMethod method, Compi } } + method.compilationInfo.encodeGraph(graph); + method.compilationInfo.setCompileOptions(compileOptions); + checkTrivial(method, graph); + } catch (Throwable ex) { GraalError error = ex instanceof GraalError ? (GraalError) ex : new GraalError(ex); error.addContext("method: " + method.format("%r %H.%n(%p)")); @@ -1176,7 +1193,7 @@ private void ensureParsed(HostedMethod method, CompileReason reason, CallTargetN } @SuppressWarnings("unused") - protected void afterParse(HostedMethod method) { + protected void afterParse(HostedMethod method, StructuredGraph graph) { } protected OptionValues getCustomizedOptions(DebugContext debug) { @@ -1261,9 +1278,13 @@ protected boolean canBeUsedForInlining(Invoke invoke) { if (!mustNotAllocateCallee(caller) && mustNotAllocate(callee)) { return false; } - if (!callee.canBeInlined()) { - return false; - } + /* + * Note that we do not check callee.canBeInlined() yet. Otherwise a @NeverInline annotation + * on a virtual target method would also prevent inlining of a concrete implementation + * method (after a later de-virtualization of the invoke) that is not annotated + * with @NeverInline. It is the responsibility of every inlining phase to check + * canBeInlined(). + */ return invoke.useForInlining(); } @@ -1314,18 +1335,6 @@ protected void ensureCompiled(HostedMethod method, CompileReason reason) { } compilationInfo.inCompileQueue = true; - if (compilationInfo.specializedArguments != null) { - // Do the specialization: replace the argument locals with the constant arguments. - StructuredGraph graph = compilationInfo.graph; - - int idx = 0; - for (ConstantNode argument : compilationInfo.specializedArguments) { - ParameterNode local = graph.getParameter(idx++); - if (local != null) { - local.replaceAndDelete(ConstantNode.forConstant(argument.asJavaConstant(), runtimeConfig.getProviders().getMetaAccess(), graph)); - } - } - } executor.execute(task); method.setCompiled(); } @@ -1356,10 +1365,19 @@ private CompilationResult defaultCompileFunction(DebugContext debug, HostedMetho try { SubstrateBackend backend = config.lookupBackend(method); - StructuredGraph graph = method.compilationInfo.graph; - VMError.guarantee(graph != null, "The following method is reachable during compilation, but was not seen during Bytecode parsing: " + method); - /* Operate on a copy, to keep the original graph intact for later inlining. */ - graph = graph.copyWithIdentifier(compilationIdentifier, debug); + VMError.guarantee(method.compilationInfo.getCompilationGraph() != null, "The following method is reachable during compilation, but was not seen during Bytecode parsing: " + method); + StructuredGraph graph = method.compilationInfo.createGraph(debug, compilationIdentifier, true); + + if (method.compilationInfo.specializedArguments != null) { + // Do the specialization: replace the argument locals with the constant arguments. + int idx = 0; + for (ConstantNode argument : method.compilationInfo.specializedArguments) { + ParameterNode local = graph.getParameter(idx++); + if (local != null) { + local.replaceAndDelete(ConstantNode.forConstant(argument.asJavaConstant(), runtimeConfig.getProviders().getMetaAccess(), graph)); + } + } + } /* Check that graph is in good shape before compilation. */ assert GraphOrder.assertSchedulableGraph(graph); @@ -1388,7 +1406,7 @@ private CompilationResult defaultCompileFunction(DebugContext debug, HostedMetho method.compilationInfo.numNodesAfterCompilation = graph.getNodeCount(); if (method.compilationInfo.isDeoptTarget()) { - assert verifyDeoptTarget(method, result); + assert verifyDeoptTarget(method, graph, result); } ensureCalleesCompiled(method, reason, result); @@ -1461,18 +1479,18 @@ private static void removeDeoptTargetOptimizations(LIRSuites lirSuites) { lirSuites.getAllocationStage().findPhaseInstance(RegisterAllocationPhase.class).setNeverSpillConstants(true); } - private static boolean verifyDeoptTarget(HostedMethod method, CompilationResult result) { + private static boolean verifyDeoptTarget(HostedMethod method, StructuredGraph graph, CompilationResult result) { Map encodedBciMap = new HashMap<>(); /* * All deopt targets must have a graph. */ - assert method.compilationInfo.graph != null : "Deopt target must have a graph."; + assert graph != null : "Deopt target must have a graph."; /* * No deopt targets can have a StackValueNode in the graph. */ - assert method.compilationInfo.graph.getNodes(StackValueNode.TYPE).isEmpty() : "No stack value nodes must be present in deopt target."; + assert graph.getNodes(StackValueNode.TYPE).isEmpty() : "No stack value nodes must be present in deopt target."; for (Infopoint infopoint : result.getInfopoints()) { if (infopoint.debugInfo != null) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/RestrictHeapAccessAnnotationChecker.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/RestrictHeapAccessAnnotationChecker.java index 3c5004227e7e..51ba6fd24f0c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/RestrictHeapAccessAnnotationChecker.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/RestrictHeapAccessAnnotationChecker.java @@ -30,12 +30,7 @@ import java.util.Map; import org.graalvm.compiler.debug.DebugContext; -import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeSourcePosition; -import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.java.AbstractNewObjectNode; -import org.graalvm.compiler.nodes.java.NewMultiArrayNode; -import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; import org.graalvm.compiler.options.Option; import org.graalvm.nativeimage.ImageSingletons; @@ -69,26 +64,15 @@ public static void check(DebugContext debug, HostedUniverse universe, Collection } } - static Node checkViolatingNode(StructuredGraph graph, Access access) { + static CompilationGraph.AllocationInfo checkViolatingNode(CompilationGraph graph) { if (graph != null) { - for (Node node : graph.getNodes()) { - if (isViolatingNode(node, access)) { - return node; - } + for (CompilationGraph.AllocationInfo node : graph.getAllocationInfos()) { + return node; } } return null; } - private static boolean isViolatingNode(Node node, Access access) { - assert access != Access.UNRESTRICTED : "does not require checks"; - return isAllocationNode(node); - } - - static boolean isAllocationNode(Node node) { - return (node instanceof CommitAllocationNode || node instanceof AbstractNewObjectNode || node instanceof NewMultiArrayNode); - } - /** A HostedMethod visitor that checks for violations of heap access restrictions. */ static class RestrictHeapAccessWarningVisitor { @@ -108,8 +92,8 @@ public void visitMethod(DebugContext debug, HostedMethod method) { return; } /* Look through the graph for this method and see if it allocates. */ - final StructuredGraph graph = method.compilationInfo.getGraph(); - if (RestrictHeapAccessAnnotationChecker.checkViolatingNode(graph, info.getAccess()) != null) { + final CompilationGraph graph = method.compilationInfo.getCompilationGraph(); + if (RestrictHeapAccessAnnotationChecker.checkViolatingNode(graph) != null) { try (DebugContext.Scope s = debug.scope("RestrictHeapAccessAnnotationChecker", graph, method, this)) { postRestrictHeapAccessWarning(method.getWrapped(), restrictHeapAccessCallees.getCallerMap()); } catch (Throwable t) { @@ -135,13 +119,13 @@ private void postRestrictHeapAccessWarning(AnalysisMethod violatingCallee, Map allocationList = new ArrayDeque<>(); for (RestrictionInfo element : callChain) { allocationList.addLast(element); - if (checkHostedViolatingNode(element.getMethod(), element.getAccess()) != null) { + if (checkHostedViolatingNode(element.getMethod()) != null) { break; } } assert !allocationList.isEmpty(); if (allocationList.size() == 1) { - final StackTraceElement allocationStackTraceElement = getViolatingStackTraceElement(violatingCallee, violatedAccess); + final StackTraceElement allocationStackTraceElement = getViolatingStackTraceElement(violatingCallee); if (allocationStackTraceElement != null) { message += "Restricted method '" + allocationStackTraceElement.toString() + "' directly violates restriction " + violatedAccess + "."; } else { @@ -159,7 +143,7 @@ private void postRestrictHeapAccessWarning(AnalysisMethod violatingCallee, Map methods) { UninterruptibleAnnotationChecker c = singleton(); for (HostedMethod method : methods) { Uninterruptible annotation = method.getAnnotation(Uninterruptible.class); - StructuredGraph graph = method.compilationInfo.getGraph(); + CompilationGraph graph = method.compilationInfo.getCompilationGraph(); c.checkSpecifiedOptions(method, annotation); c.checkOverrides(method, annotation); c.checkCallees(method, annotation, graph); @@ -156,23 +157,23 @@ private void checkOverrides(HostedMethod method, Uninterruptible methodAnnotatio * A caller can be annotated with "calleeMustBe = false" to allow calls to methods that are not * annotated with {@link Uninterruptible}, to allow the few cases where that should be allowed. */ - private void checkCallees(HostedMethod caller, Uninterruptible callerAnnotation, StructuredGraph graph) { + private void checkCallees(HostedMethod caller, Uninterruptible callerAnnotation, CompilationGraph graph) { if (callerAnnotation == null || graph == null) { return; } - for (Invoke invoke : graph.getInvokes()) { - HostedMethod callee = (HostedMethod) invoke.callTarget().targetMethod(); + for (CompilationGraph.InvokeInfo invoke : graph.getInvokeInfos()) { + HostedMethod callee = invoke.getTargetMethod(); if (Options.PrintUninterruptibleCalleeDOTGraph.getValue()) { printDotGraphEdge(caller, callee); } - Uninterruptible directCallerAnnotation = invoke.stateAfter().getMethod().getAnnotation(Uninterruptible.class); + Uninterruptible directCallerAnnotation = invoke.getDirectCaller().getAnnotation(Uninterruptible.class); if (directCallerAnnotation == null) { - violations.add("Unannotated callee: " + invoke.stateAfter().getMethod().format("%H.%n(%p)") + " inlined into annotated caller " + caller.format("%H.%n(%p)") + - System.lineSeparator() + FrameState.toSourcePosition(invoke.stateAfter())); + violations.add("Unannotated callee: " + invoke.getDirectCaller().format("%H.%n(%p)") + " inlined into annotated caller " + caller.format("%H.%n(%p)") + + System.lineSeparator() + invoke.getNodeSourcePosition()); } else if (directCallerAnnotation.calleeMustBe() && !isNotInterruptible(callee)) { violations.add("Unannotated callee: " + callee.format("%H.%n(%p)") + " called by annotated caller " + caller.format("%H.%n(%p)") + - System.lineSeparator() + FrameState.toSourcePosition(invoke.stateAfter())); + System.lineSeparator() + invoke.getNodeSourcePosition()); } } } @@ -181,12 +182,12 @@ private void checkCallees(HostedMethod caller, Uninterruptible callerAnnotation, * Check that each method that calls a method annotated with {@linkplain Uninterruptible} that * has "callerMustBe = true" is also annotated with {@linkplain Uninterruptible}. */ - private void checkCallers(HostedMethod caller, Uninterruptible callerAnnotation, StructuredGraph graph) { + private void checkCallers(HostedMethod caller, Uninterruptible callerAnnotation, CompilationGraph graph) { if (callerAnnotation != null || graph == null) { return; } - for (Invoke invoke : graph.getInvokes()) { - HostedMethod callee = (HostedMethod) invoke.callTarget().targetMethod(); + for (CompilationGraph.InvokeInfo invoke : graph.getInvokeInfos()) { + HostedMethod callee = invoke.getTargetMethod(); if (isCallerMustBe(callee)) { violations.add("Unannotated caller: " + caller.format("%H.%n(%p)") + " calls annotated callee " + callee.format("%H.%n(%p)")); } @@ -201,13 +202,17 @@ private void checkAllocations(ResolvedJavaMethod method, StructuredGraph graph) Uninterruptible methodAnnotation = method.getAnnotation(Uninterruptible.class); if (methodAnnotation != null && graph != null) { for (Node node : graph.getNodes()) { - if (RestrictHeapAccessAnnotationChecker.isAllocationNode(node)) { + if (isAllocationNode(node)) { violations.add("Annotated method: " + method.format("%H.%n(%p)") + " allocates."); } } } } + public static boolean isAllocationNode(Node node) { + return node instanceof CommitAllocationNode || node instanceof AbstractNewObjectNode || node instanceof NewMultiArrayNode; + } + private static boolean isNotInterruptible(HostedMethod method) { return (isUninterruptible(method) || isNoTransitionCFunction(method)); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java index 928b37d23049..f13e94161cc0 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java @@ -60,7 +60,10 @@ static class Options { enum Phases { DuringAnalysis("during-analysis"), AfterAnalysis("after-analysis"), - BeforeCompilation("before-compilation"); + BeforeCompilation("before-compilation"), + CompileQueueBeforeInlining("compile-queue-before-inlining"), + CompileQueueAfterInlining("compile-queue-after-inlining"), + CompileQueueAfterCompilation("compile-queue-after-compilation"); final String name; @@ -113,15 +116,29 @@ public void duringAnalysis(DuringAnalysisAccess access) { @Override public void onAnalysisExit(OnAnalysisExitAccess access) { - if (phases.contains(Phases.AfterAnalysis.getName())) { - dumpHeap(Phases.AfterAnalysis.getName()); - } + dumpHeap(Phases.AfterAnalysis); } @Override public void beforeCompilation(BeforeCompilationAccess access) { - if (phases.contains(Phases.BeforeCompilation.getName())) { - dumpHeap(Phases.BeforeCompilation.getName()); + dumpHeap(Phases.BeforeCompilation); + } + + public void beforeInlining() { + dumpHeap(Phases.CompileQueueBeforeInlining); + } + + public void afterInlining() { + dumpHeap(Phases.CompileQueueAfterInlining); + } + + public void compileQueueAfterCompilation() { + dumpHeap(Phases.CompileQueueAfterCompilation); + } + + private void dumpHeap(Phases phase) { + if (phases.contains(phase.getName())) { + dumpHeap(phase.getName()); } }