[GR-38180] Encode compilation graphs to reduce memory footprint of the image generator.

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionEpilogueNode.java

@@ -55,16 +55,18 @@ public final class CFunctionEpilogueNode extends AbstractStateSplit implements L
     public CFunctionEpilogueNode(int oldThreadStatus) {
         super(TYPE, StampFactory.forVoid());
         this.oldThreadStatus = oldThreadStatus;
-        marker = new CFunctionEpilogueMarker();
     }
 
     @Override
     protected void afterClone(Node other) {
         super.afterClone(other);
-        marker = new CFunctionEpilogueMarker();
+        assert marker == null : "Marker must be unique";
     }
 
     public CFunctionEpilogueMarker getMarker() {
+        if (marker == null) {
+            marker = new CFunctionEpilogueMarker();
+        }
         return marker;
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nodes/CFunctionPrologueNode.java

@@ -62,28 +62,32 @@ public final class CFunctionPrologueNode extends FixedWithNextNode implements Lo
 
     private final int newThreadStatus;
     /**
-     * The marker object prevents value numbering of the node. This means that the marker must be a
-     * unique object per node, even after node cloning (e.g., because of method inlining).
-     * Therefore, {@link #afterClone} properly re-initializes the field to a new marker instance.
-     *
-     * The marker is also used for LIR frame state verification, to ensure we have a proper matching
-     * of prologue and epilogue and no unexpected machine code while the thread is in Native state.
+     * The marker is used for LIR frame state verification, to ensure we have a proper matching of
+     * prologue and epilogue and no unexpected machine code while the thread is in Native state.
      */
     private CFunctionPrologueMarker marker;
 
     public CFunctionPrologueNode(int newThreadStatus) {
         super(TYPE, StampFactory.forVoid());
         this.newThreadStatus = newThreadStatus;
-        marker = new CFunctionPrologueMarker(newThreadStatus);
     }
 
     @Override
     protected void afterClone(Node other) {
         super.afterClone(other);
-        marker = new CFunctionPrologueMarker(newThreadStatus);
+        /*
+         * Note that this method is invoked by the regular method inlining, but not by the
+         * PEGraphDecoder. So the method inlining before analysis, as well as the trivial method
+         * inlining before compilation, do not invoke this method. So it is only suitable for
+         * assertion checking.
+         */
+        assert marker == null : "Marker must be unique";
     }
 
     public CFunctionPrologueMarker getMarker() {
+        if (marker == null) {
+            marker = new CFunctionPrologueMarker(newThreadStatus);
+        }
         return marker;
     }
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationGraph.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted.code;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.graalvm.compiler.graph.NodeSourcePosition;
+import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
+import org.graalvm.compiler.nodes.EncodedGraph;
+import org.graalvm.compiler.nodes.GraphEncoder;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
+
+import com.oracle.graal.pointsto.flow.AnalysisParsedGraph;
+import com.oracle.svm.hosted.meta.HostedMethod;
+
+/**
+ * A holder for an {@link EncodedGraph} while doing the AOT compilation in {@link CompileQueue}.
+ * Encoding the graph is important to reduce the memory footprint of the image generator. But a few
+ * properties of the graph need to remain accessible, for example to guide inlining decisions. All
+ * such information is stored in separate fields of this class. This ensures that an encoded graph
+ * and its extra information are always in sync.
+ */
+public final class CompilationGraph {
+
+    /** Information about an invocation in the encoded graph. */
+    public static class InvokeInfo {
+        private final InvokeKind invokeKind;
+        private final HostedMethod targetMethod;
+        private final HostedMethod directCaller;
+        private final NodeSourcePosition nodeSourcePosition;
+
+        InvokeInfo(InvokeKind invokeKind, HostedMethod targetMethod, HostedMethod directCaller, NodeSourcePosition nodeSourcePosition) {
+            this.invokeKind = invokeKind;
+            this.targetMethod = targetMethod;
+            this.directCaller = directCaller;
+            this.nodeSourcePosition = nodeSourcePosition;
+        }
+
+        public InvokeKind getInvokeKind() {
+            return invokeKind;
+        }
+
+        public HostedMethod getTargetMethod() {
+            return targetMethod;
+        }
+
+        public HostedMethod getDirectCaller() {
+            return directCaller;
+        }
+
+        public NodeSourcePosition getNodeSourcePosition() {
+            return nodeSourcePosition;
+        }
+    }
+
+    /** Information about an allocation in the encoded graph. */
+    public static class AllocationInfo {
+        private final NodeSourcePosition nodeSourcePosition;
+
+        AllocationInfo(NodeSourcePosition nodeSourcePosition) {
+            this.nodeSourcePosition = nodeSourcePosition;
+        }
+
+        public NodeSourcePosition getNodeSourcePosition() {
+            return nodeSourcePosition;
+        }
+    }
+
+    private final EncodedGraph encodedGraph;
+    private final int nodeCount;
+    private final Set<InvokeInfo> invokeInfos;
+    private final Set<AllocationInfo> allocationInfos;
+
+    private CompilationGraph(EncodedGraph encodedGraph, int nodeCount, Set<InvokeInfo> invokeInfos, Set<AllocationInfo> allocationInfos) {
+        this.encodedGraph = encodedGraph;
+        this.nodeCount = nodeCount;
+        this.invokeInfos = invokeInfos;
+        this.allocationInfos = allocationInfos;
+    }
+
+    static CompilationGraph encode(StructuredGraph graph) {
+        Set<InvokeInfo> invokeInfos = new HashSet<>();
+        Set<AllocationInfo> allocationInfos = new HashSet<>();
+        for (var n : graph.getNodes()) {
+            if (n instanceof MethodCallTargetNode) {
+                MethodCallTargetNode node = (MethodCallTargetNode) n;
+                invokeInfos.add(new InvokeInfo(
+                                node.invokeKind(),
+                                (HostedMethod) node.targetMethod(),
+                                (HostedMethod) node.invoke().stateAfter().getMethod(),
+                                node.getNodeSourcePosition()));
+            }
+            if (UninterruptibleAnnotationChecker.isAllocationNode(n)) {
+                allocationInfos.add(new AllocationInfo(n.getNodeSourcePosition()));
+            }
+        }
+
+        return new CompilationGraph(
+                        GraphEncoder.encodeSingleGraph(graph, AnalysisParsedGraph.HOST_ARCHITECTURE),
+                        graph.getNodeCount(),
+                        invokeInfos.isEmpty() ? Collections.emptySet() : invokeInfos,
+                        allocationInfos.isEmpty() ? Collections.emptySet() : allocationInfos);
+    }
+
+    public EncodedGraph getEncodedGraph() {
+        return encodedGraph;
+    }
+
+    public int getNodeCount() {
+        return nodeCount;
+    }
+
+    public Set<InvokeInfo> getInvokeInfos() {
+        return invokeInfos;
+    }
+
+    public Set<AllocationInfo> getAllocationInfos() {
+        return allocationInfos;
+    }
+}

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompilationInfo.java

@@ -27,10 +27,14 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.graalvm.compiler.core.common.CompilationIdentifier;
+import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.nodes.ConstantNode;
+import org.graalvm.compiler.nodes.GraphDecoder;
 import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.options.OptionValues;
 
-import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.graal.pointsto.flow.AnalysisParsedGraph;
 import com.oracle.svm.core.annotate.DeoptTest;
 import com.oracle.svm.core.annotate.Specialize;
 import com.oracle.svm.hosted.code.CompileQueue.CompileFunction;
@@ -48,14 +52,13 @@ public class CompilationInfo {
      */
     protected boolean inCompileQueue;
 
-    protected volatile StructuredGraph graph;
+    private volatile CompilationGraph compilationGraph;
+    private OptionValues compileOptions;
 
     protected boolean isTrivialMethod;
 
     protected boolean canDeoptForTesting;
 
-    protected boolean modifiesSpecialRegisters;
-
     /**
      * The constant arguments for a {@link DeoptTest} method called by a {@link Specialize} method.
      * Note: this is only used for testing.
@@ -126,22 +129,41 @@ public HostedMethod getDeoptTargetMethod() {
         return deoptTarget;
     }
 
-    public void setGraph(StructuredGraph graph) {
-        this.graph = graph;
+    public CompilationGraph getCompilationGraph() {
+        return compilationGraph;
     }
 
-    public void clear() {
-        graph = null;
-        specializedArguments = null;
-    }
+    @SuppressWarnings("try")
+    public StructuredGraph createGraph(DebugContext debug, CompilationIdentifier compilationId, boolean decode) {
+        var graph = new StructuredGraph.Builder(compileOptions, debug)
+                        .method(method)
+                        .recordInlinedMethods(false)
+                        .trackNodeSourcePosition(getCompilationGraph().getEncodedGraph().trackNodeSourcePosition())
+                        .compilationId(compilationId)
+                        .build();
 
-    public StructuredGraph getGraph() {
+        if (decode) {
+            try (var s = debug.scope("CreateGraph", graph, method)) {
+                var decoder = new GraphDecoder(AnalysisParsedGraph.HOST_ARCHITECTURE, graph);
+                decoder.decode(getCompilationGraph().getEncodedGraph());
+            } catch (Throwable ex) {
+                throw debug.handle(ex);
+            }
+        }
         return graph;
     }
 
-    public boolean modifiesSpecialRegisters() {
-        assert SubstrateOptions.useLLVMBackend();
-        return modifiesSpecialRegisters;
+    void encodeGraph(StructuredGraph graph) {
+        compilationGraph = CompilationGraph.encode(graph);
+    }
+
+    public void setCompileOptions(OptionValues compileOptions) {
+        this.compileOptions = compileOptions;
+    }
+
+    public void clear() {
+        compilationGraph = null;
+        specializedArguments = null;
     }
 
     public boolean isTrivialMethod() {