[GR-65719] Delay methods to the application layer

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java

@@ -346,7 +346,7 @@ public AnalysisMethod addRootMethod(Executable method, boolean invokeSpecial, Ob
 
     @Override
     public AnalysisMethod forcedAddRootMethod(AnalysisMethod method, boolean invokeSpecial, Object reason, MultiMethod.MultiMethodKey... otherRoots) {
-        AnalysisError.guarantee(isBaseLayerAnalysisEnabled());
+        AnalysisError.guarantee(isBaseLayerAnalysisEnabled() || hostVM.buildingImageLayer());
         registerDefaultMethod(method, reason);
         PointsToAnalysisMethod analysisMethod = assertPointsToAnalysisMethod(method);
         postTask(ignore -> {

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java

@@ -184,10 +184,6 @@ public boolean isCoreType(@SuppressWarnings("unused") AnalysisType type) {
         return false;
     }
 
-    public boolean useBaseLayer() {
-        return false;
-    }
-
     public boolean analyzedInPriorLayer(@SuppressWarnings("unused") AnalysisMethod method) {
         return false;
     }
@@ -288,6 +284,16 @@ public boolean hasNeverInlineDirective(ResolvedJavaMethod method) {
         return true;
     }
 
+    /**
+     * Check if the method has to be inlined.
+     *
+     * @param method the target method
+     */
+    public boolean hasAlwaysInlineDirective(ResolvedJavaMethod method) {
+        /* No force inlining by the static analysis unless explicitly overwritten by the VM. */
+        return false;
+    }
+
     public InlineBeforeAnalysisGraphDecoder createInlineBeforeAnalysisGraphDecoder(BigBang bb, AnalysisMethod method, StructuredGraph resultGraph) {
         /* No inlining by the static analysis unless explicitly overwritten by the VM. */
         return new InlineBeforeAnalysisGraphDecoder(bb, InlineBeforeAnalysisPolicy.NO_INLINING, resultGraph, bb.getProviders(method), null);
@@ -432,6 +438,14 @@ public boolean buildingImageLayer() {
         return false;
     }
 
+    public boolean buildingSharedLayer() {
+        return false;
+    }
+
+    public boolean buildingExtensionLayer() {
+        return false;
+    }
+
     @SuppressWarnings("unused")
     public boolean installableInLayer(AnalysisField aField) {
         return true;

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/InvokeTypeFlow.java

@@ -278,6 +278,10 @@ protected void linkCallee(PointsToAnalysis bb, boolean isStatic, MethodFlowsGrap
         if (!calleeFlows.isStub()) {
             bb.analysisPolicy().registerAsImplementationInvoked(this, calleeFlows.getMethod());
         }
+
+        if (calleeFlows.getMethod().isDelayed()) {
+            saturateForOpenTypeWorld(bb);
+        }
     }
 
     /*
@@ -426,4 +430,15 @@ public boolean canBeStaticallyBound() {
     public MultiMethodKey getCallerMultiMethodKey() {
         return callerMultiMethodKey;
     }
+
+    /**
+     * Saturates the actual return of the invoke type flow to ensure that the type state represents
+     * all the types that could exist in the open world.
+     */
+    public void saturateForOpenTypeWorld(PointsToAnalysis bb) {
+        if (actualReturn != null) {
+            actualReturn.enableFlow(bb);
+            actualReturn.onSaturated(bb);
+        }
+    }
 }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodFlowsGraph.java

@@ -490,10 +490,7 @@ void saturateForOpenTypeWorld(PointsToAnalysis bb) {
          */
         for (InvokeTypeFlow invokeTypeFlow : getInvokes()) {
             if (!invokeTypeFlow.isDirectInvoke() && !bb.isClosed(invokeTypeFlow.getReceiverType())) {
-                if (invokeTypeFlow.actualReturn != null) {
-                    invokeTypeFlow.actualReturn.enableFlow(bb);
-                    invokeTypeFlow.actualReturn.onSaturated(bb);
-                }
+                invokeTypeFlow.saturateForOpenTypeWorld(bb);
             }
         }
     }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java

@@ -706,6 +706,11 @@ protected void apply(boolean forceReparse, Object reason) {
 
         method.setReachableInCurrentLayer();
 
+        if (method.isDelayed()) {
+            /* The method will be analyzed in the application layer */
+            return;
+        }
+
         if (method.analyzedInPriorLayer()) {
             /*
              * We don't need to analyze this method. We already know its return type state from the

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java

@@ -333,11 +333,11 @@ private ImageHeapConstant getSnapshot(JavaConstant constant, ScanReason reason)
                 result = (ImageHeapConstant) constant;
             } else {
                 Object task = imageHeap.getSnapshot(constant);
-                if (task == null && bb.getUniverse().hostVM().useBaseLayer() && bb.getUniverse().getImageLayerLoader().hasValueForConstant(constant)) {
+                if (task == null && bb.getUniverse().hostVM().buildingExtensionLayer() && bb.getUniverse().getImageLayerLoader().hasValueForConstant(constant)) {
                     /* The constant might not have been accessed in the extension image yet */
                     task = bb.getUniverse().getImageLayerLoader().getValueForConstant(constant);
                 }
-                if (task == null && bb.getUniverse().hostVM().useBaseLayer()) {
+                if (task == null && bb.getUniverse().hostVM().buildingExtensionLayer()) {
                     /*
                      * This does not distinguish between base and extension layer constants at the
                      * moment. Doing so would require some refactoring to determine earlier if the

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java

@@ -266,7 +266,7 @@ protected ImageHeapConstant getOrCreateImageHeapConstant(JavaConstant javaConsta
         if (existingTask == null) {
             AnalysisFuture<ImageHeapConstant> newTask;
             ImageLayerLoader imageLayerLoader = universe.getImageLayerLoader();
-            if (hostVM.useBaseLayer() && imageLayerLoader.hasValueForConstant(javaConstant)) {
+            if (hostVM.buildingExtensionLayer() && imageLayerLoader.hasValueForConstant(javaConstant)) {
                 ImageHeapConstant value = imageLayerLoader.getValueForConstant(javaConstant);
                 ensureFieldPositionsComputed(value, nonNullReason);
                 AnalysisError.guarantee(value.getHostedObject().equals(javaConstant));

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisField.java

@@ -141,7 +141,7 @@ public AnalysisField(AnalysisUniverse universe, ResolvedJavaField wrappedField)
             sinkFlow = new ContextInsensitiveFieldTypeFlow(this, getType());
         }
 
-        if (universe.hostVM().useBaseLayer() && declaringClass.isInBaseLayer()) {
+        if (universe.hostVM().buildingExtensionLayer() && declaringClass.isInBaseLayer()) {
             int fid = universe.getImageLayerLoader().lookupHostedFieldInBaseLayer(this);
             if (fid != -1) {
                 /*

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java

@@ -51,6 +51,7 @@
 
 import com.oracle.graal.pointsto.BigBang;
 import com.oracle.graal.pointsto.PointsToAnalysis;
+import com.oracle.graal.pointsto.api.HostVM;
 import com.oracle.graal.pointsto.api.ImageLayerLoader;
 import com.oracle.graal.pointsto.api.ImageLayerWriter;
 import com.oracle.graal.pointsto.constraints.UnsupportedFeatureException;
@@ -126,6 +127,7 @@ public record Signature(String name, AnalysisType[] parameterTypes) {
     public final ResolvedJavaMethod wrapped;
 
     private final int id;
+    private final boolean buildingSharedLayer;
     /** Marks a method loaded from a base layer. */
     private final boolean isInBaseLayer;
     private final boolean analyzedInPriorLayer;
@@ -196,9 +198,12 @@ public record Signature(String name, AnalysisType[] parameterTypes) {
      */
     private boolean hasOpaqueReturn;
 
+    private CompilationBehavior compilationBehavior = CompilationBehavior.DEFAULT;
+
     @SuppressWarnings({"this-escape", "unchecked"})
     protected AnalysisMethod(AnalysisUniverse universe, ResolvedJavaMethod wrapped, MultiMethodKey multiMethodKey, Map<MultiMethodKey, MultiMethod> multiMethodMap) {
         super(universe.hostVM.enableTrackAcrossLayers());
+        HostVM hostVM = universe.hostVM();
         this.wrapped = wrapped;
 
         declaringClass = universe.lookup(wrapped.getDeclaringClass());
@@ -213,13 +218,14 @@ protected AnalysisMethod(AnalysisUniverse universe, ResolvedJavaMethod wrapped,
         } else {
             signature = getUniverse().lookup(wrappedSignature, wrapped.getDeclaringClass());
         }
-        hasNeverInlineDirective = universe.hostVM().hasNeverInlineDirective(wrapped);
+        hasNeverInlineDirective = hostVM.hasNeverInlineDirective(wrapped);
 
         name = createName(wrapped, multiMethodKey);
         qualifiedName = format("%H.%n(%P)");
         modifiers = wrapped.getModifiers();
 
-        if (universe.hostVM().useBaseLayer() && declaringClass.isInBaseLayer()) {
+        buildingSharedLayer = hostVM.buildingSharedLayer();
+        if (hostVM.buildingExtensionLayer() && declaringClass.isInBaseLayer()) {
             int mid = universe.getImageLayerLoader().lookupHostedMethodInBaseLayer(this);
             if (mid != -1) {
                 /*
@@ -236,7 +242,7 @@ protected AnalysisMethod(AnalysisUniverse universe, ResolvedJavaMethod wrapped,
             id = universe.computeNextMethodId();
             isInBaseLayer = false;
         }
-        analyzedInPriorLayer = isInBaseLayer && universe.hostVM().analyzedInPriorLayer(this);
+        analyzedInPriorLayer = isInBaseLayer && hostVM.analyzedInPriorLayer(this);
 
         ExceptionHandler[] original = wrapped.getExceptionHandlers();
         exceptionHandlers = new ExceptionHandler[original.length];
@@ -281,6 +287,7 @@ protected AnalysisMethod(AnalysisMethod original, MultiMethodKey multiMethodKey)
         super(original.enableTrackAcrossLayers);
         wrapped = original.wrapped;
         id = original.id;
+        buildingSharedLayer = original.buildingSharedLayer;
         isInBaseLayer = original.isInBaseLayer;
         analyzedInPriorLayer = original.analyzedInPriorLayer;
         declaringClass = original.declaringClass;
@@ -306,6 +313,35 @@ protected AnalysisMethod(AnalysisMethod original, MultiMethodKey multiMethodKey)
         this.enableReachableInCurrentLayer = original.enableReachableInCurrentLayer;
     }
 
+    public void setCompilationBehavior(CompilationBehavior compilationBehavior) {
+        this.compilationBehavior = compilationBehavior;
+    }
+
+    public CompilationBehavior getCompilationBehavior() {
+        return compilationBehavior;
+    }
+
+    /**
+     * Delays this method to the application layer. This should not be called after the method was
+     * already parsed to avoid analyzing all the method's callees.
+     */
+    public void setFullyDelayedToApplicationLayer() {
+        HostVM hostVM = getUniverse().getBigbang().getHostVM();
+        AnalysisError.guarantee(hostVM.buildingImageLayer(), "Methods can only be delayed in layered images");
+        AnalysisError.guarantee(parsedGraphCacheState.get() == GraphCacheEntry.UNPARSED, "The method %s was marked as delayed to the application layer but was already parsed", this);
+        AnalysisError.guarantee(!hostVM.hasAlwaysInlineDirective(this), "Method %s with an always inline directive cannot be delayed to the application layer as such methods cannot be inlined", this);
+        AnalysisError.guarantee(isConcrete(), "Method %s is not concrete and cannot be delayed to the application layer", this);
+        this.compilationBehavior = CompilationBehavior.FULLY_DELAYED_TO_APPLICATION_LAYER;
+    }
+
+    /**
+     * Returns true if this method is marked as delayed to the application layer and the current
+     * layer is a shared layer.
+     */
+    public boolean isDelayed() {
+        return compilationBehavior == CompilationBehavior.FULLY_DELAYED_TO_APPLICATION_LAYER && buildingSharedLayer;
+    }
+
     private static String createName(ResolvedJavaMethod wrapped, MultiMethodKey multiMethodKey) {
         String aName = wrapped.getName();
         if (multiMethodKey != ORIGINAL_METHOD) {
@@ -847,7 +883,8 @@ public Type[] getGenericParameterTypes() {
 
     @Override
     public boolean canBeInlined() {
-        return !hasNeverInlineDirective();
+        /* Delayed methods should not be inlined in the current layer */
+        return !hasNeverInlineDirective() && !isDelayed();
     }
 
     @Override
@@ -1202,6 +1239,7 @@ private AnalysisParsedGraph setGraph(BigBang bb, Stage stage, GraphCacheEntry ex
                 /* We lost the race, another thread is doing the parsing. */
                 return null;
             }
+            AnalysisError.guarantee(!isDelayed(), "The method %s was parsed even though it was marked as delayed to the application layer", this);
 
             GraphCacheEntry newEntry = graphSupplier.get(bb, this, expectedValue);
 
@@ -1360,4 +1398,40 @@ public boolean hasOpaqueReturn() {
     }
 
     protected abstract AnalysisMethod createMultiMethod(AnalysisMethod analysisMethod, MultiMethodKey newMultiMethodKey);
+
+    /**
+     * This state represents how a method should be compiled in layered images. The state of a
+     * method can only be decided in the first layer if it is marked as tracked across layers. The
+     * state has to stay the same across all the extension layers. If not specified, the state of a
+     * method will be {@link CompilationBehavior#DEFAULT}.
+     */
+    public enum CompilationBehavior {
+
+        /**
+         * Method remains unanalyzed until the application layer and any inlining in a shared layer
+         * is prevented. A call to the method in a shared layer will be replaced by an indirect
+         * call. The compilation of those methods is then forced in the application layer and the
+         * corresponding symbol is declared as global.
+         *
+         * A delayed method that is not referenced in any shared layer is treated as a
+         * {@link CompilationBehavior#DEFAULT} method in the application layer and does not have to
+         * be compiled. If it is only referenced in the application layer, it might be inlined and
+         * not compiled at all.
+         */
+        FULLY_DELAYED_TO_APPLICATION_LAYER,
+
+        /**
+         * Method can be inlined into other methods, both before analysis and during compilation,
+         * and will be compiled as a distinct compilation unit as stipulated by the normal native
+         * image generation process (i.e., the method is installed as a root and/or a reference to
+         * the method exists via a call and/or an explicit MethodReference).
+         */
+        DEFAULT,
+
+        /**
+         * Method is pinned to a specific shared layer, meaning it has to be analyzed and compiled
+         * in this specific layer. A method can only be pinned to a shared layer.
+         */
+        PINNED_TO_SHARED_LAYER,
+    }
 }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisType.java

@@ -287,7 +287,7 @@ public AnalysisType(AnalysisUniverse universe, ResolvedJavaType javaType, JavaKi
         }
 
         /* Set id after accessing super types, so that all these types get a lower id number. */
-        if (universe.hostVM().useBaseLayer()) {
+        if (universe.hostVM().buildingExtensionLayer()) {
             int tid = universe.getImageLayerLoader().lookupHostedTypeInBaseLayer(this);
             if (tid != -1) {
                 /*

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisUniverse.java

@@ -216,7 +216,7 @@ public JavaType lookupAllowUnresolved(JavaType rawType) {
         AnalysisType result = optionalLookup(type);
         if (result == null) {
             result = createType(type);
-            if (hostVM.useBaseLayer() && result.isInBaseLayer()) {
+            if (hostVM.buildingExtensionLayer() && result.isInBaseLayer()) {
                 imageLayerLoader.initializeBaseLayerType(result);
             }
         }
@@ -308,7 +308,7 @@ private AnalysisType createType(ResolvedJavaType type) {
              * ensures that typesById doesn't contain any null values. This could happen since the
              * AnalysisType constructor increments the nextTypeId counter.
              */
-            if (hostVM.useBaseLayer() && imageLayerLoader.hasDynamicHubIdentityHashCode(newValue.getId())) {
+            if (hostVM.buildingExtensionLayer() && imageLayerLoader.hasDynamicHubIdentityHashCode(newValue.getId())) {
                 hostVM.registerType(newValue, imageLayerLoader.getDynamicHubIdentityHashCode(newValue.getId()));
             } else {
                 hostVM.registerType(newValue);

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/results/StrengthenGraphs.java

@@ -40,6 +40,7 @@
 
 import com.oracle.graal.pointsto.BigBang;
 import com.oracle.graal.pointsto.PointsToAnalysis;
+import com.oracle.graal.pointsto.api.HostVM;
 import com.oracle.graal.pointsto.api.PointstoOptions;
 import com.oracle.graal.pointsto.flow.AnalysisParsedGraph;
 import com.oracle.graal.pointsto.flow.InvokeTypeFlow;
@@ -193,6 +194,8 @@ public boolean equals(Object obj) {
     /** Used to avoid aggressive optimizations for open type world analysis. */
     protected final boolean isClosedTypeWorld;
 
+    protected final boolean buildingSharedLayer;
+
     public StrengthenGraphs(BigBang bb, Universe converter) {
         this.bb = bb;
         this.converter = converter;
@@ -208,7 +211,9 @@ public StrengthenGraphs(BigBang bb, Universe converter) {
             beforeCounters = null;
             afterCounters = null;
         }
-        this.isClosedTypeWorld = converter.hostVM().isClosedTypeWorld();
+        HostVM hostVM = converter.hostVM();
+        this.isClosedTypeWorld = hostVM.isClosedTypeWorld();
+        this.buildingSharedLayer = hostVM.buildingSharedLayer();
     }
 
     private static void reportNeverNullInstanceFields(BigBang bb) {