[GR-67668] Implement PINNED_TO_INITIAL_LAYER and improve CompilationBehavior

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java

@@ -438,6 +438,10 @@ public boolean buildingImageLayer() {
         return false;
     }
 
+    public boolean buildingInitialLayer() {
+        return false;
+    }
+
     public boolean buildingSharedLayer() {
         return false;
     }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java

@@ -313,10 +313,21 @@ protected AnalysisMethod(AnalysisMethod original, MultiMethodKey multiMethodKey)
         this.enableReachableInCurrentLayer = original.enableReachableInCurrentLayer;
     }
 
+    /**
+     * This method should not be used directly, except to set the {@link CompilationBehavior} from a
+     * previous layer. To set a new {@link CompilationBehavior}, please use the associated setter.
+     */
     public void setCompilationBehavior(CompilationBehavior compilationBehavior) {
+        assert getUniverse().getBigbang().getHostVM().buildingImageLayer() : "The method compilation behavior can only be set in layered images";
         this.compilationBehavior = compilationBehavior;
     }
 
+    private void setNewCompilationBehavior(CompilationBehavior compilationBehavior) {
+        assert (!isInBaseLayer && this.compilationBehavior == CompilationBehavior.DEFAULT) || this.compilationBehavior == compilationBehavior : "The method was already assigned " +
+                        this.compilationBehavior + ", but trying to assign " + compilationBehavior;
+        setCompilationBehavior(compilationBehavior);
+    }
+
     public CompilationBehavior getCompilationBehavior() {
         return compilationBehavior;
     }
@@ -327,11 +338,11 @@ public CompilationBehavior getCompilationBehavior() {
      */
     public void setFullyDelayedToApplicationLayer() {
         HostVM hostVM = getUniverse().getBigbang().getHostVM();
-        AnalysisError.guarantee(hostVM.buildingImageLayer(), "Methods can only be delayed in layered images");
+        AnalysisError.guarantee(hostVM.buildingImageLayer(), "Methods can only be delayed in layered images: %s", this);
         AnalysisError.guarantee(parsedGraphCacheState.get() == GraphCacheEntry.UNPARSED, "The method %s was marked as delayed to the application layer but was already parsed", this);
         AnalysisError.guarantee(!hostVM.hasAlwaysInlineDirective(this), "Method %s with an always inline directive cannot be delayed to the application layer as such methods cannot be inlined", this);
         AnalysisError.guarantee(isConcrete(), "Method %s is not concrete and cannot be delayed to the application layer", this);
-        this.compilationBehavior = CompilationBehavior.FULLY_DELAYED_TO_APPLICATION_LAYER;
+        setNewCompilationBehavior(CompilationBehavior.FULLY_DELAYED_TO_APPLICATION_LAYER);
     }
 
     /**
@@ -342,6 +353,22 @@ public boolean isDelayed() {
         return compilationBehavior == CompilationBehavior.FULLY_DELAYED_TO_APPLICATION_LAYER && buildingSharedLayer;
     }
 
+    public void setPinnedToInitialLayer() {
+        BigBang bigbang = getUniverse().getBigbang();
+        AnalysisError.guarantee(bigbang.getHostVM().buildingInitialLayer(), "Methods can only be pinned to the initial layer: %s", this);
+        boolean nonAbstractInstanceClass = !declaringClass.isArray() && declaringClass.isInstanceClass() && !declaringClass.isAbstract();
+        AnalysisError.guarantee(nonAbstractInstanceClass, "Only methods from non abstract instance class can be delayed: %s", this);
+        bigbang.forcedAddRootMethod(this, true, "Method is pinned to the initial layer");
+        if (!isStatic()) {
+            declaringClass.registerAsInstantiated(this + " is pinned to the initial layer");
+        }
+        setNewCompilationBehavior(CompilationBehavior.PINNED_TO_INITIAL_LAYER);
+    }
+
+    public boolean isPinnedToInitialLayer() {
+        return compilationBehavior == CompilationBehavior.PINNED_TO_INITIAL_LAYER;
+    }
+
     private static String createName(ResolvedJavaMethod wrapped, MultiMethodKey multiMethodKey) {
         String aName = wrapped.getName();
         if (multiMethodKey != ORIGINAL_METHOD) {
@@ -1429,9 +1456,9 @@ public enum CompilationBehavior {
         DEFAULT,
 
         /**
-         * Method is pinned to a specific shared layer, meaning it has to be analyzed and compiled
-         * in this specific layer. A method can only be pinned to a shared layer.
+         * Method is pinned to the initial layer, meaning it has to be analyzed and compiled in this
+         * specific layer.
          */
-        PINNED_TO_SHARED_LAYER,
+        PINNED_TO_INITIAL_LAYER,
     }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java

@@ -215,6 +215,7 @@ public enum UsageKind {
 
     private final int layerId;
     private final boolean buildingImageLayer = ImageLayerBuildingSupport.buildingImageLayer();
+    private final boolean buildingInitialLayer = ImageLayerBuildingSupport.buildingInitialLayer();
     private final boolean buildingSharedLayer = ImageLayerBuildingSupport.buildingSharedLayer();
     private final boolean buildingExtensionLayer = ImageLayerBuildingSupport.buildingExtensionLayer();
 
@@ -284,6 +285,11 @@ public boolean buildingImageLayer() {
         return buildingImageLayer;
     }
 
+    @Override
+    public boolean buildingInitialLayer() {
+        return buildingInitialLayer;
+    }
+
     @Override
     public boolean buildingSharedLayer() {
         return buildingSharedLayer;

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerCompilationBehaviorFeature.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted.imagelayer;
+
+import com.oracle.graal.pointsto.BigBang;
+import com.oracle.graal.pointsto.meta.AnalysisMethod;
+import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
+import com.oracle.svm.core.feature.InternalFeature;
+import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
+import com.oracle.svm.core.util.VMError;
+import com.oracle.svm.hosted.FeatureImpl;
+import com.oracle.svm.hosted.meta.HostedMethod;
+
+@AutomaticallyRegisteredFeature
+public class LayerCompilationBehaviorFeature implements InternalFeature {
+    @Override
+    public boolean isInConfiguration(IsInConfigurationAccess access) {
+        return ImageLayerBuildingSupport.buildingImageLayer();
+    }
+
+    @Override
+    public void beforeAnalysis(BeforeAnalysisAccess a) {
+        if (ImageLayerBuildingSupport.buildingApplicationLayer()) {
+            /*
+             * Methods fully delayed to the application layer that are referenced in a shared layer
+             * need to be marked as root and compiled in the application layer to avoid any
+             * undefined reference.
+             */
+            SVMImageLayerLoader loader = HostedImageLayerBuildingSupport.singleton().getLoader();
+            FeatureImpl.BeforeAnalysisAccessImpl access = (FeatureImpl.BeforeAnalysisAccessImpl) a;
+            BigBang bigbang = access.getUniverse().getBigbang();
+            for (int methodId : HostedDynamicLayerInfo.singleton().getPreviousLayerDelayedMethodIds()) {
+                AnalysisMethod method = loader.getAnalysisMethodForBaseLayerId(methodId);
+                bigbang.forcedAddRootMethod(method, method.isConstructor(), "Fully delayed to application layer");
+            }
+        }
+    }
+
+    @Override
+    public void afterHeapLayout(AfterHeapLayoutAccess a) {
+        boolean buildingInitialLayer = ImageLayerBuildingSupport.buildingInitialLayer();
+        /* Methods pinned to a shared layer need to be compiled in the corresponding layer. */
+        FeatureImpl.AfterHeapLayoutAccessImpl access = (FeatureImpl.AfterHeapLayoutAccessImpl) a;
+        for (HostedMethod method : access.getHeap().hUniverse.getMethods()) {
+            if (method.wrapped.isPinnedToInitialLayer()) {
+                String msg = "The method %s was pinned to the initial layer but was not compiled in the initial layer";
+                if (buildingInitialLayer) {
+                    VMError.guarantee(method.isCompiled(), msg, method);
+                } else {
+                    VMError.guarantee(method.isCompiledInPriorLayer(), msg, method);
+                }
+            }
+        }
+    }
+}