[GR-62449] Ensure that class initialization status is stable between layers.

PullRequest: graal/20758

Author: cstancu

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java

@@ -363,10 +363,12 @@ private ImageHeapArray createImageHeapObjectArray(JavaConstant constant, Analysi
 
     public void registerBaseLayerValue(ImageHeapConstant constant, Object reason) {
         JavaConstant hostedValue = constant.getHostedObject();
+        AnalysisError.guarantee(hostedValue.isNonNull(), "A relinked constant cannot have a NULL_CONSTANT hosted value.");
         Object existingSnapshot = imageHeap.getSnapshot(hostedValue);
         if (existingSnapshot != null) {
             AnalysisError.guarantee(existingSnapshot == constant || existingSnapshot instanceof AnalysisFuture<?> task && task.ensureDone() == constant,
-                            "Found unexpected snapshot value for base layer value. Reason: %s.", reason);
+                            "Found unexpected snapshot value for base layer value.%nExisting value: %s.%nNew value: %s.%nHosted value: %s.%nReason: %s.",
+                            existingSnapshot, constant, hostedValue, reason);
         } else {
             imageHeap.setValue(hostedValue, constant);
         }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/BaseLayerType.java

@@ -56,7 +56,6 @@ public class BaseLayerType extends BaseLayerElement implements ResolvedJavaType,
     private final boolean isInterface;
     private final boolean isEnum;
     private final boolean isInitialized;
-    private final boolean isInitializedAtBuildTime;
     private final boolean isLinked;
     private final String sourceFileName;
     private final ResolvedJavaType enclosingType;
@@ -67,7 +66,7 @@ public class BaseLayerType extends BaseLayerElement implements ResolvedJavaType,
     private ResolvedJavaField[] instanceFields;
     private ResolvedJavaField[] instanceFieldsWithSuper;
 
-    public BaseLayerType(String name, int baseLayerId, int modifiers, boolean isInterface, boolean isEnum, boolean isInitialized, boolean initializedAtBuildTime, boolean isLinked,
+    public BaseLayerType(String name, int baseLayerId, int modifiers, boolean isInterface, boolean isEnum, boolean isInitialized, boolean isLinked,
                     String sourceFileName, ResolvedJavaType enclosingType, ResolvedJavaType componentType, ResolvedJavaType superClass, ResolvedJavaType[] interfaces, ResolvedJavaType objectType,
                     Annotation[] annotations) {
         super(annotations);
@@ -77,7 +76,6 @@ public BaseLayerType(String name, int baseLayerId, int modifiers, boolean isInte
         this.isInterface = isInterface;
         this.isEnum = isEnum;
         this.isInitialized = isInitialized;
-        this.isInitializedAtBuildTime = initializedAtBuildTime;
         this.isLinked = isLinked;
         this.sourceFileName = sourceFileName;
         this.enclosingType = enclosingType;
@@ -341,8 +339,4 @@ public ResolvedJavaType unwrapTowardsOriginalType() {
     public int getBaseLayerId() {
         return baseLayerId;
     }
-
-    public boolean initializedAtBuildTime() {
-        return isInitializedAtBuildTime;
-    }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md

@@ -201,6 +201,16 @@ native-image --module-path target/AwesomeLib-1.0-SNAPSHOT.jar --shared
 - A shared layer is using the _.so_ extension to conform with the standard OS loader restrictions. However, it is not a
   standard shared library file, and it cannot be used with other applications.
 
+### Class Initialization
+
+With Native Image Layers class initialization needs to be coherent between layers.
+To achieve this we enforce that the initialization state of types in shared layers stays exactly the same in the
+subsequent dependent layers.
+More concretely if a class `A` is initialized at build time in a base layer, then it will be automatically
+initialized at build time in the extension layers. The same holds for run time initialization.
+In the future we plan to relax this restriction and allow run-time initialized types in base layers to be promoted
+into build-time initialized types in the extension layers.
+
 ## Packaging Native Image Layers
 
 At build time a shared layer is stored in a layer archive that contains the following artifacts:

substratevm/src/com.oracle.svm.hosted/resources/SharedLayerSnapshotCapnProtoSchema.capnp

@@ -1,3 +1,7 @@
+# Schema file for the layer snapshot.
+# After modifying this file regenerate the schema SharedLayerSnapshotCapnProtoSchemaHolder.java file with:
+# mx capnp-compile
+
 @0x9eb32e19f86ee174;
 using Java = import "/capnp/java.capnp";
 $Java.package("com.oracle.svm.hosted.imagelayer");
@@ -21,35 +25,35 @@ struct PersistedAnalysisType {
   # Most of these fields apply only to instances and could be in a union or a separate structure:
   isInterface @7 :Bool;
   isEnum @8 :Bool;
+  # True if the type's initialization status was computed as BUILD_TIME. Build-time initialized types are not simulated.
   isInitialized @9 :Bool;
-  isInitializedAtBuildTime @10 :Bool;
-  isLinked @11 :Bool;
-  sourceFileName @12 :Text;
-  enclosingTypeId @13 :TypeId;
-  componentTypeId @14 :TypeId;
-  superClassTypeId @15 :TypeId;
-  isInstantiated @16 :Bool;
-  isUnsafeAllocated @17 :Bool;
-  isReachable @18 :Bool;
-  interfaces @19 :List(TypeId);
-  instanceFieldIds @20 :List(FieldId);
-  instanceFieldIdsWithSuper @21 :List(FieldId);
-  staticFieldIds @22 :List(FieldId);
-  annotationList @23 :List(Annotation);
-  classInitializationInfo @24 :ClassInitializationInfo;
-  hasArrayType @25 :Bool;
-  subTypes @26 :List(TypeId);
-  isAnySubtypeInstantiated @27 :Bool;
+  isLinked @10 :Bool;
+  sourceFileName @11 :Text;
+  enclosingTypeId @12 :TypeId;
+  componentTypeId @13 :TypeId;
+  superClassTypeId @14 :TypeId;
+  isInstantiated @15 :Bool;
+  isUnsafeAllocated @16 :Bool;
+  isReachable @17 :Bool;
+  interfaces @18 :List(TypeId);
+  instanceFieldIds @19 :List(FieldId);
+  instanceFieldIdsWithSuper @20 :List(FieldId);
+  staticFieldIds @21 :List(FieldId);
+  annotationList @22 :List(Annotation);
+  classInitializationInfo @23 :ClassInitializationInfo;
+  hasArrayType @24 :Bool;
+  subTypes @25 :List(TypeId);
+  isAnySubtypeInstantiated @26 :Bool;
   wrappedType :union {
-    none @28 :Void; # default
+    none @27 :Void; # default
     serializationGenerated :group {
-      rawDeclaringClass @29 :Text;
-      rawTargetConstructor @30 :Text;
+      rawDeclaringClass @28 :Text;
+      rawTargetConstructor @29 :Text;
     }
     lambda :group {
-      capturingClass @31 :Text;
+      capturingClass @30 :Text;
     }
-    proxyType @32 :Void;
+    proxyType @31 :Void;
   }
 }
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java

@@ -815,7 +815,7 @@ protected boolean runPointsToAnalysis(String imageName, OptionValues options, De
                 ServiceCatalogSupport.singleton().enableServiceCatalogMapTransformer(config);
                 featureHandler.forEachFeature(feature -> feature.beforeAnalysis(config));
                 ServiceCatalogSupport.singleton().seal();
-                bb.getHostVM().getClassInitializationSupport().setConfigurationSealed(true);
+                bb.getHostVM().getClassInitializationSupport().sealConfiguration();
                 if (ImageLayerBuildingSupport.buildingImageLayer()) {
                     ImageSingletons.lookup(LoadImageSingletonFeature.class).processRegisteredSingletons(aUniverse);
                 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ClassInitializationConfiguration.java

@@ -35,6 +35,7 @@
 import org.graalvm.collections.Pair;
 
 import com.oracle.svm.core.util.UserError;
+import com.oracle.svm.core.util.VMError;
 
 /**
  * Maintains user and system configuration for class initialization in Native Image.
@@ -52,12 +53,47 @@
  *
  * Every node tracks a list of reasons for the set configuration. This list helps the users debug
  * conflicts in the configuration.
+ * 
+ * A {@code strict} configuration, as specified by {@link InitializationNode#strict}, defines
+ * whether an initialization kind, as specified by {@link InitializationNode#kind}, was explicitly
+ * configured for the corresponding type. A {@code non-strict} configuration is a configuration
+ * inherited from a parent package configuration.
  */
 final class ClassInitializationConfiguration {
     private static final String ROOT_QUALIFIER = "";
     private static final int MAX_NUMBER_OF_REASONS = 10;
 
-    private InitializationNode root = new InitializationNode("", null, null, false);
+    private final InitializationNode root = new InitializationNode("", null, null, false);
+
+    synchronized void updateStrict(String classOrPackage, InitKind expectedKind, InitKind newKind, String reason) {
+        updateStrictRec(root, qualifierList(classOrPackage), expectedKind, newKind, reason);
+    }
+
+    private void updateStrictRec(InitializationNode node, List<String> classOrPackage, InitKind expectedKind, InitKind newKind, String reason) {
+        assert !classOrPackage.isEmpty();
+        assert node.qualifier.equals(classOrPackage.getFirst());
+        if (classOrPackage.size() == 1) {
+            VMError.guarantee(node.strict, "Expected a strict initialization policy for %s.", qualifiedName(node), newKind);
+            VMError.guarantee(newKind != expectedKind, "The initialization policy for %s is already %s.", qualifiedName(node), newKind);
+            VMError.guarantee(node.kind == expectedKind, "Found unexpected initialization policy for %s: expected %s, found %s.", qualifiedName(node), expectedKind, node.kind);
+            updateReason(node, reason);
+            node.kind = newKind;
+        } else {
+            List<String> tail = new ArrayList<>(classOrPackage);
+            tail.removeFirst();
+            String nextQualifier = tail.getFirst();
+            VMError.guarantee(node.children.containsKey(nextQualifier), "Expected that initialization node for %s already contains a child for %s.", qualifiedName(node), nextQualifier);
+            updateStrictRec(node.children.get(nextQualifier), tail, expectedKind, newKind, reason);
+        }
+    }
+
+    private static void updateReason(InitializationNode node, String reason) {
+        if (node.reasons.size() < MAX_NUMBER_OF_REASONS) {
+            node.reasons.add(reason);
+        } else if (node.reasons.size() == MAX_NUMBER_OF_REASONS) {
+            node.reasons.add("others");
+        }
+    }
 
     public synchronized void insert(String classOrPackage, InitKind kind, String reason, boolean strict) {
         assert kind != null;
@@ -93,11 +129,7 @@ private void insertRec(InitializationNode node, List<String> classOrPackage, Ini
                 node.strict = strict;
                 node.reasons.add(reason);
             } else if (node.kind == kind) {
-                if (node.reasons.size() < MAX_NUMBER_OF_REASONS) {
-                    node.reasons.add(reason);
-                } else if (node.reasons.size() == MAX_NUMBER_OF_REASONS) {
-                    node.reasons.add("others");
-                }
+                updateReason(node, reason);
             } else {
                 if (node.strict) {
                     throw UserError.abort("Incompatible change of initialization policy for %s: trying to change %s %s to %s %s",

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ClassInitializationSupport.java

@@ -44,18 +44,19 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import jdk.graal.compiler.core.common.ContextClassLoaderScope;
-import org.graalvm.nativeimage.libgraal.hosted.LibGraalLoader;
 import org.graalvm.collections.EconomicSet;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.impl.RuntimeClassInitializationSupport;
 import org.graalvm.nativeimage.impl.clinit.ClassInitializationTracking;
+import org.graalvm.nativeimage.libgraal.hosted.LibGraalLoader;
 
+import com.oracle.graal.pointsto.BigBang;
 import com.oracle.graal.pointsto.infrastructure.OriginalClassProvider;
 import com.oracle.graal.pointsto.meta.AnalysisType;
 import com.oracle.graal.pointsto.meta.BaseLayerType;
 import com.oracle.graal.pointsto.reports.ReportUtils;
 import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
 import com.oracle.svm.core.option.AccumulatingLocatableMultiOptionValue;
 import com.oracle.svm.core.option.SubstrateOptionsParser;
 import com.oracle.svm.core.util.UserError;
@@ -65,6 +66,7 @@
 import com.oracle.svm.util.LogUtils;
 import com.oracle.svm.util.ModuleSupport;
 
+import jdk.graal.compiler.core.common.ContextClassLoaderScope;
 import jdk.graal.compiler.java.LambdaUtils;
 import jdk.internal.misc.Unsafe;
 import jdk.vm.ci.meta.MetaAccessProvider;
@@ -73,6 +75,38 @@
 /**
  * The core class for deciding whether a class should be initialized during image building or class
  * initialization should be delayed to runtime.
+ * <p>
+ * The initialization kind for all classes is encoded in the two registries:
+ * {@link #classInitializationConfiguration}, the user-configured initialization state, and
+ * {@link #classInitKinds}, the actual computed initialization state.
+ * <p>
+ * If for example the configured initialization kind, as registered in
+ * {@link #classInitializationConfiguration}, is {@link InitKind#BUILD_TIME} but invoking
+ * {@link #ensureClassInitialized(Class, boolean)} results in an error, e.g., a
+ * {@link NoClassDefFoundError}, then the actual initialization kind registered in
+ * {@link #classInitKinds} may be {@link InitKind#RUN_TIME} depending on the error resolution policy
+ * dictated by {@link LinkAtBuildTimeSupport#linkAtBuildTime(Class)}.
+ * <p>
+ * Classes with a simulated class initializer are neither registered as initialized at
+ * {@link InitKind#RUN_TIME} nor {@link InitKind#BUILD_TIME}. Instead
+ * {@link SimulateClassInitializerSupport} queries their initialization state to decide if
+ * simulation should be tried. If a class has a computed {@link InitKind#BUILD_TIME} initialization
+ * kind, i.e., its {@link AnalysisType#isInitialized()} returns true, simulation is skipped since
+ * the type is already considered as starting out as initialized at image run time (see
+ * {@link SimulateClassInitializerSupport#trySimulateClassInitializer(BigBang, AnalysisType)}). If a
+ * class was explicitly configured as {@link InitKind#RUN_TIME} initialized this will prevent it
+ * from being simulated.
+ * <p>
+ * There are some similarities and differences between simulated and build-time initialized classes.
+ * At image execution time they both start out as initialized: there are no run-time class
+ * initialization checks and the class initializer itself is not even present, it was not AOT
+ * compiled. However, for a simulated class its initialization status in the hosting VM that runs
+ * the image generator does not matter; it may or may not have been initialized. Whereas, a
+ * build-time initialized class is by definition initialized in the hosting VM. Consequently, the
+ * static fields of a simulated class reference image heap values computed by the class initializer
+ * simulation, but they do not correspond to hosted objects. In contrast, static fields of a
+ * build-time initialized class reference image heap values that were copied from the corresponding
+ * fields in the hosting VM.
  */
 public class ClassInitializationSupport implements RuntimeClassInitializationSupport {
 
@@ -125,7 +159,29 @@ public ClassInitializationSupport(MetaAccessProvider metaAccess, ImageClassLoade
         this.loader = loader;
     }
 
-    public void setConfigurationSealed(boolean sealed) {
+    /**
+     * Seal the configuration, blocking if another thread is trying to seal the configuration or an
+     * unsealed-configuration window is currently open in another thread.
+     */
+    public synchronized void sealConfiguration() {
+        setConfigurationSealed(true);
+    }
+
+    /**
+     * Run the action in an unsealed-configuration window, blocking if another thread is trying to
+     * seal the configuration or an unsealed-configuration window is currently open in another
+     * thread. The window is reentrant, i.e., it will not block the thread that opened the window
+     * from trying to reenter the window. Note that if the configuration was not sealed when the
+     * window was opened this will not affect the seal status.
+     */
+    public synchronized void withUnsealedConfiguration(Runnable action) {
+        var previouslySealed = configurationSealed;
+        setConfigurationSealed(false);
+        action.run();
+        setConfigurationSealed(previouslySealed);
+    }
+
+    private void setConfigurationSealed(boolean sealed) {
         configurationSealed = sealed;
         if (configurationSealed && ClassInitializationOptions.PrintClassInitialization.getValue()) {
             List<ClassOrPackageConfig> allConfigs = classInitializationConfiguration.allConfigs();
@@ -170,7 +226,7 @@ Set<Class<?>> classesWithKind(InitKind kind) {
      */
     public boolean maybeInitializeAtBuildTime(ResolvedJavaType type) {
         if (type instanceof AnalysisType analysisType && analysisType.getWrapped() instanceof BaseLayerType baseLayerType) {
-            return baseLayerType.initializedAtBuildTime();
+            return baseLayerType.isInitialized();
         }
         return maybeInitializeAtBuildTime(OriginalClassProvider.getJavaClass(type));
     }
@@ -350,8 +406,26 @@ public void forceInitializeHosted(Class<?> clazz, String reason, boolean allowIn
         if (clazz == null) {
             return;
         }
+
         classInitializationConfiguration.insert(clazz.getTypeName(), InitKind.BUILD_TIME, reason, true);
         InitKind initKind = ensureClassInitialized(clazz, allowInitializationErrors);
+        if (initKind == InitKind.RUN_TIME) {
+            assert allowInitializationErrors || !LinkAtBuildTimeSupport.singleton().linkAtBuildTime(clazz);
+            if (ImageLayerBuildingSupport.buildingApplicationLayer()) {
+                /*
+                 * Special case for application layer building. If a base layer class was configured
+                 * with --initialize-at-build-time but its initialization failed, then the computed
+                 * init kind will be RUN_TIME, different from its configured init kind of
+                 * BUILD_TIME. In the app layer the computed init kind from the previous layer is
+                 * registered as the configured init kind, but if the --initialize-at-build-time was
+                 * already processed for the class then it will already have a conflicting
+                 * configured init kind of BUILD_TIME. Update the configuration registry to allow
+                 * the RUN_TIME registration coming from the base layer. (GR-65405)
+                 */
+                classInitializationConfiguration.updateStrict(clazz.getTypeName(), InitKind.BUILD_TIME, InitKind.RUN_TIME,
+                                "Allow the registration of the computed run time initialization kind from a previous layer for classes whose build time initialization fails.");
+            }
+        }
         classInitKinds.put(clazz, initKind);
 
         forceInitializeHosted(clazz.getSuperclass(), "super type of " + clazz.getTypeName(), allowInitializationErrors);
@@ -369,7 +443,8 @@ private void forceInitializeInterfaces(Class<?>[] interfaces, String reason) {
             if (metaAccess.lookupJavaType(iface).declaresDefaultMethods()) {
                 classInitializationConfiguration.insert(iface.getTypeName(), InitKind.BUILD_TIME, reason, true);
 
-                ensureClassInitialized(iface, false);
+                InitKind initKind = ensureClassInitialized(iface, false);
+                VMError.guarantee(initKind == InitKind.BUILD_TIME, "Initialization of %s failed so all interfaces with default methods must be already initialized.", iface.getTypeName());
                 classInitKinds.put(iface, InitKind.BUILD_TIME);
             }
             forceInitializeInterfaces(iface.getInterfaces(), "super type of " + iface.getTypeName());