[GR-43620] Improve OutOfMemoryError handling.

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ExitStatus.java

@@ -29,7 +29,11 @@ public enum ExitStatus {
     OK(0),
     BUILDER_ERROR(1),
     FALLBACK_IMAGE(2),
-    BUILDER_INTERRUPT_WITHOUT_REASON(3),
+
+    // 3 used by `-XX:+ExitOnOutOfMemoryError` (see src/hotspot/share/utilities/debug.cpp)
+    OUT_OF_MEMORY(3),
+
+    BUILDER_INTERRUPT_WITHOUT_REASON(4),
     DRIVER_ERROR(20),
     DRIVER_TO_BUILDER_ERROR(21),
     WATCHDOG_EXIT(30),

substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -804,7 +804,11 @@ private void prepareImageBuildArgs() {
         if (!"0".equals(xmxVal)) {
             addImageBuilderJavaArgs(oXmx + xmxVal);
         }
-        /* Prevent JVM that runs the image builder to steal focus */
+
+        /* Let builder exit on first OutOfMemoryError. */
+        addImageBuilderJavaArgs("-XX:+ExitOnOutOfMemoryError");
+
+        /* Prevent JVM that runs the image builder to steal focus. */
         addImageBuilderJavaArgs("-Djava.awt.headless=true");
         addImageBuilderJavaArgs("-Dorg.graalvm.version=" + graalvmVersion);
         addImageBuilderJavaArgs("-Dorg.graalvm.vendor=" + graalvmVendor);
@@ -1515,7 +1519,7 @@ protected static void build(BuildConfiguration config, Function<BuildConfigurati
                         break;
                     case BUILDER_ERROR:
                         /* Exit, builder has handled error reporting. */
-                        System.exit(ExitStatus.BUILDER_ERROR.getValue());
+                        System.exit(exitStatusCode);
                         break;
                     case FALLBACK_IMAGE:
                         nativeImage.showMessage("Generating fallback image...");
@@ -1525,6 +1529,10 @@ protected static void build(BuildConfiguration config, Function<BuildConfigurati
                                         "(use --" + SubstrateOptions.OptionNameNoFallback +
                                         " to suppress fallback image generation and to print more detailed information why a fallback image was necessary).");
                         break;
+                    case OUT_OF_MEMORY:
+                        nativeImage.showOutOfMemoryWarning();
+                        System.exit(exitStatusCode);
+                        break;
                     default:
                         String message = String.format("Image build request for '%s' (pid: %d, path: %s) failed with exit status %d",
                                         nativeImage.imageName, nativeImage.imageBuilderPid, nativeImage.imagePath, exitStatusCode);
@@ -1838,6 +1846,19 @@ void showMessagePart(String message) {
         }, message);
     }
 
+    void showOutOfMemoryWarning() {
+        String lastMaxHeapValue = null;
+        for (String arg : imageBuilderJavaArgs) {
+            if (arg.startsWith(oXmx)) {
+                lastMaxHeapValue = arg.substring(oXmx.length());
+            }
+        }
+        String maxHeapText = lastMaxHeapValue == null ? "" : " (The maximum heap size of the process was set to '" + lastMaxHeapValue + "'.)";
+        String additionalAction = lastMaxHeapValue == null ? "" : " or increase the maximum heap size using the '" + oXmx + "' option";
+        showMessage(String.format("The Native Image build process ran out of memory.%s%nPlease make sure your build system has more memory available%s.",
+                        maxHeapText, additionalAction));
+    }
+
     public static void showWarning(String message) {
         show(System.err::println, "Warning: " + message);
     }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/EarlyClassInitializerAnalysis.java

@@ -51,12 +51,17 @@
 import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
 import org.graalvm.compiler.nodes.java.AccessFieldNode;
+import org.graalvm.compiler.nodes.java.NewArrayNode;
+import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.OptimisticOptimizations;
 import org.graalvm.compiler.phases.tiers.HighTierContext;
 import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
+import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.impl.RuntimeClassInitializationSupport;
 
+import com.oracle.graal.pointsto.infrastructure.OriginalClassProvider;
 import com.oracle.graal.pointsto.phases.NoClassInitializationPlugin;
 import com.oracle.graal.pointsto.util.GraalAccess;
 import com.oracle.svm.core.ParsingReason;
@@ -68,6 +73,7 @@
 import com.oracle.svm.hosted.snippets.ReflectionPlugins;
 import com.oracle.svm.hosted.snippets.SubstrateGraphBuilderPlugins;
 
+import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.ResolvedJavaField;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
@@ -276,6 +282,31 @@ public void nodeAdded(Node node) {
             throw new ClassInitializerHasSideEffectsException("Access of thread-local value");
         } else if (node instanceof UnsafeAccessNode) {
             throw VMError.shouldNotReachHere("Intrinsification of Unsafe methods is not enabled during bytecode parsing");
+
+        } else if (node instanceof NewArrayNode) {
+            checkArrayAllocationLength(((NewArrayNode) node).length());
+        } else if (node instanceof NewMultiArrayNode) {
+            var dimensions = ((NewMultiArrayNode) node).dimensions();
+            for (var dimension : dimensions) {
+                checkArrayAllocationLength(dimension);
+            }
+        }
+    }
+
+    private static void checkArrayAllocationLength(ValueNode lengthNode) {
+        JavaConstant lengthConstant = lengthNode.asJavaConstant();
+        if (lengthConstant != null) {
+            int length = lengthConstant.asInt();
+            if (length < 0 || length > 100_000) {
+                /*
+                 * Ensure that also the late class initialization after static analysis does not
+                 * attempt to initialize.
+                 */
+                Class<?> clazz = OriginalClassProvider.getJavaClass(lengthNode.graph().method().getDeclaringClass());
+                ((ProvenSafeClassInitializationSupport) ImageSingletons.lookup(RuntimeClassInitializationSupport.class)).mustNotBeProvenSafe.add(clazz);
+
+                throw new ClassInitializerHasSideEffectsException("Allocation of too large array in class initializer");
+            }
         }
     }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ProvenSafeClassInitializationSupport.java

@@ -64,6 +64,7 @@ class ProvenSafeClassInitializationSupport extends ClassInitializationSupport {
     private final EarlyClassInitializerAnalysis earlyClassInitializerAnalysis;
     private final Set<Class<?>> provenSafeEarly = ConcurrentHashMap.newKeySet();
     private Set<Class<?>> provenSafeLate = ConcurrentHashMap.newKeySet();
+    final Set<Class<?>> mustNotBeProvenSafe = ConcurrentHashMap.newKeySet();
 
     ProvenSafeClassInitializationSupport(MetaAccessProvider metaAccess, ImageClassLoader loader) {
         super(metaAccess, loader);
@@ -76,6 +77,9 @@ InitKind computeInitKindAndMaybeInitializeClass(Class<?> clazz) {
     }
 
     boolean canBeProvenSafe(Class<?> clazz) {
+        if (mustNotBeProvenSafe.contains(clazz)) {
+            return false;
+        }
         InitKind initKind = specifiedInitKindFor(clazz);
         return initKind == null || (initKind.isRunTime() && !isStrictlyDefined(clazz));
     }

substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/clinit/TestClassInitializationMustBeSafeEarly.java

@@ -447,6 +447,26 @@ private static DevirtualizedCallSuperMustBeSafeEarly createProvider() {
     }
 }
 
+class LargeAllocation1MustBeDelayed {
+    static final Object value = computeValue();
+
+    private static Object computeValue() {
+        Object[] result = new Object[1_000_000];
+        for (int i = 0; i < result.length; i++) {
+            result[i] = new int[1_000_000];
+        }
+        return result;
+    }
+}
+
+class LargeAllocation2MustBeDelayed {
+    static final Object value = computeValue();
+
+    private static Object computeValue() {
+        return new int[Integer.MAX_VALUE][Integer.MAX_VALUE];
+    }
+}
+
 class TestClassInitializationMustBeSafeEarlyFeature implements Feature {
 
     static final Class<?>[] checkedClasses = new Class<?>[]{
@@ -479,7 +499,8 @@ class TestClassInitializationMustBeSafeEarlyFeature implements Feature {
                     ReferencesOtherPureClassMustBeSafeEarly.class, HelperClassMustBeSafeEarly.class,
                     CycleMustBeSafeLate.class, HelperClassMustBeSafeLate.class,
                     ReflectionMustBeSafeEarly.class, ForNameMustBeSafeEarly.class, ForNameMustBeDelayed.class,
-                    DevirtualizedCallMustBeDelayed.class, DevirtualizedCallSuperMustBeSafeEarly.class, DevirtualizedCallSubMustBeSafeEarly.class, DevirtualizedCallUsageMustBeDelayed.class
+                    DevirtualizedCallMustBeDelayed.class, DevirtualizedCallSuperMustBeSafeEarly.class, DevirtualizedCallSubMustBeSafeEarly.class, DevirtualizedCallUsageMustBeDelayed.class,
+                    LargeAllocation1MustBeDelayed.class, LargeAllocation2MustBeDelayed.class,
     };
 
     private static void checkClasses(boolean checkSafeEarly, boolean checkSafeLate) {
@@ -660,6 +681,15 @@ public static void main(String[] args) {
         assertSame("field", ReflectionMustBeSafeEarly.f2.getName());
 
         System.out.println(DevirtualizedCallUsageMustBeDelayed.value);
+
+        if (System.currentTimeMillis() == 0) {
+            /*
+             * Make the class initializers reachable at run time, but do not actually execute them
+             * because they will allocate a lot of memory before throwing an OutOfMemoryError.
+             */
+            System.out.println(LargeAllocation1MustBeDelayed.value);
+            System.out.println(LargeAllocation2MustBeDelayed.value);
+        }
     }
 
     private static void assertSame(Object expected, Object actual) {