[WIP] [GR-44787] Parallelize full garbage collections.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java

@@ -243,6 +243,7 @@ public UnsignedWord getCurrentHeapCapacity() {
     }
 
     @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public UnsignedWord getSurvivorSpacesCapacity() {
         assert VMOperation.isGCInProgress() : "use only during GC";
         guaranteeSizeParametersInitialized();
@@ -290,6 +291,7 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() {
     }
 
     @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public int getTenuringAge() {
         assert VMOperation.isGCInProgress() : "use only during GC";
         return tenuringThreshold;

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java

@@ -83,6 +83,7 @@ private AlignedHeapChunk() { // all static
     public interface AlignedHeader extends HeapChunk.Header<AlignedHeader> {
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) {
         HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), chunkSize);
     }
@@ -101,6 +102,7 @@ public static Pointer getObjectsEnd(AlignedHeader that) {
     }
 
     /** Allocate uninitialized memory within this AlignedHeapChunk. */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) {
         Pointer result = WordFactory.nullPointer();
         UnsignedWord available = HeapChunk.availableObjectMemory(that);
@@ -112,6 +114,15 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) {
         return result;
     }
 
+    /** Retract the latest allocation. Used by parallel collector. */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) {
+        Pointer newTop = HeapChunk.getTopPointer(that).subtract(size);
+        assert newTop.aboveOrEqual(getObjectsStart(that));
+        HeapChunk.setTopPointer(that, newTop);
+        return newTop;
+    }
+
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static UnsignedWord getCommittedObjectMemory(AlignedHeader that) {
         return HeapChunk.getEndOffset(that).subtract(getObjectsStartOffset());
@@ -129,6 +140,7 @@ public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) {
     }
 
     /** Return the offset of an object within the objects part of a chunk. */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPointer) {
         Pointer objectsStart = getObjectsStart(that);
         return objectPointer.subtract(objectsStart);
@@ -139,7 +151,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) {
     }
 
     @AlwaysInline("GC performance")
-    static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) {
+    public static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) {
         return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor);
     }
 
@@ -154,7 +166,7 @@ static MemoryWalker.HeapChunkAccess<AlignedHeapChunk.AlignedHeader> getMemoryWal
     }
 
     /** Methods for a {@link MemoryWalker} to access an aligned heap chunk. */
-    @AutomaticallyRegisteredImageSingleton(onlyWith = UseSerialOrEpsilonGC.class)
+    @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class)
     static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl<AlignedHeapChunk.AlignedHeader> {
 
         @Platforms(Platform.HOSTED_ONLY.class)

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java

@@ -32,6 +32,7 @@
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.SubstrateGCOptions;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.GCCause;
 import com.oracle.svm.core.heap.PhysicalMemory;
 import com.oracle.svm.core.heap.ReferenceAccess;
@@ -167,6 +168,7 @@ public UnsignedWord getMaximumSurvivorSize() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public UnsignedWord getSurvivorSpacesCapacity() {
             return WordFactory.zero();
         }
@@ -202,6 +204,7 @@ public final UnsignedWord getMaximumFreeAlignedChunksSize() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public int getTenuringAge() {
             return 1;
         }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java

@@ -60,6 +60,7 @@ public void reset() {
         unalignedChunkBytes = WordFactory.zero();
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public UnsignedWord getChunkBytes() {
         return getAlignedChunkBytes().add(getUnalignedChunkBytes());
     }
@@ -96,6 +97,7 @@ void noteAlignedHeapChunk() {
         }
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void unnoteAlignedHeapChunk() {
         alignedCount--;
         if (parent != null) {
@@ -117,10 +119,12 @@ private void noteUnaligned(UnsignedWord size) {
         }
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void unnoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) {
         unnoteUnaligned(UnalignedHeapChunk.getCommittedObjectMemory(chunk));
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private void unnoteUnaligned(UnsignedWord size) {
         unalignedCount--;
         unalignedChunkBytes = unalignedChunkBytes.subtract(size);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java

@@ -30,6 +30,7 @@
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.GCCause;
 import com.oracle.svm.core.heap.PhysicalMemory;
 import com.oracle.svm.core.util.UserError;
@@ -176,6 +177,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) {
      * survivor-to spaces of all ages. In other words, when copying during a collection, up to 2x
      * this amount can be used for surviving objects.
      */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     UnsignedWord getSurvivorSpacesCapacity();
 
     /** The capacity of the young generation, comprising the eden and survivor spaces. */
@@ -200,6 +202,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) {
      * 1 (straight from eden) and the {@linkplain HeapParameters#getMaxSurvivorSpaces() number of
      * survivor spaces + 1}.
      */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     int getTenuringAge();
 
     /** Called at the beginning of a collection, in the safepoint operation. */

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java

@@ -30,6 +30,7 @@
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.AlwaysInline;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.log.Log;
 
 /**
@@ -154,6 +155,7 @@ void beforeCollection(boolean completeCollection) {
 
     /** Called after an object has been promoted from the young generation to the old generation. */
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void onSurvivorOverflowed() {
         lastIncrementalCollectionOverflowedSurvivors = true;
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -64,6 +64,7 @@
 import com.oracle.svm.core.genscavenge.BasicCollectionPolicies.NeverCollect;
 import com.oracle.svm.core.genscavenge.HeapChunk.Header;
 import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
+import com.oracle.svm.core.genscavenge.parallel.ParallelGC;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.graal.RuntimeCompilation;
 import com.oracle.svm.core.heap.CodeReferenceMapDecoder;
@@ -128,6 +129,8 @@ public final class GCImpl implements GC {
     public String getName() {
         if (SubstrateOptions.UseEpsilonGC.getValue()) {
             return "Epsilon GC";
+        } else if (SubstrateOptions.UseParallelGC.getValue()) {
+            return "Parallel GC";
         } else {
             return "Serial GC";
         }
@@ -198,6 +201,10 @@ private void collectOperation(CollectionVMOperationData data) {
         assert VMOperation.isGCInProgress() : "Collection should be a VMOperation.";
         assert getCollectionEpoch().equal(data.getRequestingEpoch());
 
+        if (SubstrateOptions.UseParallelGC.getValue()) {
+            ParallelGC.singleton().initialize();
+        }
+
         timers.mutator.closeAt(data.getRequestingNanoTime());
         startCollectionOrExit();
 
@@ -525,6 +532,7 @@ public void collectCompletely(GCCause cause) {
         collect(cause, true);
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean isCompleteCollection() {
         return completeCollection;
     }
@@ -1052,6 +1060,8 @@ private void scanGreyObjects(boolean isIncremental) {
         try {
             if (isIncremental) {
                 scanGreyObjectsLoop();
+            } else if (ParallelGC.isEnabled()) {
+                ParallelGC.singleton().waitForIdle();
             } else {
                 HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects();
             }
@@ -1072,6 +1082,7 @@ private static void scanGreyObjectsLoop() {
     }
 
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     @SuppressWarnings("static-method")
     Object promoteObject(Object original, UnsignedWord header) {
         HeapImpl heap = HeapImpl.getHeapImpl();
@@ -1105,6 +1116,7 @@ Object promoteObject(Object original, UnsignedWord header) {
         return result;
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static Header<?> getChunk(Object obj, boolean isAligned) {
         if (isAligned) {
             return AlignedHeapChunk.getEnclosingChunk(obj);
@@ -1217,7 +1229,8 @@ public static boolean hasNeverCollectPolicy() {
         return getPolicy() instanceof NeverCollect;
     }
 
-    GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() {
+    @Fold
+    public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() {
         return greyToBlackObjectVisitor;
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java

@@ -46,7 +46,7 @@ public class GenScavengeMemoryPoolMXBeans {
 
     @Platforms(Platform.HOSTED_ONLY.class)
     public static MemoryPoolMXBean[] createMemoryPoolMXBeans() {
-        if (SubstrateOptions.UseSerialGC.getValue()) {
+        if (SubstrateOptions.useMarkAndCopyGC()) {
             mxBeans = new AbstractMemoryPoolMXBean[]{
                             new EdenMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER),
                             new SurvivorMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER),

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java

@@ -27,6 +27,8 @@
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
+import com.oracle.svm.core.AlwaysInline;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.log.Log;
 
@@ -66,6 +68,8 @@ public String getName() {
      *         promotion was done by copying, or {@code null} if there was insufficient capacity in
      *         this generation.
      */
+    @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace);
 
     /**
@@ -79,6 +83,8 @@ public String getName() {
      *         was promoted through HeapChunk motion, or {@code null} if there was insufficient
      *         capacity in this generation.
      */
+    @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace);
 
     /**