Make the inner-most part of the GC uninterruptible.

Author: christianhaeubl

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java

@@ -243,6 +243,7 @@ public UnsignedWord getCurrentHeapCapacity() {
     }
 
     @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public UnsignedWord getSurvivorSpacesCapacity() {
         assert VMOperation.isGCInProgress() : "use only during GC";
         guaranteeSizeParametersInitialized();
@@ -290,6 +291,7 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() {
     }
 
     @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public int getTenuringAge() {
         assert VMOperation.isGCInProgress() : "use only during GC";
         return tenuringThreshold;

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java

@@ -83,6 +83,7 @@ private AlignedHeapChunk() { // all static
     public interface AlignedHeader extends HeapChunk.Header<AlignedHeader> {
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) {
         HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), chunkSize);
     }
@@ -101,6 +102,7 @@ public static Pointer getObjectsEnd(AlignedHeader that) {
     }
 
     /** Allocate uninitialized memory within this AlignedHeapChunk. */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) {
         Pointer result = WordFactory.nullPointer();
         UnsignedWord available = HeapChunk.availableObjectMemory(that);
@@ -113,6 +115,7 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) {
     }
 
     /** Retract the latest allocation. Used by parallel collector. */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) {
         Pointer newTop = HeapChunk.getTopPointer(that).subtract(size);
         assert newTop.aboveThan(HeapChunk.asPointer(that));
@@ -137,6 +140,7 @@ public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) {
     }
 
     /** Return the offset of an object within the objects part of a chunk. */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPointer) {
         Pointer objectsStart = getObjectsStart(that);
         return objectPointer.subtract(objectsStart);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java

@@ -32,6 +32,7 @@
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.SubstrateGCOptions;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.GCCause;
 import com.oracle.svm.core.heap.PhysicalMemory;
 import com.oracle.svm.core.heap.ReferenceAccess;
@@ -167,6 +168,7 @@ public UnsignedWord getMaximumSurvivorSize() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public UnsignedWord getSurvivorSpacesCapacity() {
             return WordFactory.zero();
         }
@@ -202,6 +204,7 @@ public final UnsignedWord getMaximumFreeAlignedChunksSize() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public int getTenuringAge() {
             return 1;
         }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java

@@ -60,6 +60,7 @@ public void reset() {
         unalignedChunkBytes = WordFactory.zero();
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public UnsignedWord getChunkBytes() {
         return getAlignedChunkBytes().add(getUnalignedChunkBytes());
     }
@@ -117,10 +118,12 @@ private void noteUnaligned(UnsignedWord size) {
         }
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void unnoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) {
         unnoteUnaligned(UnalignedHeapChunk.getCommittedObjectMemory(chunk));
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private void unnoteUnaligned(UnsignedWord size) {
         unalignedCount--;
         unalignedChunkBytes = unalignedChunkBytes.subtract(size);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java

@@ -30,6 +30,7 @@
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.GCCause;
 import com.oracle.svm.core.heap.PhysicalMemory;
 import com.oracle.svm.core.util.UserError;
@@ -176,6 +177,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) {
      * survivor-to spaces of all ages. In other words, when copying during a collection, up to 2x
      * this amount can be used for surviving objects.
      */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     UnsignedWord getSurvivorSpacesCapacity();
 
     /** The capacity of the young generation, comprising the eden and survivor spaces. */
@@ -200,6 +202,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) {
      * 1 (straight from eden) and the {@linkplain HeapParameters#getMaxSurvivorSpaces() number of
      * survivor spaces + 1}.
      */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     int getTenuringAge();
 
     /** Called at the beginning of a collection, in the safepoint operation. */

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java

@@ -30,6 +30,7 @@
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.AlwaysInline;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.log.Log;
 
 /**
@@ -154,6 +155,7 @@ void beforeCollection(boolean completeCollection) {
 
     /** Called after an object has been promoted from the young generation to the old generation. */
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void onSurvivorOverflowed() {
         lastIncrementalCollectionOverflowedSurvivors = true;
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -535,6 +535,7 @@ public void collectCompletely(GCCause cause) {
         collect(cause, true);
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean isCompleteCollection() {
         return completeCollection;
     }
@@ -1085,6 +1086,7 @@ private static void scanGreyObjectsLoop() {
     }
 
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     @SuppressWarnings("static-method")
     Object promoteObject(Object original, UnsignedWord header) {
         HeapImpl heap = HeapImpl.getHeapImpl();
@@ -1118,6 +1120,7 @@ Object promoteObject(Object original, UnsignedWord header) {
         return result;
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static Header<?> getChunk(Object obj, boolean isAligned) {
         if (isAligned) {
             return AlignedHeapChunk.getEnclosingChunk(obj);
@@ -1234,6 +1237,7 @@ public static boolean hasNeverCollectPolicy() {
         return getPolicy() instanceof NeverCollect;
     }
 
+    @Fold
     public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() {
         return greyToBlackObjectVisitor;
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java

@@ -27,6 +27,8 @@
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
+import com.oracle.svm.core.AlwaysInline;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.log.Log;
 
@@ -66,6 +68,8 @@ public String getName() {
      *         promotion was done by copying, or {@code null} if there was insufficient capacity in
      *         this generation.
      */
+    @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace);
 
     /**
@@ -79,6 +83,8 @@ public String getName() {
      *         was promoted through HeapChunk motion, or {@code null} if there was insufficient
      *         capacity in this generation.
      */
+    @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace);
 
     /**

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java

@@ -30,6 +30,7 @@
 import org.graalvm.word.Pointer;
 
 import com.oracle.svm.core.AlwaysInline;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.heap.ReferenceAccess;
@@ -57,12 +58,14 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor {
     }
 
     @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) {
         return visitObjectReferenceInline(objRef, 0, compressed, holderObject);
     }
 
     @Override
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) {
         assert innerOffset >= 0;
         assert !objRef.isNull();
@@ -128,16 +131,22 @@ public interface Counters extends AutoCloseable {
         @Override
         void close();
 
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         void noteObjRef();
 
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         void noteNullReferent();
 
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         void noteForwardedReferent();
 
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         void noteNonHeapReferent();
 
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         void noteCopiedReferent();
 
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         void noteUnmodifiedReference();
 
         void toLog();
@@ -182,32 +191,39 @@ public void close() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteObjRef() {
             objRef += 1L;
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteNullReferent() {
             nullReferent += 1L;
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteForwardedReferent() {
             forwardedReferent += 1L;
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteNonHeapReferent() {
             nonHeapReferent += 1L;
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteCopiedReferent() {
             copiedReferent += 1L;
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteUnmodifiedReference() {
+            // TEMP (chaeubl): this counter would break
             unmodifiedReference += 1L;
         }
 
@@ -241,26 +257,32 @@ public void close() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteObjRef() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteNullReferent() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteForwardedReferent() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteNonHeapReferent() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteCopiedReferent() {
         }
 
         @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteUnmodifiedReference() {
         }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java

@@ -29,6 +29,7 @@
 
 import com.oracle.svm.core.AlwaysInline;
 import com.oracle.svm.core.NeverInline;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.hub.InteriorObjRefWalker;
 import com.oracle.svm.core.util.VMError;
@@ -55,6 +56,7 @@ public boolean visitObject(Object o) {
 
     @Override
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean visitObjectInline(Object o) {
         ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor);
         InteriorObjRefWalker.walkObjectInline(o, objRefVisitor);