[GR-60468] Introduce a metaspace.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AddressRangeCommittedMemoryProvider.java

@@ -108,6 +108,7 @@ public class AddressRangeCommittedMemoryProvider extends ChunkBasedCommittedMemo
     protected static final int COMMIT_FAILED = 2;
 
     private static final OutOfMemoryError NODE_ALLOCATION_FAILED = new OutOfMemoryError("Could not allocate node for free list, OS may be out of memory.");
+    private static final OutOfMemoryError OUT_OF_METASPACE = new OutOfMemoryError("Could not allocate a metaspace chunk because the metaspace is exhausted.");
     private static final OutOfMemoryError ALIGNED_OUT_OF_ADDRESS_SPACE = new OutOfMemoryError("Could not allocate an aligned heap chunk because the heap address space is exhausted. " +
                     "Consider increasing the address space size (see option -XX:ReservedAddressSpaceSize).");
     private static final OutOfMemoryError UNALIGNED_OUT_OF_ADDRESS_SPACE = new OutOfMemoryError("Could not allocate an unaligned heap chunk because the heap address space is exhausted. " +
@@ -129,7 +130,8 @@ public class AddressRangeCommittedMemoryProvider extends ChunkBasedCommittedMemo
     protected FreeListNode unusedListHead;
     protected long unusedListCount;
 
-    protected UnsignedWord reservedSpaceSize;
+    protected UnsignedWord reservedAddressSpaceSize;
+    protected UnsignedWord reservedMetaspaceSize;
 
     protected Pointer collectedHeapBegin;
     protected UnsignedWord collectedHeapSize;
@@ -212,7 +214,7 @@ private static int initialize(Pointer spaceBegin, UnsignedWord spaceSize, Pointe
     @SuppressWarnings("hiding")
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     protected int initializeFields(Pointer spaceBegin, UnsignedWord reservedSpaceSize, Pointer collectedHeapBegin) {
-        this.reservedSpaceSize = reservedSpaceSize;
+        this.reservedAddressSpaceSize = reservedSpaceSize;
         this.collectedHeapBegin = collectedHeapBegin;
         this.collectedHeapSize = spaceBegin.add(reservedSpaceSize).subtract(collectedHeapBegin);
 
@@ -320,12 +322,38 @@ public int tearDown() {
 
     @Uninterruptible(reason = "Tear-down in progress.")
     protected int unmapAddressSpace(PointerBase heapBase) {
-        if (VirtualMemoryProvider.get().free(heapBase, reservedSpaceSize) != 0) {
+        if (VirtualMemoryProvider.get().free(heapBase, reservedAddressSpaceSize) != 0) {
             return CEntryPointErrors.FREE_ADDRESS_SPACE_FAILED;
         }
         return CEntryPointErrors.NO_ERROR;
     }
 
+    @Override
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public Pointer allocateMetaspaceChunk(UnsignedWord nbytes, UnsignedWord alignment) {
+        WordPointer allocOut = UnsafeStackValue.get(WordPointer.class);
+        int error = allocateInHeapAddressSpace(nbytes, alignment, allocOut);
+        if (error == NO_ERROR) {
+            if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
+                NativeMemoryTracking.singleton().trackCommit(nbytes, NmtCategory.Metaspace);
+            }
+            return allocOut.read();
+        }
+        throw reportMetaspaceChunkAllocationFailed(error);
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    protected OutOfMemoryError reportMetaspaceChunkAllocationFailed(int error) {
+        /* Explicitly don't use OutOfMemoryUtil as the metaspace is not part of the Java heap. */
+        if (error == OUT_OF_ADDRESS_SPACE) {
+            throw OUT_OF_METASPACE;
+        } else if (error == COMMIT_FAILED) {
+            throw METASPACE_CHUNK_COMMIT_FAILED;
+        } else {
+            throw VMError.shouldNotReachHereAtRuntime();
+        }
+    }
+
     @Override
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) {
@@ -698,7 +726,7 @@ protected void mergeNodes(FreeListNode target, FreeListNode obsolete) {
     private void increaseBounds(FreeListNode node, Pointer otherStart, UnsignedWord otherSize) {
         assert getNodeEnd(node).equal(otherStart) || otherStart.add(otherSize).equal(node.getStart()) : "must be adjacent";
         assert UnsignedUtils.isAMultiple(otherSize, getGranularity());
-        assert otherSize.belowOrEqual(reservedSpaceSize);
+        assert otherSize.belowOrEqual(reservedAddressSpaceSize);
 
         Pointer newStart = PointerUtils.min(node.getStart(), otherStart);
         UnsignedWord newSize = node.getSize().add(otherSize);
@@ -707,7 +735,7 @@ private void increaseBounds(FreeListNode node, Pointer otherStart, UnsignedWord
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     protected void trimBounds(FreeListNode fit, Pointer newStart, UnsignedWord newSize) {
-        assert newSize.belowOrEqual(reservedSpaceSize);
+        assert newSize.belowOrEqual(reservedAddressSpaceSize);
         assert fit.getStart().equal(newStart) && newSize.belowThan(fit.getSize()) ||
                         fit.getStart().belowThan(newStart) && getNodeEnd(fit).equal(newStart.add(newSize));
 
@@ -790,7 +818,12 @@ private boolean isInAllocList(FreeListNode node) {
 
     @Override
     public UnsignedWord getReservedAddressSpaceSize() {
-        return reservedSpaceSize;
+        return reservedAddressSpaceSize;
+    }
+
+    @Override
+    public UnsignedWord getReservedMetaspaceSize() {
+        return reservedMetaspaceSize;
     }
 
     /** Keeps track of unused memory. */

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java

@@ -24,6 +24,8 @@
  */
 package com.oracle.svm.core.genscavenge;
 
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
+
 import org.graalvm.nativeimage.c.struct.RawField;
 import org.graalvm.nativeimage.c.struct.RawStructure;
 import org.graalvm.word.Pointer;
@@ -112,19 +114,21 @@ public static boolean isEmpty(AlignedHeader that) {
 
     /** Allocate uninitialized memory within this AlignedHeapChunk. */
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) {
-        Pointer result = Word.nullPointer();
+    public static Pointer tryAllocateMemory(AlignedHeader that, UnsignedWord size) {
         UnsignedWord available = HeapChunk.availableObjectMemory(that);
-        if (size.belowOrEqual(available)) {
-            result = HeapChunk.getTopPointer(that);
-            Pointer newTop = result.add(size);
-            HeapChunk.setTopPointerCarefully(that, newTop);
+        if (size.aboveThan(available)) {
+            return Word.nullPointer();
         }
+
+        Pointer result = HeapChunk.getTopPointer(that);
+        Pointer newTop = result.add(size);
+        HeapChunk.setTopPointerCarefully(that, newTop);
         return result;
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static AlignedHeader getEnclosingChunk(Object obj) {
+        assert ObjectHeaderImpl.isAlignedObject(obj);
         Pointer ptr = Word.objectToUntrackedPointer(obj);
         return getEnclosingChunkFromObjectPointer(ptr);
     }
@@ -144,7 +148,8 @@ public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPoi
         return objectPointer.subtract(objectsStart);
     }
 
-    static void walkObjects(AlignedHeader that, ObjectVisitor visitor) {
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public static void walkObjects(AlignedHeader that, ObjectVisitor visitor) {
         HeapChunk.walkObjectsFrom(that, getObjectsStart(that), visitor);
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java

@@ -45,6 +45,7 @@
 import com.oracle.svm.core.genscavenge.compacting.PlanningVisitor;
 import com.oracle.svm.core.genscavenge.compacting.RuntimeCodeCacheFixupWalker;
 import com.oracle.svm.core.genscavenge.compacting.SweepingVisitor;
+import com.oracle.svm.core.genscavenge.metaspace.MetaspaceImpl;
 import com.oracle.svm.core.genscavenge.remset.BrickTable;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.graal.RuntimeCompilation;
@@ -107,7 +108,7 @@
  */
 final class CompactingOldGeneration extends OldGeneration {
 
-    private final Space space = new Space("Old", "O", false, HeapParameters.getMaxSurvivorSpaces() + 1);
+    private final Space space = new Space("Old", "O", false, getAge());
     private final MarkStack markStack = new MarkStack();
 
     private final GreyObjectsWalker toGreyObjectsWalker = new GreyObjectsWalker();
@@ -141,11 +142,6 @@ void absorb(YoungGeneration youngGen) {
         }
     }
 
-    @Override
-    void appendChunk(AlignedHeapChunk.AlignedHeader hdr) {
-        space.appendAlignedHeapChunk(hdr);
-    }
-
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor, GreyToBlackObjRefVisitor refVisitor) {
@@ -178,7 +174,7 @@ boolean scanGreyObjects(boolean incrementalGc) {
     public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) {
         if (!GCImpl.getGCImpl().isCompleteCollection()) {
             assert originalSpace.isFromSpace();
-            return space.copyAlignedObject(original, originalSpace);
+            return ObjectPromoter.copyAlignedObject(original, originalSpace, space);
         }
         assert originalSpace == space;
         ObjectHeader oh = Heap.getHeap().getObjectHeader();
@@ -195,7 +191,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead
              * change during compaction, so we must add a field to store it, which increases the
              * object's size. The easiest way to handle this is to copy the object.
              */
-            result = space.copyAlignedObject(original, originalSpace);
+            result = ObjectPromoter.copyAlignedObject(original, originalSpace, space);
             assert !ObjectHeaderImpl.hasIdentityHashFromAddressInline(oh.readHeaderFromObject(result));
         }
         ObjectHeaderImpl.setMarked(result);
@@ -209,7 +205,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead
     protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) {
         if (!GCImpl.getGCImpl().isCompleteCollection()) {
             assert originalSpace.isFromSpace();
-            space.promoteUnalignedHeapChunk(originalChunk, originalSpace);
+            ObjectPromoter.promoteUnalignedHeapChunk(originalChunk, originalSpace, space);
             return original;
         }
         assert originalSpace == space;
@@ -226,9 +222,9 @@ protected boolean promotePinnedObject(Object obj, HeapChunk.Header<?> originalCh
         if (!GCImpl.getGCImpl().isCompleteCollection()) {
             assert originalSpace != space && originalSpace.isFromSpace();
             if (isAligned) {
-                space.promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace);
+                ObjectPromoter.promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace, space);
             } else {
-                space.promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace);
+                ObjectPromoter.promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace, space);
             }
             return true;
         }
@@ -309,6 +305,13 @@ private void fixupReferencesBeforeCompaction(ChunkReleaser chunkReleaser, Timers
             oldFixupImageHeapTimer.stop();
         }
 
+        Timer oldFixupMetaspaceTimer = timers.oldFixupMetaspace.start();
+        try {
+            fixupMetaspace();
+        } finally {
+            oldFixupMetaspaceTimer.stop();
+        }
+
         Timer oldFixupThreadLocalsTimer = timers.oldFixupThreadLocals.start();
         try {
             for (IsolateThread isolateThread = VMThreads.firstThread(); isolateThread.isNonNull(); isolateThread = VMThreads.nextThread(isolateThread)) {
@@ -356,6 +359,20 @@ private void fixupImageHeapRoots(ImageHeapInfo info) {
         }
     }
 
+    @Uninterruptible(reason = "Avoid unnecessary safepoint checks in GC for performance.")
+    private void fixupMetaspace() {
+        if (!MetaspaceImpl.isSupported()) {
+            return;
+        }
+
+        if (SerialGCOptions.useRememberedSet()) {
+            /* Cards have been cleaned and roots re-marked during the initial scan. */
+            MetaspaceImpl.singleton().walkDirtyObjects(fixupVisitor, refFixupVisitor, false);
+        } else {
+            MetaspaceImpl.singleton().walkObjects(fixupVisitor);
+        }
+    }
+
     @Uninterruptible(reason = "Avoid unnecessary safepoint checks in GC for performance.")
     private void fixupUnalignedChunkReferences(ChunkReleaser chunkReleaser) {
         UnalignedHeapChunk.UnalignedHeader uChunk = space.getFirstUnalignedHeapChunk();

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java

@@ -52,9 +52,8 @@ final class CopyingOldGeneration extends OldGeneration {
     @Platforms(Platform.HOSTED_ONLY.class)
     CopyingOldGeneration(String name) {
         super(name);
-        int age = HeapParameters.getMaxSurvivorSpaces() + 1;
-        this.fromSpace = new Space("Old", "O", false, age);
-        this.toSpace = new Space("Old To", "O", true, age);
+        this.fromSpace = new Space("Old", "O", false, getAge());
+        this.toSpace = new Space("Old To", "O", true, getAge());
     }
 
     @Override
@@ -76,15 +75,15 @@ public void walkObjects(ObjectVisitor visitor) {
     @Override
     public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) {
         assert originalSpace.isFromSpace();
-        return getToSpace().copyAlignedObject(original, originalSpace);
+        return ObjectPromoter.copyAlignedObject(original, originalSpace, getToSpace());
     }
 
     @AlwaysInline("GC performance")
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     @Override
     protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) {
         assert originalSpace.isFromSpace();
-        getToSpace().promoteUnalignedHeapChunk(originalChunk, originalSpace);
+        ObjectPromoter.promoteUnalignedHeapChunk(originalChunk, originalSpace, getToSpace());
         return original;
     }
 
@@ -93,9 +92,9 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal
     protected boolean promotePinnedObject(Object obj, HeapChunk.Header<?> originalChunk, boolean isAligned, Space originalSpace) {
         assert originalSpace.isFromSpace();
         if (isAligned) {
-            getToSpace().promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace);
+            ObjectPromoter.promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace, getToSpace());
         } else {
-            getToSpace().promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace);
+            ObjectPromoter.promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace, getToSpace());
         }
         return true;
     }
@@ -146,11 +145,6 @@ Space getToSpace() {
         return toSpace;
     }
 
-    @Override
-    void appendChunk(AlignedHeapChunk.AlignedHeader hdr) {
-        getToSpace().appendAlignedHeapChunk(hdr);
-    }
-
     @Override
     void swapSpaces() {
         assert getFromSpace().isEmpty() : "fromSpace should be empty.";