[GR-60085] [GR-68380] [GR-53964] [GR-68546] Use a contiguous address space for the metaspace and support heap dumping.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AddressRangeCommittedMemoryProvider.java

@@ -61,6 +61,7 @@
 import com.oracle.svm.core.locks.VMMutex;
 import com.oracle.svm.core.log.Log;
 import com.oracle.svm.core.memory.NullableNativeMemory;
+import com.oracle.svm.core.metaspace.Metaspace;
 import com.oracle.svm.core.nmt.NativeMemoryTracking;
 import com.oracle.svm.core.nmt.NmtCategory;
 import com.oracle.svm.core.os.ChunkBasedCommittedMemoryProvider;
@@ -77,7 +78,8 @@
 
 /**
  * Reserves a fixed-size address range and provides memory from it by committing and uncommitting
- * virtual memory within that range.
+ * virtual memory within that range. The address space is shared by the null regions, the
+ * {@link Metaspace}, the image heap, and the collected Java heap.
  * <p>
  * The main objective of this code is to keep external fragmentation low so that an
  * {@linkplain Isolate} is unlikely to run out of memory because its address space is exhausted. To
@@ -122,59 +124,65 @@ public class AddressRangeCommittedMemoryProvider extends ChunkBasedCommittedMemo
      */
     private final VMMutex lock = new VMMutex("freeList");
 
-    /** Contains free blocks that are large enough to fit allocations. */
+    protected UnsignedWord reservedAddressSpaceSize;
+    private Pointer metaspaceBegin;
+    private Pointer metaspaceTop;
+    private Pointer metaspaceEnd;
+    protected Pointer collectedHeapBegin;
+    protected UnsignedWord collectedHeapSize;
+
+    /**
+     * Contains free blocks for the collected Java heap that are large enough to fit allocations.
+     */
     protected FreeListNode allocListHead;
     protected long allocListCount;
 
-    /** Contains all free blocks, including small blocks that are needed for coalescing. */
+    /**
+     * Contains all free blocks for the collected Java heap, including small blocks that are needed
+     * for coalescing.
+     */
     protected FreeListNode unusedListHead;
     protected long unusedListCount;
 
-    protected UnsignedWord reservedAddressSpaceSize;
-    protected UnsignedWord reservedMetaspaceSize;
-
-    protected Pointer collectedHeapBegin;
-    protected UnsignedWord collectedHeapSize;
-
     @Platforms(Platform.HOSTED_ONLY.class)
     public AddressRangeCommittedMemoryProvider() {
         assert SubstrateOptions.SpawnIsolates.getValue();
     }
 
     @Override
     @Uninterruptible(reason = "Still being initialized.")
-    public int initialize(WordPointer heapBasePointer, IsolateArguments arguments) {
-        UnsignedWord reserved = Word.unsigned(IsolateArgumentAccess.readLong(arguments, IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.ReservedAddressSpaceSize)));
-        if (reserved.equal(0)) {
+    public int initialize(WordPointer heapBaseOut, IsolateArguments arguments) {
+        UnsignedWord reservedSize = Word.unsigned(IsolateArgumentAccess.readLong(arguments, IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.ReservedAddressSpaceSize)));
+        if (reservedSize.equal(0)) {
             /*
              * Reserve a 32 GB address space, except if a larger heap size was specified, or if the
              * maximum address space size is less than that.
              */
             UnsignedWord maxHeapSize = Word.unsigned(IsolateArgumentAccess.readLong(arguments, IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.MaxHeapSize)));
-            reserved = UnsignedUtils.max(maxHeapSize, Word.unsigned(MIN_RESERVED_ADDRESS_SPACE_SIZE));
+            reservedSize = UnsignedUtils.max(maxHeapSize, Word.unsigned(MIN_RESERVED_ADDRESS_SPACE_SIZE));
         }
-        reserved = UnsignedUtils.min(reserved, ReferenceAccess.singleton().getMaxAddressSpaceSize());
+        reservedSize = UnsignedUtils.min(reservedSize, ReferenceAccess.singleton().getMaxAddressSpaceSize());
 
-        UnsignedWord alignment = unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment());
-        WordPointer beginOut = StackValue.get(WordPointer.class);
-        int errorCode = reserveHeapMemory(reserved, alignment, arguments, beginOut);
+        UnsignedWord alignment = unsigned(Heap.getHeap().getHeapBaseAlignment());
+        WordPointer reservedBeginPtr = StackValue.get(WordPointer.class);
+        int errorCode = reserveHeapMemory(reservedSize, alignment, arguments, reservedBeginPtr);
         if (errorCode != CEntryPointErrors.NO_ERROR) {
             return errorCode;
         }
 
-        Pointer begin = beginOut.read();
+        Pointer reservedBegin = reservedBeginPtr.read();
         WordPointer imageHeapEndOut = StackValue.get(WordPointer.class);
-        errorCode = ImageHeapProvider.get().initialize(begin, reserved, heapBasePointer, imageHeapEndOut);
+        errorCode = ImageHeapProvider.get().initialize(reservedBegin, reservedSize, heapBaseOut, imageHeapEndOut);
         if (errorCode != CEntryPointErrors.NO_ERROR) {
-            freeOnInitializeError(begin, reserved);
+            freeOnInitializeError(reservedBegin, reservedSize);
             return errorCode;
         }
 
-        CEntryPointSnippets.initBaseRegisters(heapBasePointer.read());
+        CEntryPointSnippets.initBaseRegisters(heapBaseOut.read());
         WordPointer runtimeHeapBeginOut = StackValue.get(WordPointer.class);
-        errorCode = getCollectedHeapBegin(arguments, begin, reserved, imageHeapEndOut.read(), runtimeHeapBeginOut);
+        errorCode = initializeCollectedHeapBegin(arguments, reservedBegin, reservedSize, imageHeapEndOut.read(), runtimeHeapBeginOut);
         if (errorCode != CEntryPointErrors.NO_ERROR) {
-            freeOnInitializeError(begin, reserved);
+            freeOnInitializeError(reservedBegin, reservedSize);
             return errorCode;
         }
 
@@ -183,40 +191,58 @@ public int initialize(WordPointer heapBasePointer, IsolateArguments arguments) {
          * because the image heap was not initialized when we were called, so we invoke a static
          * method that loads a new reference to our instance.
          */
-        errorCode = initialize(begin, reserved, runtimeHeapBeginOut.read());
+        errorCode = initialize(reservedBegin, reservedSize, runtimeHeapBeginOut.read());
         if (errorCode != CEntryPointErrors.NO_ERROR) {
-            freeOnInitializeError(begin, reserved);
+            freeOnInitializeError(reservedBegin, reservedSize);
         }
         return errorCode;
     }
 
     @Uninterruptible(reason = "Still being initialized.")
-    protected int getCollectedHeapBegin(@SuppressWarnings("unused") IsolateArguments arguments, @SuppressWarnings("unused") Pointer begin, @SuppressWarnings("unused") UnsignedWord reserved,
-                    Pointer imageHeapEnd, WordPointer collectedHeapBeginOut) {
-        Pointer result = roundUp(imageHeapEnd, getGranularity());
-        collectedHeapBeginOut.write(result);
+    protected int initializeCollectedHeapBegin(@SuppressWarnings("unused") IsolateArguments arguments, @SuppressWarnings("unused") Pointer reservedBegin,
+                    @SuppressWarnings("unused") UnsignedWord reservedSize, Pointer imageHeapEnd, WordPointer collectedHeapBeginOut) {
+        assert PointerUtils.isAMultiple(imageHeapEnd, Word.unsigned(SubstrateOptions.getPageSize()));
+        collectedHeapBeginOut.write(imageHeapEnd);
         return CEntryPointErrors.NO_ERROR;
     }
 
     @NeverInline("Ensure a newly looked up value is used as 'this', now that the image heap is initialized")
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE)
-    private static int initialize(Pointer spaceBegin, UnsignedWord spaceSize, Pointer collectedHeapBegin) {
-        if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
-            UnsignedWord imageHeapAddressSpace = ImageHeapProvider.get().getImageHeapAddressSpaceSize();
-            UnsignedWord javaHeapAddressSpace = spaceSize.subtract(imageHeapAddressSpace);
-            NativeMemoryTracking.singleton().trackReserve(javaHeapAddressSpace, NmtCategory.JavaHeap);
-        }
-
+    private static int initialize(Pointer reservedBegin, UnsignedWord reservedSize, Pointer collectedHeapBegin) {
         AddressRangeCommittedMemoryProvider provider = (AddressRangeCommittedMemoryProvider) ChunkBasedCommittedMemoryProvider.get();
-        return provider.initializeFields(spaceBegin, spaceSize, collectedHeapBegin);
+        return provider.initializeFields(reservedBegin, reservedSize, collectedHeapBegin);
     }
 
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     @SuppressWarnings("hiding")
+    protected int initializeFields(Pointer reservedBegin, UnsignedWord reservedSize, Pointer collectedHeapBegin) {
+        this.reservedAddressSpaceSize = reservedSize;
+
+        initializeMetaspaceFields();
+        return initializeCollectedHeapFields(reservedBegin, reservedSize, collectedHeapBegin);
+    }
+
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    protected int initializeFields(Pointer spaceBegin, UnsignedWord reservedSpaceSize, Pointer collectedHeapBegin) {
-        this.reservedAddressSpaceSize = reservedSpaceSize;
+    protected void initializeMetaspaceFields() {
+        int metaspaceSize = SerialAndEpsilonGCOptions.getReservedMetaspaceSize();
+        this.metaspaceBegin = KnownIntrinsics.heapBase().add(HeapImpl.getMetaspaceOffsetInAddressSpace());
+        this.metaspaceTop = metaspaceBegin;
+        this.metaspaceEnd = metaspaceTop.add(metaspaceSize);
+
+        if (VMInspectionOptions.hasNativeMemoryTrackingSupport() && metaspaceSize > 0) {
+            NativeMemoryTracking.singleton().trackReserve(metaspaceSize, NmtCategory.Metaspace);
+        }
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    @SuppressWarnings("hiding")
+    private int initializeCollectedHeapFields(Pointer reservedBegin, UnsignedWord reservedSize, Pointer collectedHeapBegin) {
         this.collectedHeapBegin = collectedHeapBegin;
-        this.collectedHeapSize = spaceBegin.add(reservedSpaceSize).subtract(collectedHeapBegin);
+        this.collectedHeapSize = reservedSize.subtract(collectedHeapBegin.subtract(reservedBegin));
+
+        if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
+            NativeMemoryTracking.singleton().trackReserve(collectedHeapSize, NmtCategory.JavaHeap);
+        }
 
         FreeListNode node = allocNodeOrNull(collectedHeapBegin, collectedHeapSize);
         if (node.isNull()) {
@@ -227,10 +253,20 @@ protected int initializeFields(Pointer spaceBegin, UnsignedWord reservedSpaceSiz
         this.unusedListCount = 1;
         this.allocListHead = node;
         this.allocListCount = 1;
-
         return CEntryPointErrors.NO_ERROR;
     }
 
+    @Override
+    public UnsignedWord getCollectedHeapAddressSpaceSize() {
+        return collectedHeapSize;
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public boolean isInMetaspace(Pointer ptr) {
+        /* Checking against begin and end does not need any locking. */
+        return ptr.aboveOrEqual(metaspaceBegin) && ptr.belowThan(metaspaceEnd);
+    }
+
     @Uninterruptible(reason = "Still being initialized.")
     protected int reserveHeapMemory(UnsignedWord reserved, UnsignedWord alignment, IsolateArguments arguments, WordPointer beginOut) {
         Pointer begin = reserveHeapMemory0(reserved, alignment, arguments);
@@ -329,29 +365,48 @@ protected int unmapAddressSpace(PointerBase heapBase) {
     }
 
     @Override
-    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.")
     public Pointer allocateMetaspaceChunk(UnsignedWord nbytes, UnsignedWord alignment) {
-        WordPointer allocOut = UnsafeStackValue.get(WordPointer.class);
-        int error = allocateInHeapAddressSpace(nbytes, alignment, allocOut);
-        if (error == NO_ERROR) {
-            if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
-                NativeMemoryTracking.singleton().trackCommit(nbytes, NmtCategory.Metaspace);
-            }
-            return allocOut.read();
+        lock.lockNoTransition();
+        try {
+            return allocateMetaspaceChunk0(nbytes, alignment);
+        } finally {
+            lock.unlock();
         }
-        throw reportMetaspaceChunkAllocationFailed(error);
     }
 
-    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    protected OutOfMemoryError reportMetaspaceChunkAllocationFailed(int error) {
-        /* Explicitly don't use OutOfMemoryUtil as the metaspace is not part of the Java heap. */
-        if (error == OUT_OF_ADDRESS_SPACE) {
+    /**
+     * This method intentionally does not use {@link OutOfMemoryUtil} when reporting
+     * {@link OutOfMemoryError}s as the metaspace is not part of the Java heap.
+     */
+    @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.")
+    private Pointer allocateMetaspaceChunk0(UnsignedWord nbytes, UnsignedWord alignment) {
+        assert lock.isOwner();
+
+        Pointer result = metaspaceTop;
+        Pointer newTop = metaspaceTop.add(nbytes);
+        assert result.isNonNull();
+        assert PointerUtils.isAMultiple(result, alignment);
+        assert UnsignedUtils.isAMultiple(newTop, alignment);
+
+        /* Check if the allocation fits into the reserved address space. */
+        if (newTop.aboveThan(metaspaceEnd)) {
             throw OUT_OF_METASPACE;
-        } else if (error == COMMIT_FAILED) {
+        }
+
+        /* Try to commit the memory. */
+        int access = VirtualMemoryProvider.Access.READ | VirtualMemoryProvider.Access.WRITE;
+        Pointer actualBegin = VirtualMemoryProvider.get().commit(result, nbytes, access);
+        if (actualBegin.isNull()) {
             throw METASPACE_CHUNK_COMMIT_FAILED;
-        } else {
-            throw VMError.shouldNotReachHereAtRuntime();
         }
+
+        /* Update top and NMT statistics. */
+        metaspaceTop = newTop;
+        if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
+            NativeMemoryTracking.singleton().trackCommit(nbytes, NmtCategory.Metaspace);
+        }
+        return actualBegin;
     }
 
     @Override
@@ -821,11 +876,6 @@ public UnsignedWord getReservedAddressSpaceSize() {
         return reservedAddressSpaceSize;
     }
 
-    @Override
-    public UnsignedWord getReservedMetaspaceSize() {
-        return reservedMetaspaceSize;
-    }
-
     /** Keeps track of unused memory. */
     @RawStructure
     protected interface FreeListNode extends PointerBase {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java

@@ -36,6 +36,7 @@
 import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.Chunk;
 import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.UnalignedChunk;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.image.ImageHeap;
 import com.oracle.svm.core.image.ImageHeapLayoutInfo;
@@ -88,6 +89,8 @@ public class ChunkedImageHeapLayouter implements ImageHeapLayouter {
     /** @param startOffset Offset relative to the heap base. */
     @SuppressWarnings("this-escape")
     public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset) {
+        assert startOffset % Heap.getHeap().getImageHeapAlignment() == 0 : "the start of each image heap must be aligned";
+
         this.partitions = new ChunkedImageHeapPartition[PARTITION_COUNT];
         this.partitions[READ_ONLY_REGULAR] = new ChunkedImageHeapPartition("readOnly", false, false);
         this.partitions[READ_ONLY_RELOCATABLE] = new ChunkedImageHeapPartition("readOnlyRelocatable", false, false);
@@ -98,6 +101,7 @@ public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset) {
 
         this.heapInfo = heapInfo;
         this.startOffset = startOffset;
+
         UnsignedWord alignedHeaderSize = RememberedSet.get().getHeaderSizeOfAlignedChunk();
         UnsignedWord hugeThreshold = HeapParameters.getAlignedHeapChunkSize().subtract(alignedHeaderSize);
         this.hugeObjectThreshold = hugeThreshold.rawValue();
@@ -172,7 +176,6 @@ public ImageHeapLayoutInfo layout(ImageHeap imageHeap, int pageSize, ImageHeapLa
             assert partition.getStartOffset() % objectAlignment == 0 : partition;
             assert (partition.getStartOffset() + partition.getSize()) % objectAlignment == 0 : partition;
         }
-        assert layoutInfo.getImageHeapSize() % pageSize == 0 : "Image heap size is not a multiple of page size";
         return layoutInfo;
     }
 
@@ -223,8 +226,7 @@ private ImageHeapLayoutInfo populateInfoObjects(int dynamicHubCount, int pageSiz
         long writableSize = writableEnd - offsetOfFirstWritableAlignedChunk;
         /* Aligning the end to the page size can be required for mapping into memory. */
         long imageHeapEnd = NumUtil.roundUp(getReadOnlyHuge().getStartOffset() + getReadOnlyHuge().getSize(), pageSize);
-        long imageHeapSize = imageHeapEnd - startOffset;
-        return new ImageHeapLayoutInfo(startOffset, imageHeapSize, offsetOfFirstWritableAlignedChunk, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(),
+        return new ImageHeapLayoutInfo(startOffset, imageHeapEnd, offsetOfFirstWritableAlignedChunk, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(),
                         getWritablePatched().getStartOffset(), getWritablePatched().getSize());
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java

@@ -53,6 +53,7 @@
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.log.Log;
+import com.oracle.svm.core.metaspace.Metaspace;
 import com.oracle.svm.core.thread.VMThreads;
 import com.oracle.svm.core.threadlocal.VMThreadLocalSupport;
 import com.oracle.svm.core.util.Timer;
@@ -361,7 +362,7 @@ private void fixupImageHeapRoots(ImageHeapInfo info) {
 
     @Uninterruptible(reason = "Avoid unnecessary safepoint checks in GC for performance.")
     private void fixupMetaspace() {
-        if (!MetaspaceImpl.isSupported()) {
+        if (!Metaspace.isSupported()) {
             return;
         }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -92,6 +92,7 @@
 import com.oracle.svm.core.jfr.JfrTicks;
 import com.oracle.svm.core.jfr.events.AllocationRequiringGCEvent;
 import com.oracle.svm.core.log.Log;
+import com.oracle.svm.core.metaspace.Metaspace;
 import com.oracle.svm.core.os.ChunkBasedCommittedMemoryProvider;
 import com.oracle.svm.core.snippets.ImplicitExceptions;
 import com.oracle.svm.core.snippets.KnownIntrinsics;
@@ -968,7 +969,7 @@ private void blackenDirtyCardRoots() {
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private void blackenMetaspace() {
-        if (!MetaspaceImpl.isSupported()) {
+        if (!Metaspace.isSupported()) {
             return;
         }