From 7fc077f84b6f0be61f993cec925ec3c1a3b91e71 Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Mon, 27 Nov 2023 15:26:21 -0500 Subject: [PATCH 1/3] add virtual memory tracking --- .../svm/core/foreign/TrampolineSet.java | 3 +- .../posix/PosixVirtualMemoryProvider.java | 117 +++++- .../posix/linux/LinuxImageHeapProvider.java | 42 ++- .../windows/WindowsImageHeapProvider.java | 9 +- .../windows/WindowsVirtualMemoryProvider.java | 50 ++- .../src/com/oracle/svm/core/Isolates.java | 5 +- .../graal/snippets/CEntryPointSnippets.java | 14 +- .../svm/core/nmt/NativeMemoryTracking.java | 151 ++++++-- .../com/oracle/svm/core/nmt/NmtCategory.java | 2 + .../oracle/svm/core/nmt/NmtMemoryRegion.java | 62 +++ .../svm/core/nmt/NmtMemoryRegionAccess.java | 153 ++++++++ .../core/nmt/NmtMemoryRegionListAccess.java | 156 ++++++++ .../svm/core/nmt/NmtPreImageHeapData.java | 48 +++ .../core/nmt/NmtPreImageHeapDataAccess.java | 102 +++++ .../svm/core/nmt/NmtReservedRegion.java | 43 +++ .../svm/core/nmt/NmtReservedRegionAccess.java | 257 +++++++++++++ .../svm/core/nmt/NmtVirtualMemoryInfo.java | 99 +++++ .../svm/core/nmt/NmtVirtualMemoryTracker.java | 202 ++++++++++ .../os/AbstractCommittedMemoryProvider.java | 13 +- .../os/AbstractCopyingImageHeapProvider.java | 28 +- .../svm/core/os/CommittedMemoryProvider.java | 4 +- .../oracle/svm/core/os/ImageHeapProvider.java | 3 +- .../core/os/OSCommittedMemoryProvider.java | 5 +- .../svm/core/os/VirtualMemoryProvider.java | 17 +- .../test/nmt/NativeMemoryTrackingTests.java | 356 ++++++++++++++++++ 25 files changed, 1854 insertions(+), 87 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegion.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionAccess.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionListAccess.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapData.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapDataAccess.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegion.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegionAccess.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryTracker.java diff --git a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java index 61974e0518b6..11d6073c969c 100644 --- a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java +++ b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java @@ -36,6 +36,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.VMError; @@ -111,7 +112,7 @@ private Pointer prepareTrampolines(PinnedObject mhsArray, PinnedObject stubsArra VirtualMemoryProvider memoryProvider = VirtualMemoryProvider.get(); UnsignedWord pageSize = allocationSize(); /* We request a specific alignment to guarantee correctness of getAllocationBase */ - Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE); + Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE, NmtCategory.Internal); if (page.isNull()) { throw new OutOfMemoryError("Could not allocate memory for trampolines."); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java index cffcbbb4550d..e296b2286402 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java @@ -54,10 +54,15 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; +import com.oracle.svm.core.nmt.NmtPreImageHeapDataAccess; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.posix.headers.Unistd; import com.oracle.svm.core.util.PointerUtils; import com.oracle.svm.core.util.UnsignedUtils; +import com.oracle.svm.core.VMInspectionOptions; @AutomaticallyRegisteredFeature class PosixVirtualMemoryProviderFeature implements InternalFeature { @@ -109,7 +114,18 @@ public UnsignedWord getGranularity() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData) { + return reserve0(nbytes, alignment, executable, nmtData, NmtCategory.ImageHeap); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtCategory category) { + return reserve0(nbytes, alignment, executable, WordFactory.nullPointer(), category); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData, NmtCategory category) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -128,25 +144,54 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec return nullPointer(); } if (!customAlignment) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackReserve(mappingBegin, mappingSize, category); + } else { + NmtPreImageHeapDataAccess.enqueueReserve(nmtData, mappingBegin, mappingSize, category); + } + } return mappingBegin; } + UnsignedWord unmappedSize = WordFactory.zero(); Pointer begin = PointerUtils.roundUp(mappingBegin, alignment); UnsignedWord clippedBegin = begin.subtract(mappingBegin); if (clippedBegin.aboveOrEqual(granularity)) { - munmap(mappingBegin, UnsignedUtils.roundDown(clippedBegin, granularity)); + UnsignedWord unmapSize = UnsignedUtils.roundDown(clippedBegin, granularity); + munmap(mappingBegin, unmapSize); + unmappedSize = unmappedSize.add(unmapSize); } Pointer mappingEnd = mappingBegin.add(mappingSize); UnsignedWord clippedEnd = mappingEnd.subtract(begin.add(nbytes)); if (clippedEnd.aboveOrEqual(granularity)) { UnsignedWord rounded = UnsignedUtils.roundDown(clippedEnd, granularity); munmap(mappingEnd.subtract(rounded), rounded); + unmappedSize = unmappedSize.add(rounded); + } + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackReserve(begin, mappingSize.subtract(unmappedSize), category); + } else { + NmtPreImageHeapDataAccess.enqueueReserve(nmtData, begin, mappingSize.subtract(unmappedSize), category); + } } return begin; } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) { + public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData) { + return mapFile0(start, nbytes, fileHandle, offset, access, nmtData, NmtCategory.ImageHeap); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtCategory category) { + return mapFile0(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer(), category); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer mapFile0(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData, NmtCategory category) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -157,12 +202,27 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand } int fd = (int) fileHandle.rawValue(); Pointer result = mmap(start, nbytes, accessAsProt(access), flags, fd, offset.rawValue()); - return result.notEqual(MAP_FAILED()) ? result : WordFactory.nullPointer(); + if (result.notEqual(MAP_FAILED())) { + trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category); + return result; + } + return WordFactory.nullPointer(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { + public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData) { + return commit0(start, nbytes, access, nmtData, NmtCategory.ImageHeap); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtCategory category) { + return commit0(start, nbytes, access, WordFactory.nullPointer(), category); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer commit0(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData, NmtCategory category) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -177,7 +237,31 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { } /* The memory returned by mmap is guaranteed to be zeroed. */ final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET); - return result.notEqual(MAP_FAILED()) ? result : nullPointer(); + if (result.notEqual(MAP_FAILED())) { + trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category); + return result; + } + return nullPointer(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void trackCommitAndMaybeReserve(PointerBase baseAddr, PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData, NmtCategory category) { + if (!VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + return; + } + // Account for possible reserve before commit + if (start.isNull()) { + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackReserve(baseAddr, nbytes, category); + } else { + NmtPreImageHeapDataAccess.enqueueReserve(nmtData, baseAddr, nbytes, category); + } + } + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackCommit(baseAddr, nbytes, category); + } else { + NmtPreImageHeapDataAccess.enqueueCommit(nmtData, baseAddr, nbytes, category); + } } @Override @@ -196,14 +280,25 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) { if (start.isNull() || !isAligned(start) || nbytes.equal(0)) { return -1; } - final Pointer result = mmap(start, nbytes, PROT_NONE(), MAP_FIXED() | MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET); - return result.notEqual(MAP_FAILED()) ? 0 : -1; + if (result.notEqual(MAP_FAILED())) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().trackUncommit(start, nbytes); + } + return 0; + } + return -1; } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int free(PointerBase start, UnsignedWord nbytes) { + return free(start, nbytes, WordFactory.nullPointer()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData) { if (start.isNull() || !isAligned(start) || nbytes.equal(0)) { return -1; } @@ -211,7 +306,11 @@ public int free(PointerBase start, UnsignedWord nbytes) { UnsignedWord granularity = getGranularity(); Pointer mappingBegin = PointerUtils.roundDown(start, granularity); UnsignedWord mappingSize = UnsignedUtils.roundUp(nbytes, granularity); - return munmap(mappingBegin, mappingSize); + int ret = munmap(mappingBegin, mappingSize); + if (ret == 0 && nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().trackFree(start); + } + return ret; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java index f8daf0193386..5bafe375d050 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java @@ -68,6 +68,7 @@ import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; import com.oracle.svm.core.imagelayer.ImageLayerSection; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; import com.oracle.svm.core.os.AbstractImageHeapProvider; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; @@ -156,7 +157,7 @@ protected UnsignedWord getImageHeapAddressSpaceSize() { } @Uninterruptible(reason = "Called during isolate initialization.") - protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReservedHeapBase, UnsignedWord initialRemainingSize, WordPointer endPointer) { + protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReservedHeapBase, UnsignedWord initialRemainingSize, WordPointer endPointer, NmtPreImageHeapData nmtData) { int result = -1; UnsignedWord remainingSize = initialRemainingSize; @@ -178,9 +179,9 @@ protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReserve cachedFDPointer, cachedOffsetsPointer, MAGIC.get(), heapBegin, heapEnd, heapRelocBegin, heapAnyRelocPointer, heapRelocEnd, - heapWritableBegin, heapWritableEnd); + heapWritableBegin, heapWritableEnd, nmtData); if (result != CEntryPointErrors.NO_ERROR) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return result; } Pointer newHeapStart = endPointer.read(); // aligned @@ -196,12 +197,12 @@ protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReserve @Override @Uninterruptible(reason = "Called during isolate initialization.") - public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) { + public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtPreImageHeapData nmtData) { Pointer selfReservedMemory = WordFactory.nullPointer(); UnsignedWord requiredSize = getTotalRequiredAddressSpaceSize(); if (reservedAddressSpace.isNull()) { UnsignedWord alignment = WordFactory.unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment()); - selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false); + selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false, nmtData); if (selfReservedMemory.isNull()) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } @@ -225,13 +226,13 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W int error = DynamicMethodAddressResolutionHeapSupport.get().initialize(); if (error != CEntryPointErrors.NO_ERROR) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return error; } error = DynamicMethodAddressResolutionHeapSupport.get().install(heapBase); if (error != CEntryPointErrors.NO_ERROR) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return error; } } else { @@ -248,19 +249,20 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W CACHED_IMAGE_FDS.get(), CACHED_IMAGE_HEAP_OFFSETS.get(), MAGIC.get(), IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get(), IMAGE_HEAP_RELOCATABLE_BEGIN.get(), IMAGE_HEAP_A_RELOCATABLE_POINTER.get(), IMAGE_HEAP_RELOCATABLE_END.get(), - IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get()); + IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get(), nmtData); if (result != CEntryPointErrors.NO_ERROR) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); } return result; } else { - return initializeLayeredImage(imageHeapStart, selfReservedHeapBase, remainingSize, endPointer); + return initializeLayeredImage(imageHeapStart, selfReservedHeapBase, remainingSize, endPointer, nmtData); } } @Uninterruptible(reason = "Called during isolate initialization.") private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedSize, WordPointer endPointer, WordPointer cachedFd, WordPointer cachedOffsetInFile, - Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) { + Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym, + NmtPreImageHeapData nmtData) { assert heapBeginSym.belowOrEqual(heapWritableSym) && heapWritableSym.belowOrEqual(heapWritableEndSym) && heapWritableEndSym.belowOrEqual(heapEndSym); assert heapBeginSym.belowOrEqual(heapRelocsSym) && heapRelocsSym.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.belowOrEqual(heapEndSym); assert heapAnyRelocPointer.isNull() || (heapRelocsSym.belowOrEqual(heapAnyRelocPointer) && heapAnyRelocPointer.belowThan(heapRelocsEndSym)); @@ -297,12 +299,12 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS * heap must be in pristine condition for that). */ if (fd.equal(CANNOT_OPEN_FD)) { - return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym); + return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym, nmtData); } // Create memory mappings from the image file. UnsignedWord fileOffset = cachedOffsetInFile.read(); - Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ); + Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ, nmtData); if (mappedImageHeap.isNull() || mappedImageHeap != imageHeap) { return CEntryPointErrors.MAP_HEAP_FAILED; } @@ -323,7 +325,7 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS * be part of a chunk with writable objects, in which case the chunk header must * also be writable, and all the chunk's pages will be unprotected below. */ - Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE); + Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE, nmtData); if (committedRelocsBegin.isNull() || committedRelocsBegin != relocsBoundary) { return CEntryPointErrors.PROTECT_HEAP_FAILED; } @@ -351,8 +353,9 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS } @Uninterruptible(reason = "Called during isolate initialization.") - private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym) { - Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE); + private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym, + NmtPreImageHeapData nmtData) { + Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE, nmtData); if (committedBegin.isNull()) { return CEntryPointErrors.MAP_HEAP_FAILED; } @@ -445,6 +448,11 @@ private static boolean checkImageFileMagic(int mapfd, int imagefd, CCharPointer @Override @Uninterruptible(reason = "Called during isolate tear-down.") public int freeImageHeap(PointerBase heapBase) { + return freeImageHeap(heapBase, WordFactory.nullPointer()); + } + + @Uninterruptible(reason = "Called during isolate tear-down.") + private int freeImageHeap(PointerBase heapBase, NmtPreImageHeapData nmtData) { if (heapBase.isNull()) { // no memory allocated return CEntryPointErrors.NO_ERROR; } @@ -454,7 +462,7 @@ public int freeImageHeap(PointerBase heapBase) { if (DynamicMethodAddressResolutionHeapSupport.isEnabled()) { addressSpaceStart = addressSpaceStart.subtract(getPreHeapAlignedSizeForDynamicMethodAddressResolver()); } - if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize()) != 0) { + if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize(), nmtData) != 0) { return CEntryPointErrors.FREE_IMAGE_HEAP_FAILED; } return CEntryPointErrors.NO_ERROR; diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java index 5b06c4420457..fed39da7ab23 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java @@ -46,6 +46,7 @@ import com.oracle.svm.core.c.function.CEntryPointActions; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; import com.oracle.svm.core.os.AbstractCopyingImageHeapProvider; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; @@ -64,18 +65,18 @@ public class WindowsImageHeapProvider extends AbstractCopyingImageHeapProvider { @Override @Uninterruptible(reason = "Called during isolate initialization.") - protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap) { + protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap, NmtPreImageHeapData nmtData) { HANDLE imageHeapFileMapping = getImageHeapFileMapping(); if (imageHeapFileMapping.isNull()) { /* Fall back to copying from memory. */ - return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap); + return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap, nmtData); } /* Map a copy-on-write view of the image heap. */ if (VirtualMemoryProvider.get().mapFile(newImageHeap, imageHeapSize, imageHeapFileMapping, getImageHeapFileOffset(), - Access.READ | Access.WRITE).isNull()) { + Access.READ | Access.WRITE, nmtData).isNull()) { /* Fall back to copying from memory. */ - return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap); + return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap, nmtData); } /* Copy relocatable pages. */ diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java index b833a15f32f7..989dfc6498df 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java @@ -46,6 +46,8 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.c.function.CEntryPointActions; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.PointerUtils; import com.oracle.svm.core.util.UnsignedUtils; @@ -125,7 +127,19 @@ private static int accessAsProt(int access) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData) { + return reserve0(nbytes, alignment, executable, nmtData, NmtCategory.ImageHeap); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtCategory category) { + return reserve0(nbytes, alignment, executable, WordFactory.nullPointer(), category); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unused") + private static Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData, NmtCategory category) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -283,7 +297,19 @@ private interface MEM_ADDRESS_REQUIREMENTS extends PointerBase { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) { + public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData) { + return mapFile0(start, nbytes, fileHandle, offset, access, nmtData, NmtCategory.ImageHeap); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtCategory category) { + return mapFile0(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer(), category); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unused") + private Pointer mapFile0(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData, NmtCategory category) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -351,7 +377,19 @@ Pointer invoke(HANDLE fileMapping, HANDLE process, PointerBase baseAddress, long @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { + public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData) { + return commit0(start, nbytes, access, nmtData, NmtCategory.ImageHeap); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtCategory category) { + return commit0(start, nbytes, access, WordFactory.nullPointer(), category); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unused") + private Pointer commit0(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData, NmtCategory category) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -389,6 +427,12 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int free(PointerBase start, UnsignedWord nbytes) { + return free(start, nbytes, WordFactory.nullPointer()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData) { if (start.isNull() || !isAligned(start) || nbytes.equal(0)) { return -1; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java index c563364605fd..cb0be9ac3556 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java @@ -39,6 +39,7 @@ import com.oracle.svm.core.c.function.CEntryPointCreateIsolateParameters; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.c.function.CEntryPointSetup; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.VMError; @@ -137,9 +138,9 @@ public static int checkIsolate(Isolate isolate) { } @Uninterruptible(reason = "Thread state not yet set up.") - public static int create(WordPointer isolatePointer, CEntryPointCreateIsolateParameters parameters) { + public static int create(WordPointer isolatePointer, CEntryPointCreateIsolateParameters parameters, NmtPreImageHeapData nmtData) { WordPointer heapBasePointer = StackValue.get(WordPointer.class); - int result = CommittedMemoryProvider.get().initialize(heapBasePointer, parameters); + int result = CommittedMemoryProvider.get().initialize(heapBasePointer, parameters, nmtData); if (result != CEntryPointErrors.NO_ERROR) { return result; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 3c7eba65613c..105200a6eeee 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -33,6 +33,7 @@ import java.util.Map; +import com.oracle.svm.core.VMInspectionOptions; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Isolate; @@ -78,6 +79,8 @@ import com.oracle.svm.core.jdk.PlatformNativeLibrarySupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; +import com.oracle.svm.core.nmt.NmtPreImageHeapDataAccess; import com.oracle.svm.core.option.RuntimeOptionParser; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.os.MemoryProtectionProvider; @@ -238,7 +241,12 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara } WordPointer isolatePtr = StackValue.get(WordPointer.class); - int error = Isolates.create(isolatePtr, parameters); + NmtPreImageHeapData nmtData = WordFactory.nullPointer(); + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + nmtData = NmtPreImageHeapDataAccess.create(); + } + + int error = Isolates.create(isolatePtr, parameters, nmtData); if (error != CEntryPointErrors.NO_ERROR) { return error; } @@ -247,6 +255,10 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara setHeapBase(Isolates.getHeapBase(isolate)); } + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NmtPreImageHeapDataAccess.trackAndTeardown(nmtData); + } + return createIsolate0(isolate, parameters, parsedArgs); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java index 149d897ede1a..13187ca1b705 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -58,15 +58,25 @@ public class NativeMemoryTracking { private static final UnsignedWord ALIGNMENT = WordFactory.unsigned(16); private static final int MAGIC = 0xF0F1F2F3; - private final NmtMallocMemoryInfo[] categories; - private final NmtMallocMemoryInfo total; + private final NmtMallocMemoryInfo[] mallocCategories; + private final NmtVirtualMemoryInfo[] virtualCategories; + private final NmtMallocMemoryInfo mallocTotal; + private final NmtVirtualMemoryInfo virtualTotal; + private final NmtVirtualMemoryTracker virtualMemoryTracker = new NmtVirtualMemoryTracker(); @Platforms(Platform.HOSTED_ONLY.class) public NativeMemoryTracking() { - total = new NmtMallocMemoryInfo(); - categories = new NmtMallocMemoryInfo[NmtCategory.values().length]; - for (int i = 0; i < categories.length; i++) { - categories[i] = new NmtMallocMemoryInfo(); + mallocTotal = new NmtMallocMemoryInfo(); + virtualTotal = new NmtVirtualMemoryInfo(); + + mallocCategories = new NmtMallocMemoryInfo[NmtCategory.values().length]; + for (int i = 0; i < mallocCategories.length; i++) { + mallocCategories[i] = new NmtMallocMemoryInfo(); + } + + virtualCategories = new NmtVirtualMemoryInfo[NmtCategory.values().length]; + for (int i = 0; i < virtualCategories.length; i++) { + virtualCategories[i] = new NmtVirtualMemoryInfo(); } } @@ -116,9 +126,9 @@ public void track(PointerBase innerPtr) { UnsignedWord allocationSize = header.getAllocationSize(); UnsignedWord totalSize = allocationSize.add(nmtHeaderSize); - getInfo(header.getCategory()).track(allocationSize); - getInfo(NmtCategory.NMT).track(nmtHeaderSize); - total.track(totalSize); + getMallocInfo(header.getCategory()).track(allocationSize); + getMallocInfo(NmtCategory.NMT).track(nmtHeaderSize); + mallocTotal.track(totalSize); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -134,9 +144,9 @@ public PointerBase untrack(PointerBase innerPtr) { @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void untrack(UnsignedWord size, int category) { - getInfo(category).untrack(size); - getInfo(NmtCategory.NMT).untrack(sizeOfNmtHeader()); - total.untrack(size.add(sizeOfNmtHeader())); + getMallocInfo(category).untrack(size); + getMallocInfo(NmtCategory.NMT).untrack(sizeOfNmtHeader()); + mallocTotal.untrack(size.add(sizeOfNmtHeader())); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -151,54 +161,130 @@ private static Pointer getInnerPointer(NmtMallocHeader mallocHeader) { return ((Pointer) mallocHeader).add(sizeOfNmtHeader()); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackReserve(PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + NativeMemoryTracking.singleton().virtualMemoryTracker.trackReserve(baseAddr, size, category); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void recordReserve(UnsignedWord size, NmtCategory category) { + getVirtualInfo(category).trackReserved(size); + virtualTotal.trackReserved(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackCommit(PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + NativeMemoryTracking.singleton().virtualMemoryTracker.trackCommit(baseAddr, size, category); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void recordCommit(UnsignedWord size, NmtCategory category) { + getVirtualInfo(category).trackCommitted(size); + virtualTotal.trackCommitted(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackUncommit(PointerBase baseAddr, UnsignedWord size) { + virtualMemoryTracker.trackUncommit(baseAddr, size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void recordUncommit(UnsignedWord size, NmtCategory category) { + getVirtualInfo(category).trackUncommit(size); + virtualTotal.trackUncommit(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackFree(PointerBase baseAddr) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().virtualMemoryTracker.trackFree(baseAddr); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void recordFree(UnsignedWord size, NmtCategory category) { + getVirtualInfo(category).trackFree(size); + virtualTotal.trackFree(size); + } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getUsedMemory(NmtCategory category) { - return getInfo(category).getUsed(); + return getMallocInfo(category).getUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getPeakUsedMemory(NmtCategory category) { - return getInfo(category).getPeakUsed(); + return getMallocInfo(category).getPeakUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getCountAtPeakUsage(NmtCategory category) { - return getInfo(category).getCountAtPeakUsage(); + return getMallocInfo(category).getCountAtPeakUsage(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getTotalCount() { - return total.getCount(); + return mallocTotal.getCount(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getTotalUsedMemory() { - return total.getUsed(); + return mallocTotal.getUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getPeakTotalUsedMemory() { - return total.getPeakUsed(); + return mallocTotal.getPeakUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getCountAtTotalPeakUsage() { - return total.getCountAtPeakUsage(); + return mallocTotal.getCountAtPeakUsage(); + } + + public long getReservedByCategory(NmtCategory category) { + return NativeMemoryTracking.singleton().getVirtualInfo(category).getReservedSize(); + } + + public long getCommittedByCategory(NmtCategory category) { + return getVirtualInfo(category).getCommittedSize(); + } + + public long getPeakCommittedByCategory(NmtCategory category) { + return getVirtualInfo(category).getPeakCommittedSize(); + } + + public long getPeakReservedByCategory(NmtCategory category) { + return getVirtualInfo(category).getPeakReservedSize(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private NmtMallocMemoryInfo getMallocInfo(NmtCategory category) { + return getMallocInfo(category.ordinal()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private NmtMallocMemoryInfo getInfo(NmtCategory category) { - return getInfo(category.ordinal()); + private NmtMallocMemoryInfo getMallocInfo(int category) { + assert category < mallocCategories.length; + return mallocCategories[category]; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private NmtMallocMemoryInfo getInfo(int category) { - assert category < categories.length; - return categories[category]; + private NmtVirtualMemoryInfo getVirtualInfo(NmtCategory category) { + return getVirtualInfo(category.ordinal()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private NmtVirtualMemoryInfo getVirtualInfo(int category) { + assert category < virtualCategories.length; + return virtualCategories[category]; } public static RuntimeSupport.Hook shutdownHook() { - return isFirstIsolate -> NativeMemoryTracking.singleton().printStatistics(); + return isFirstIsolate -> { + NativeMemoryTracking.singleton().printStatistics(); + NativeMemoryTracking.teardown(); + }; } private void printStatistics() { @@ -206,19 +292,32 @@ private void printStatistics() { System.out.println(); System.out.println("Native memory tracking"); System.out.println(" Peak total used memory: " + getPeakTotalUsedMemory() + " bytes"); + System.out.println(" Peak Total committed memory: " + virtualTotal.getPeakCommittedSize() + " bytes"); + System.out.println(" Peak Total reserved memory: " + virtualTotal.getPeakReservedSize() + " bytes"); System.out.println(" Total alive allocations at peak usage: " + getCountAtTotalPeakUsage()); System.out.println(" Total used memory: " + getTotalUsedMemory() + " bytes"); System.out.println(" Total alive allocations: " + getTotalCount()); + System.out.println(" Total committed memory: " + virtualTotal.getCommittedSize() + " bytes"); + System.out.println(" Total reserved memory: " + virtualTotal.getReservedSize() + " bytes"); for (int i = 0; i < NmtCategory.values().length; i++) { String name = NmtCategory.values()[i].getName(); - NmtMallocMemoryInfo info = getInfo(i); + NmtMallocMemoryInfo info = getMallocInfo(i); + NmtVirtualMemoryInfo vMemInfo = getVirtualInfo(i); System.out.println(" " + name + " peak used memory: " + info.getPeakUsed() + " bytes"); System.out.println(" " + name + " alive allocations at peak: " + info.getCountAtPeakUsage()); System.out.println(" " + name + " currently used memory: " + info.getUsed() + " bytes"); System.out.println(" " + name + " currently alive allocations: " + info.getCount()); + System.out.println(" " + name + " committed memory: " + vMemInfo.getCommittedSize()); + System.out.println(" " + name + " reserved memory: " + vMemInfo.getReservedSize()); + System.out.println(" " + name + " peak committed memory: " + vMemInfo.getPeakCommittedSize()); + System.out.println(" " + name + " peak reserved memory: " + vMemInfo.getPeakReservedSize()); } } } + + private static void teardown() { + NativeMemoryTracking.singleton().virtualMemoryTracker.teardown(); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java index ff71cdfabaed..c4fb16ae7037 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java @@ -37,6 +37,8 @@ public enum NmtCategory { GC("GC"), /** Heap dumping infrastructure. */ HeapDump("Heap Dump"), + ImageHeap("Image Heap"), + JavaHeap("Java Heap"), /** Java Flight Recorder. */ JFR("JFR"), /** Java Native Interface. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegion.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegion.java new file mode 100644 index 000000000000..e7769db1c259 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegion.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.c.struct.PinnedObjectField; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +@RawStructure +public interface NmtMemoryRegion extends PointerBase { + @RawField + NmtMemoryRegion getNext(); + + @RawField + void setNext(NmtMemoryRegion value); + + @RawField + PointerBase getBaseAddr(); + + @RawField + void setBaseAddr(PointerBase value); + + @RawField + UnsignedWord getSize(); + + @RawField + void setSize(UnsignedWord value); + + @PinnedObjectField + @RawField + NmtCategory getCategory(); + + @PinnedObjectField + @RawField + void setCategory(NmtCategory category); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionAccess.java new file mode 100644 index 000000000000..73685b922ace --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionAccess.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import jdk.graal.compiler.api.replacements.Fold; +import org.graalvm.word.WordFactory; +import com.oracle.svm.core.jdk.UninterruptibleUtils.Math; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.UnsignedUtils; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +class NmtMemoryRegionAccess { + + @Fold + static UnsignedWord getSize() { + return UnsignedUtils.roundUp(SizeOf.unsigned(NmtMemoryRegion.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static NmtMemoryRegion allocate(PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + NmtMemoryRegion result = NullableNativeMemory.malloc(getSize(), NmtCategory.NMT); + initialize(result, baseAddr, size, category); + return result; + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static void initialize(NmtMemoryRegion memoryRegion, PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + if (memoryRegion.isNonNull()) { + memoryRegion.setCategory(category); + memoryRegion.setNext(WordFactory.nullPointer()); + memoryRegion.setSize(size); + memoryRegion.setBaseAddr(baseAddr); + } + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static boolean isEqual(NmtMemoryRegion r1, NmtMemoryRegion r2) { + if (r1.isNonNull() && r2.isNonNull() && r1.getSize().equal(r2.getSize()) && baseAddr(r1).equal(baseAddr(r2))) { + return true; + } + return false; + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static boolean contains(NmtMemoryRegion outer, NmtMemoryRegion inner) { + if (containsAddress(outer, baseAddr(inner)) && containsAddress(outer, endAddr(inner))) { + return true; + } + return false; + } + + /** "Contains" also includes the outer bounds. */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static boolean containsAddress(NmtMemoryRegion region, UnsignedWord addr) { + if (baseAddr(region).belowOrEqual(addr) && addr.belowOrEqual(endAddr(region))) { + return true; + } + return false; + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static boolean isOverlapping(NmtMemoryRegion r1, NmtMemoryRegion r2) { + if (getOverlapSize(r1, r2) > 0) { + return true; + } + return false; + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static long getOverlapSize(NmtMemoryRegion r1, NmtMemoryRegion r2) { + long region1Base = baseAddr(r1).rawValue(); + long region1End = endAddr(r1).rawValue(); + long region2Base = baseAddr(r2).rawValue(); + long region2End = endAddr(r2).rawValue(); + return Math.min(region1End, region2End) - Math.max(region1Base, region2Base); + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static void excludeRegion(NmtMemoryRegion outerRegion, UnsignedWord start, UnsignedWord size) { + UnsignedWord newSize = outerRegion.getSize().subtract(size); + if (baseAddr(outerRegion).equal(start)) { + outerRegion.setBaseAddr(WordFactory.pointer(start.add(size).rawValue())); + } + outerRegion.setSize(newSize); + } + + /** r1 is the region to potentially expand, r2 is the expansion. */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static boolean tryMergeWith(NmtMemoryRegion r1, NmtMemoryRegion r2) { + if (isAdjacent(r1, r2)) { + merge(r1, r2); + return true; + } + return false; + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean isAdjacent(NmtMemoryRegion r1, NmtMemoryRegion r2) { + if (r1.isNull() || r2.isNull()) { + return false; + } + return endAddr(r1).equal(baseAddr(r2)) || endAddr(r2).equal(baseAddr(r1)); + } + + /** r1 is the region to expand, r2 is the expansion. */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void merge(NmtMemoryRegion r1, NmtMemoryRegion r2) { + // If we are expanding backwards, we must update the base pointer. + if (baseAddr(r1).equal(endAddr(r2))) { + r1.setBaseAddr(r2.getBaseAddr()); + } + UnsignedWord newSize = r1.getSize().add(r2.getSize()); + r1.setSize(newSize); + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static UnsignedWord baseAddr(NmtMemoryRegion region) { + return WordFactory.unsigned(region.getBaseAddr().rawValue()); + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static UnsignedWord endAddr(NmtMemoryRegion region) { + return baseAddr(region).add(region.getSize()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionListAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionListAccess.java new file mode 100644 index 000000000000..15811a4de930 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMemoryRegionListAccess.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import org.graalvm.word.WordFactory; +import com.oracle.svm.core.memory.NullableNativeMemory; + +/** Static methods for operating on memory region lists. */ +public class NmtMemoryRegionListAccess { + /** + * Adds a region to the provided list, returns the new list head. Maintains sorted nature of + * list. + */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static NmtMemoryRegion addSorted(NmtMemoryRegion listHead, NmtMemoryRegion newRegion) { + // If adding an invalid region, do nothing. + if (newRegion.isNull()) { + return listHead; + } + + NmtMemoryRegion current = listHead; + NmtMemoryRegion prev = WordFactory.nullPointer(); + while (current.isNonNull()) { + if (newRegion.getBaseAddr().rawValue() < current.getBaseAddr().rawValue()) { + assert !NmtMemoryRegionAccess.isOverlapping(newRegion, current); + newRegion.setNext(current); + if (prev.isNonNull()) { + // New region is in middle of the list + assert !NmtMemoryRegionAccess.isOverlapping(newRegion, prev); + prev.setNext(newRegion); + return listHead; + } + // New region is the new head of the list + return newRegion; + } + prev = current; + current = current.getNext(); + } + + // New region is the new tail of the list + if (prev.isNonNull()) { + assert !NmtMemoryRegionAccess.isEqual(newRegion, prev); + assert !NmtMemoryRegionAccess.isOverlapping(newRegion, prev); + prev.setNext(newRegion); + return listHead; + } + + // List was previously empty + return newRegion; + } + + /** + * Removes a region from the provided list. Assumes the list is sorted (since it relies on + * findPrecedingRegion). Returns the new list head. + */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static NmtMemoryRegion remove(NmtMemoryRegion listHead, NmtMemoryRegion targetRegion) { + assert listHead.isNonNull(); + NmtMemoryRegion prev = findPrecedingRegion(listHead, targetRegion); + + /* Handle removing the head. */ + if (prev.isNull()) { + assert NmtMemoryRegionAccess.isEqual(listHead, targetRegion); + NmtMemoryRegion newHead = listHead.getNext(); + NullableNativeMemory.free(listHead); + return newHead; + } + + /* We are removing a node that is not head. */ + NmtMemoryRegion current = prev.getNext(); + assert NmtMemoryRegionAccess.isEqual(current, targetRegion); + assert !NmtMemoryRegionAccess.isEqual(listHead, targetRegion); + prev.setNext(current.getNext()); + NullableNativeMemory.free(current); + return listHead; + } + + /** + * Finds the region that contains or matches the target region. Or nullPointer if unsuccessful. + */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static NmtMemoryRegion findContainingRegion(NmtMemoryRegion listHead, NmtMemoryRegion targetRegion) { + NmtMemoryRegion current = listHead; + while (current.isNonNull()) { + if (NmtMemoryRegionAccess.contains(current, targetRegion)) { + return current; + } + current = current.getNext(); + } + return WordFactory.nullPointer(); + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static NmtMemoryRegion findRegionMatchingBase(NmtMemoryRegion listHead, org.graalvm.word.PointerBase baseAddr) { + NmtMemoryRegion current = listHead; + while (current.isNonNull()) { + if (current.getBaseAddr().rawValue() == baseAddr.rawValue()) { + return current; + } + current = current.getNext(); + } + return WordFactory.nullPointer(); + } + + /** Assumes the list is sorted. */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static NmtMemoryRegion findPrecedingRegion(NmtMemoryRegion listHead, NmtMemoryRegion targetRegion) { + NmtMemoryRegion current = listHead; + NmtMemoryRegion preceding = WordFactory.nullPointer(); + while (current.isNonNull()) { + long currentEnd = current.getBaseAddr().rawValue() + current.getSize().rawValue(); + if (currentEnd > targetRegion.getBaseAddr().rawValue()) { + return preceding; + } + preceding = current; + current = current.getNext(); + } + // tail of the list precedes the target region + return preceding; + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static void teardown(NmtMemoryRegion listHead) { + NmtMemoryRegion current = listHead; + while (current.isNonNull()) { + NmtMemoryRegion next = current.getNext(); + NullableNativeMemory.free(current); + current = next; + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapData.java new file mode 100644 index 000000000000..1c623b27a172 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapData.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; + +/** This is only used before the image heap is mapped. */ +@RawStructure +public interface NmtPreImageHeapData extends PointerBase { + + @RawField + NmtMemoryRegion getReservedListHead(); + + @RawField + void setReservedListHead(NmtMemoryRegion reservedListHead); + + @RawField + NmtMemoryRegion getCommittedListHead(); + + @RawField + void setCommittedListHead(NmtMemoryRegion committedListHead); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapDataAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapDataAccess.java new file mode 100644 index 000000000000..9867cc8d23a0 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtPreImageHeapDataAccess.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; +import com.oracle.svm.core.util.UnsignedUtils; +import org.graalvm.nativeimage.c.struct.SizeOf; + +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +/** + * This class is used to access {@link com.oracle.svm.core.nmt.NmtPreImageHeapData}. It must use + * untracked native memory for all its operations since it is meant to be used before the image heap + * has been mapped. + */ +public class NmtPreImageHeapDataAccess { + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static NmtPreImageHeapData create() { + NmtPreImageHeapData data = UntrackedNullableNativeMemory + .malloc(UnsignedUtils.roundUp(SizeOf.unsigned(NmtPreImageHeapData.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize))); + if (data.isNonNull()) { + data.setCommittedListHead(WordFactory.nullPointer()); + data.setReservedListHead(WordFactory.nullPointer()); + } + return data; + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void enqueueReserve(NmtPreImageHeapData data, PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + assert data.isNonNull(); + NmtMemoryRegion head = data.getReservedListHead(); + NmtMemoryRegion newRgn = UntrackedNullableNativeMemory.malloc(NmtMemoryRegionAccess.getSize()); + NmtMemoryRegionAccess.initialize(newRgn, baseAddr, size, category); + newRgn.setNext(head); + data.setReservedListHead(newRgn); + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void enqueueCommit(NmtPreImageHeapData data, PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + assert data.isNonNull(); + NmtMemoryRegion head = data.getCommittedListHead(); + NmtMemoryRegion newRgn = UntrackedNullableNativeMemory.malloc(NmtMemoryRegionAccess.getSize()); + NmtMemoryRegionAccess.initialize(newRgn, baseAddr, size, category); + newRgn.setNext(head); + data.setCommittedListHead(newRgn); + } + + /** + * This method should be called after the image heap is ready. This method frees all allocated + * memory and tracks all queued reserves and commits. + */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void trackAndTeardown(NmtPreImageHeapData data) { + /* Account for reserves first since committed regions are dependent on them. */ + NmtMemoryRegion current = data.getReservedListHead(); + while (current.isNonNull()) { + NmtMemoryRegion next = current.getNext(); + NativeMemoryTracking.singleton().trackReserve(current.getBaseAddr(), current.getSize(), current.getCategory()); + UntrackedNullableNativeMemory.free(current); + current = next; + } + + current = data.getCommittedListHead(); + + while (current.isNonNull()) { + NmtMemoryRegion next = current.getNext(); + NativeMemoryTracking.singleton().trackCommit(current.getBaseAddr(), current.getSize(), current.getCategory()); + UntrackedNullableNativeMemory.free(current); + current = next; + } + + UntrackedNullableNativeMemory.free(data); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegion.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegion.java new file mode 100644 index 000000000000..082fd488e897 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegion.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; + +/** + * Reserved regions maintain a list of committed regions within themselves. The list is sorted by + * base address. + */ +@RawStructure +public interface NmtReservedRegion extends NmtMemoryRegion { + @RawField + NmtMemoryRegion getCommittedRegions(); + + @RawField + void setCommittedRegions(NmtMemoryRegion value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegionAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegionAccess.java new file mode 100644 index 000000000000..5bfc0f52930f --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtReservedRegionAccess.java @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.util.BasedOnJDKFile; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +class NmtReservedRegionAccess { + @Fold + static UnsignedWord getReservedRegionSize() { + return UnsignedUtils.roundUp(SizeOf.unsigned(NmtReservedRegion.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + } + + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static NmtReservedRegion createReservedRegion(PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + NmtReservedRegion result = NullableNativeMemory.malloc(getReservedRegionSize(), NmtCategory.NMT); + if (result.isNonNull()) { + result.setCommittedRegions(WordFactory.nullPointer()); + result.setCategory(category); + result.setNext(WordFactory.nullPointer()); + result.setSize(size); + result.setBaseAddr(baseAddr); + } + return result; + } + + /** + * This method adds a committed region to a reserved region and updates committed virtual memory + * accounting. + */ + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23+13/src/hotspot/share/nmt/virtualMemoryTracker.cpp#L116-L167") + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static void addCommittedRegion(NmtReservedRegion reservedRegion, NmtMemoryRegion targetRegionOnStack) { + if (reservedRegion.isNull()) { + return; + } + + NmtMemoryRegion prev = NmtMemoryRegionListAccess.findPrecedingRegion(reservedRegion.getCommittedRegions(), targetRegionOnStack); + NmtMemoryRegion next = prev.isNonNull() ? prev.getNext() : reservedRegion.getCommittedRegions(); + + /* Deal with overlap if target is before last region in list. */ + if (next.isNonNull()) { + /* Do nothing if the region already exists. */ + if (NmtMemoryRegionAccess.isEqual(targetRegionOnStack, next)) { + return; + } + /* Remove overlap, so we can later add the full committed region. */ + if (NmtMemoryRegionAccess.isOverlapping(next, targetRegionOnStack)) { + removeUncommittedRegion(reservedRegion, targetRegionOnStack); + /* Refresh prev and next pointers */ + prev = NmtMemoryRegionListAccess.findPrecedingRegion(reservedRegion.getCommittedRegions(), targetRegionOnStack); + next = prev.isNonNull() ? prev.getNext() : reservedRegion.getCommittedRegions(); + } + } + + /* + * Now, the new region should not overlap any existing regions. Record the full committed + * region. + */ + assert NmtMemoryRegionListAccess.findContainingRegion(reservedRegion.getCommittedRegions(), targetRegionOnStack).isNull(); + + if (NmtMemoryRegionAccess.tryMergeWith(prev, targetRegionOnStack)) { + /* We were able to merge prev with target. Try to further coalesce prev with next. */ + if (NmtMemoryRegionAccess.tryMergeWith(prev, next)) { + /* + * prev has been merged with target and next. Remove next from the committed region + * list. + */ + removeCommittedRegion(reservedRegion, next); + } + } else if (NmtMemoryRegionAccess.tryMergeWith(targetRegionOnStack, next)) { + /* We couldn't merge with prev, but we were able to merge with next. */ + } else { + /* We are unable to merge with existing regions. So insert an entirely new one. */ + NmtMemoryRegion newCommittedRegion = NmtMemoryRegionAccess.allocate(targetRegionOnStack.getBaseAddr(), targetRegionOnStack.getSize(), targetRegionOnStack.getCategory()); + if (newCommittedRegion.isNull()) { + /* Allocation of the new committed region failed. Do not record the commit. */ + return; + } + + /* Add region to committed region list maintaining sorted ordering. */ + if (prev.isNull()) { + reservedRegion.setCommittedRegions(newCommittedRegion); + } else { + prev.setNext(newCommittedRegion); + } + newCommittedRegion.setNext(next); + } + NativeMemoryTracking.singleton().recordCommit(targetRegionOnStack.getSize(), targetRegionOnStack.getCategory()); + } + + /** + * Only remove an entire existing committed region from the reserved region's list of committed + * regions and update the list head. No NMT accounting. + */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void removeCommittedRegion(NmtReservedRegion reservedRegion, NmtMemoryRegion targetRegion) { + NmtMemoryRegion newHead = NmtMemoryRegionListAccess.remove(reservedRegion.getCommittedRegions(), targetRegion); + reservedRegion.setCommittedRegions(newHead); + } + + /** + * This method removes a specified region from a reserved region's committed list. The target + * region may span multiple existing committed regions. The boundaries of the target region may + * not align with the boundaries of any existing committed region. Any committed memory that is + * uncommitted, will be accounted in NMT. + */ + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23+13/src/hotspot/share/nmt/virtualMemoryTracker.cpp#L202-L257") + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static void removeUncommittedRegion(NmtReservedRegion reservedRegion, NmtMemoryRegion targetRegion) { + if (reservedRegion.isNull()) { + return; + } + + NmtMemoryRegion current = reservedRegion.getCommittedRegions(); + while (current.isNonNull()) { + UnsignedWord currentBase = NmtMemoryRegionAccess.baseAddr(current); + UnsignedWord currentEnd = NmtMemoryRegionAccess.endAddr(current); + UnsignedWord targetBase = NmtMemoryRegionAccess.baseAddr(targetRegion); + UnsignedWord targetEnd = NmtMemoryRegionAccess.endAddr(targetRegion); + + /* Exact match. */ + if (NmtMemoryRegionAccess.isEqual(current, targetRegion)) { + removeCommittedRegion(reservedRegion, current); + NativeMemoryTracking.singleton().recordUncommit(current.getSize(), current.getCategory()); + return; + } + + /* Target region contains current region. */ + if (NmtMemoryRegionAccess.contains(targetRegion, current)) { + NativeMemoryTracking.singleton().recordUncommit(current.getSize(), current.getCategory()); + NmtMemoryRegion next = current.getNext(); + removeCommittedRegion(reservedRegion, current); + current = next; + continue; + } + + if (NmtMemoryRegionAccess.containsAddress(current, targetBase)) { + /* Target region's base address is within the current region. */ + if (NmtMemoryRegionAccess.containsAddress(current, targetEnd.subtract(1))) { + /* Target region is contained by current region. */ + removeRegionFromExistingCommittedRegion(current, targetRegion); + return; + } else { + /* Only start of target region is contained in current region. */ + UnsignedWord exclusionSize = currentEnd.subtract(targetBase); + NmtMemoryRegionAccess.excludeRegion(current, targetBase, exclusionSize); + NativeMemoryTracking.singleton().recordUncommit(exclusionSize, current.getCategory()); + } + } else if (NmtMemoryRegionAccess.containsAddress(current, targetEnd.subtract(1))) { + /* Only end of target region is contained in current region. */ + UnsignedWord exclusionSize = targetEnd.subtract(currentBase); + NmtMemoryRegionAccess.excludeRegion(current, currentBase, exclusionSize); + NativeMemoryTracking.singleton().recordUncommit(exclusionSize, current.getCategory()); + return; + } + + current = current.getNext(); + } + } + + /** + * This removes an uncommitted region from within an existing committed region. Uncommitted + * memory, will be accounted in NMT. + */ + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23+13/src/hotspot/share/nmt/virtualMemoryTracker.cpp#L169-L200") + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void removeRegionFromExistingCommittedRegion(NmtMemoryRegion outerRegion, NmtMemoryRegion innerRegion) { + assert NmtMemoryRegionAccess.isOverlapping(outerRegion, innerRegion); + UnsignedWord outerEnd = NmtMemoryRegionAccess.endAddr(outerRegion); + UnsignedWord innerBase = NmtMemoryRegionAccess.baseAddr(innerRegion); + UnsignedWord innerEnd = NmtMemoryRegionAccess.endAddr(innerRegion); + + /* Try to shorten the outer region. */ + if (outerRegion.getBaseAddr() == innerRegion.getBaseAddr() || outerEnd.equal(innerEnd)) { + NmtMemoryRegionAccess.excludeRegion(outerRegion, innerBase, innerRegion.getSize()); + } else { + /* Hollow out and split the outer region. */ + + /* Exclude the whole upper part, only keeping the lower part. */ + UnsignedWord exclusionSize = outerEnd.subtract(innerBase); + NmtMemoryRegionAccess.excludeRegion(outerRegion, innerBase, exclusionSize); + + /* + * Add the upper part we intend to keep as a new committed region, effectively splitting + * the original region. + */ + UnsignedWord highBase = innerEnd; + UnsignedWord highSize = outerEnd.subtract(innerEnd); + NmtMemoryRegion newCommittedRegion = NmtMemoryRegionAccess.allocate(WordFactory.pointer(highBase.rawValue()), highSize, outerRegion.getCategory()); + if (newCommittedRegion.isNull()) { + return; + } + /* + * Add region to committed region list maintaining sorted ordering. List head remains + * the same. + */ + NmtMemoryRegion oldNext = outerRegion.getNext(); + outerRegion.setNext(newCommittedRegion); + newCommittedRegion.setNext(oldNext); + } + NativeMemoryTracking.singleton().recordUncommit(innerRegion.getSize(), outerRegion.getCategory()); + + } + + /** Ensures the provided list is sorted and no regions are overlapping. */ + @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + static boolean verifyReservedList(NmtMemoryRegion listHead) { + if (listHead.isNull()) { + return false; + } + NmtMemoryRegion current = listHead; + NmtMemoryRegion prev = WordFactory.nullPointer(); + while (current.isNonNull()) { + if (prev.isNonNull()) { + if (prev.getBaseAddr().rawValue() >= current.getBaseAddr().rawValue() || NmtMemoryRegionAccess.isOverlapping(prev, current)) { + return false; + } + } + prev = current; + current = current.getNext(); + } + return true; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java new file mode 100644 index 000000000000..5a1970f4de8b --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.UnsignedWord; + +class NmtVirtualMemoryInfo { + + private final AtomicLong peakReservedSize = new AtomicLong(0); + private final AtomicLong peakCommittedSize = new AtomicLong(0); + private final AtomicLong reservedSize = new AtomicLong(0); + private final AtomicLong committedSize = new AtomicLong(0); + + @Platforms(Platform.HOSTED_ONLY.class) + NmtVirtualMemoryInfo() { + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void trackReserved(UnsignedWord size) { + long newReservedSize = reservedSize.addAndGet(size.rawValue()); + updatePeak(newReservedSize, peakReservedSize); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void trackCommitted(UnsignedWord size) { + long newCommittedSize = committedSize.addAndGet(size.rawValue()); + updatePeak(newCommittedSize, peakCommittedSize); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void trackUncommit(UnsignedWord size) { + long lastSize = committedSize.addAndGet(-size.rawValue()); + assert lastSize >= 0; + + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void trackFree(UnsignedWord size) { + long lastSize = reservedSize.addAndGet(-size.rawValue()); + assert lastSize >= 0; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void updatePeak(long newSize, AtomicLong peakToUpdate) { + long oldPeak = peakToUpdate.get(); + while (oldPeak < newSize) { + if (peakToUpdate.compareAndSet(oldPeak, newSize)) { + return; + } + oldPeak = peakToUpdate.get(); + } + } + + long getReservedSize() { + return reservedSize.get(); + } + + long getCommittedSize() { + return committedSize.get(); + } + + long getPeakReservedSize() { + return peakReservedSize.get(); + } + + long getPeakCommittedSize() { + return peakCommittedSize.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryTracker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryTracker.java new file mode 100644 index 000000000000..81e59614cd11 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryTracker.java @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.util.BasedOnJDKFile; +import com.oracle.svm.core.Uninterruptible; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import org.graalvm.nativeimage.StackValue; +import com.oracle.svm.core.thread.JavaSpinLockUtils; + +import jdk.internal.misc.Unsafe; + +/** + * {@link com.oracle.svm.core.nmt.NativeMemoryTracking} delegates virtual memory accounting to this + * class. This class maintains a model of used virtual memory. The tracker maintains a sorted list + * of reserved regions, and each reserved region maintains its own sorted list of internal committed + * regions. + */ +public class NmtVirtualMemoryTracker { + private static final Unsafe U = Unsafe.getUnsafe(); + /* + * Can't use VmMutex because it tracks owners and this class may be used during isolate + * creation. Cannot reuse lock from NativeMemoryTracking because this class may malloc and the + * lock is not reentrant. + */ + private static final long VMEM_LOCK_OFFSET = U.objectFieldOffset(NmtVirtualMemoryTracker.class, "vMemLock"); + @SuppressWarnings("unused") private volatile int vMemLock; + /* + * It may be possible that virtual memory is freed after the teardown is run. We should avoid + * accounting in that case. + */ + private boolean tornDown = false; + + private NmtReservedRegion reservedRegionListHead; + + @Platforms(Platform.HOSTED_ONLY.class) + NmtVirtualMemoryTracker() { + } + + /** Track a new reserved region. This region will not overlap existing reserved regions. */ + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + void trackReserve(PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + lockNoTransition(); + try { + if (tornDown) { + return; + } + NmtReservedRegion newReservedRegion = NmtReservedRegionAccess.createReservedRegion(baseAddr, size, category); + if (newReservedRegion.isNull()) { + return; + } + reservedRegionListHead = (NmtReservedRegion) NmtMemoryRegionListAccess.addSorted(reservedRegionListHead, newReservedRegion); + assert NmtReservedRegionAccess.verifyReservedList(reservedRegionListHead); + NativeMemoryTracking.singleton().recordReserve(size, category); + } finally { + unlock(); + } + } + + /** + * Track a new committed region. This region may overlap multiple or zero existing committed + * regions. The regions must be entirely within a single existing reserved region. + */ + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23+13/src/hotspot/share/nmt/virtualMemoryTracker.cpp#L434-L453") + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + void trackCommit(PointerBase baseAddr, UnsignedWord size, NmtCategory category) { + lockNoTransition(); + try { + if (tornDown) { + return; + } + /* Find the reserved region the committed region belongs to. */ + NmtMemoryRegion targetRegionOnStack = StackValue.get(NmtMemoryRegion.class); + targetRegionOnStack.setSize(size); + targetRegionOnStack.setBaseAddr(baseAddr); + targetRegionOnStack.setCategory(category); + NmtReservedRegion reservedRegion = (NmtReservedRegion) NmtMemoryRegionListAccess.findContainingRegion(reservedRegionListHead, targetRegionOnStack); + + /* Update the reserved region to include the new committed region. */ + NmtReservedRegionAccess.addCommittedRegion(reservedRegion, targetRegionOnStack); + } finally { + unlock(); + } + } + + /** Uncommit a region that may overlap multiple or zero committed regions. */ + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23+13/src/hotspot/share/nmt/virtualMemoryTracker.cpp#L455-L469") + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + void trackUncommit(PointerBase baseAddr, UnsignedWord size) { + lockNoTransition(); + try { + if (tornDown) { + return; + } + /* Find the reserved region we are uncommitting from. */ + NmtMemoryRegion targetRegion = StackValue.get(NmtMemoryRegion.class); + targetRegion.setSize(size); + targetRegion.setBaseAddr(baseAddr); + NmtReservedRegion reservedRegion = (NmtReservedRegion) NmtMemoryRegionListAccess.findContainingRegion(reservedRegionListHead, targetRegion); + + /* Uncommit the specified region from that reserved region. */ + NmtReservedRegionAccess.removeUncommittedRegion(reservedRegion, targetRegion); + } finally { + unlock(); + } + } + + /** + * Untrack a reserved region and all committed regions contained within. The entire reserved + * region must be requested to be freed. See + * {@link com.oracle.svm.core.os.VirtualMemoryProvider#free(PointerBase, UnsignedWord)}. + */ + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23+13/src/hotspot/share/nmt/virtualMemoryTracker.cpp#L491-L533") + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + void trackFree(PointerBase baseAddr) { + lockNoTransition(); + try { + if (tornDown) { + return; + } + + /* Find the reserved region, so we can access its committed list. */ + NmtReservedRegion reservedRegion = (NmtReservedRegion) NmtMemoryRegionListAccess.findRegionMatchingBase(reservedRegionListHead, baseAddr); + + /* + * It's possible that malloc failed to initially create the reserved region. + */ + if (reservedRegion.isNull()) { + return; + } + assert baseAddr == reservedRegion.getBaseAddr(); + + NmtReservedRegionAccess.removeUncommittedRegion(reservedRegion, reservedRegion); + + NativeMemoryTracking.singleton().recordFree(reservedRegion.getSize(), reservedRegion.getCategory()); + reservedRegionListHead = (NmtReservedRegion) NmtMemoryRegionListAccess.remove(reservedRegionListHead, reservedRegion); + assert NmtReservedRegionAccess.verifyReservedList(reservedRegionListHead); + + } finally { + unlock(); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) + private void lockNoTransition() { + JavaSpinLockUtils.lockNoTransition(this, VMEM_LOCK_OFFSET); + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) + private void unlock() { + JavaSpinLockUtils.unlock(this, VMEM_LOCK_OFFSET); + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + void teardown() { + lockNoTransition(); + try { + NmtReservedRegion current = reservedRegionListHead; + + while (current.isNonNull()) { + NmtReservedRegion next = (NmtReservedRegion) current.getNext(); + NmtMemoryRegionListAccess.teardown(current.getCommittedRegions()); + current.setCommittedRegions(WordFactory.nullPointer()); + current = next; + } + NmtMemoryRegionListAccess.teardown(reservedRegionListHead); + reservedRegionListHead = WordFactory.nullPointer(); + tornDown = true; + } finally { + unlock(); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index c4d94e712589..31bd6a345c59 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -40,6 +40,7 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.UnsignedUtils; import jdk.graal.compiler.api.replacements.Fold; @@ -75,24 +76,24 @@ protected static int protectSingleIsolateImageHeap() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { - return allocate(nbytes, alignment, false); + return allocate(nbytes, alignment, false, NmtCategory.JavaHeap); } @Override public Pointer allocateUnalignedChunk(UnsignedWord nbytes) { - return allocate(nbytes, getAlignmentForUnalignedChunks(), false); + return allocate(nbytes, getAlignmentForUnalignedChunks(), false, NmtCategory.JavaHeap); } @Override public Pointer allocateExecutableMemory(UnsignedWord nbytes, UnsignedWord alignment) { - return allocate(nbytes, alignment, true); + return allocate(nbytes, alignment, true, NmtCategory.Code); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable) { + private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable, NmtCategory category) { Pointer reserved = WordFactory.nullPointer(); if (!UnsignedUtils.isAMultiple(getGranularity(), alignment)) { - reserved = VirtualMemoryProvider.get().reserve(size, alignment, executable); + reserved = VirtualMemoryProvider.get().reserve(size, alignment, executable, category); if (reserved.isNull()) { return nullPointer(); } @@ -101,7 +102,7 @@ private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean exec if (executable) { access |= VirtualMemoryProvider.Access.FUTURE_EXECUTE; } - Pointer committed = VirtualMemoryProvider.get().commit(reserved, size, access); + Pointer committed = VirtualMemoryProvider.get().commit(reserved, size, access, category); if (committed.isNull()) { if (reserved.isNonNull()) { VirtualMemoryProvider.get().free(reserved, size); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java index 1dd8ca8906b4..840f5da17ca4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java @@ -40,18 +40,19 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.code.DynamicMethodAddressResolutionHeapSupport; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; import com.oracle.svm.core.util.UnsignedUtils; public abstract class AbstractCopyingImageHeapProvider extends AbstractImageHeapProvider { @Override @Uninterruptible(reason = "Called during isolate initialization.") - public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) { + public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtPreImageHeapData nmtData) { Pointer selfReservedMemory = WordFactory.nullPointer(); UnsignedWord requiredSize = getTotalRequiredAddressSpaceSize(); if (reservedAddressSpace.isNull()) { UnsignedWord alignment = WordFactory.unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment()); - selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false); + selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false, nmtData); if (selfReservedMemory.isNull()) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } @@ -73,13 +74,13 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W int error = DynamicMethodAddressResolutionHeapSupport.get().initialize(); if (error != CEntryPointErrors.NO_ERROR) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return error; } error = DynamicMethodAddressResolutionHeapSupport.get().install(heapBase); if (error != CEntryPointErrors.NO_ERROR) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return error; } } else { @@ -90,9 +91,9 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W // Copy the memory to the reserved address space. UnsignedWord imageHeapSizeInFile = getImageHeapSizeInFile(IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get()); Pointer imageHeap = getImageHeapBegin(heapBase); - int result = commitAndCopyMemory(IMAGE_HEAP_BEGIN.get(), imageHeapSizeInFile, imageHeap); + int result = commitAndCopyMemory(IMAGE_HEAP_BEGIN.get(), imageHeapSizeInFile, imageHeap, nmtData); if (result != CEntryPointErrors.NO_ERROR) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return result; } @@ -101,7 +102,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W UnsignedWord writableBeginPageOffset = UnsignedUtils.roundDown(IMAGE_HEAP_WRITABLE_BEGIN.get().subtract(IMAGE_HEAP_BEGIN.get()), pageSize); if (writableBeginPageOffset.aboveThan(0)) { if (VirtualMemoryProvider.get().protect(imageHeap, writableBeginPageOffset, Access.READ) != 0) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return CEntryPointErrors.PROTECT_HEAP_FAILED; } } @@ -112,7 +113,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Pointer afterWritableBoundary = imageHeap.add(writableEndPageOffset); UnsignedWord afterWritableSize = imageHeapSizeInFile.subtract(writableEndPageOffset); if (VirtualMemoryProvider.get().protect(afterWritableBoundary, afterWritableSize, Access.READ) != 0) { - freeImageHeap(selfReservedHeapBase); + freeImageHeap(selfReservedHeapBase, nmtData); return CEntryPointErrors.PROTECT_HEAP_FAILED; } } @@ -126,8 +127,8 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W } @Uninterruptible(reason = "Called during isolate initialization.") - protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap) { - Pointer actualNewImageHeap = VirtualMemoryProvider.get().commit(newImageHeap, imageHeapSize, Access.READ | Access.WRITE); + protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap, NmtPreImageHeapData nmtData) { + Pointer actualNewImageHeap = VirtualMemoryProvider.get().commit(newImageHeap, imageHeapSize, Access.READ | Access.WRITE, nmtData); if (actualNewImageHeap.isNull() || actualNewImageHeap.notEqual(newImageHeap)) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } @@ -140,13 +141,18 @@ protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHea @Override @Uninterruptible(reason = "Called during isolate tear-down.") public int freeImageHeap(PointerBase heapBase) { + return freeImageHeap(heapBase, WordFactory.nullPointer()); + } + + @Uninterruptible(reason = "Called during isolate tear-down.") + private int freeImageHeap(PointerBase heapBase, NmtPreImageHeapData nmtData) { if (heapBase.isNonNull()) { Pointer addressSpaceStart = (Pointer) heapBase; if (DynamicMethodAddressResolutionHeapSupport.isEnabled()) { addressSpaceStart = addressSpaceStart.subtract(getPreHeapAlignedSizeForDynamicMethodAddressResolver()); } - if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize()) != 0) { + if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize(), nmtData) != 0) { return CEntryPointErrors.FREE_IMAGE_HEAP_FAILED; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java index 41d3f8cf1ebf..87678d23c61e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java @@ -33,6 +33,8 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointCreateIsolateParameters; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; + import jdk.graal.compiler.api.replacements.Fold; /** @@ -52,7 +54,7 @@ static CommittedMemoryProvider get() { * @return zero in case of success, non-zero in case of an error. */ @Uninterruptible(reason = "Still being initialized.") - int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters); + int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters, NmtPreImageHeapData nmtData); /** * Tear down for the current isolate. This must be the last method of this interface diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java index 118ea194d590..35e877acf2c9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java @@ -32,6 +32,7 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; import jdk.graal.compiler.api.replacements.Fold; @@ -71,7 +72,7 @@ static ImageHeapProvider get() { * written. May be null if this value is not required. * @return a result code from {@link CEntryPointErrors}. */ - int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer); + int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtPreImageHeapData nmtData); /** * Disposes an instance of the image heap that was created with this provider. This method must diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java index b09b8bddcd04..6cd1668b0bdc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; public class OSCommittedMemoryProvider extends AbstractCommittedMemoryProvider { @Platforms(Platform.HOSTED_ONLY.class) @@ -49,7 +50,7 @@ public OSCommittedMemoryProvider() { @Override @Uninterruptible(reason = "Still being initialized.") - public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters) { + public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters, NmtPreImageHeapData nmtData) { if (!SubstrateOptions.SpawnIsolates.getValue()) { int result = protectSingleIsolateImageHeap(); if (result == CEntryPointErrors.NO_ERROR) { @@ -57,7 +58,7 @@ public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParam } return result; } - return ImageHeapProvider.get().initialize(nullPointer(), zero(), heapBasePointer, nullPointer()); + return ImageHeapProvider.get().initialize(nullPointer(), zero(), heapBasePointer, nullPointer(), nmtData); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java index 7aa211c1bd41..c7d0fdcf48c0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java @@ -31,6 +31,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.nmt.NmtPreImageHeapData; /** * Primitive operations for low-level virtual memory management. @@ -92,7 +94,9 @@ default UnsignedWord getAlignment() { * @return An {@linkplain #getAlignment aligned} pointer to the beginning of the reserved * address range, or {@link WordFactory#nullPointer()} in case of an error. */ - Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean code); + Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean code, NmtCategory category); + + Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean code, NmtPreImageHeapData nmtData); /** * Map a region of an open file to the specified address range. When {@linkplain Access#WRITE @@ -114,7 +118,9 @@ default UnsignedWord getAlignment() { * @return The start of the mapped address range, or {@link WordFactory#nullPointer()} in case * of an error. */ - Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access); + Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtCategory category); + + Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData); /** * Commit an address range so that physical memory or swap memory can be provisioned for it, and @@ -142,7 +148,10 @@ default UnsignedWord getAlignment() { * @return The start of the committed address range, or {@link WordFactory#nullPointer()} in * case of an error, such as inadequate physical memory. */ - Pointer commit(PointerBase start, UnsignedWord nbytes, int access); + + Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtCategory category); + + Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData); /** * Change the protection of a committed address range, or of a subrange of a committed address @@ -189,4 +198,6 @@ default UnsignedWord getAlignment() { * @return 0 when successful, or a non-zero implementation-specific error code. */ int free(PointerBase start, UnsignedWord nbytes); + + int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData); } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java index 6f633651ae44..c09a334105a1 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java @@ -30,16 +30,23 @@ import static org.junit.Assert.assertTrue; import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; +import org.graalvm.word.UnsignedWord; import org.junit.Test; import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.nmt.NativeMemoryTracking; import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.os.VirtualMemoryProvider; +import com.oracle.svm.core.genscavenge.HeapParameters; public class NativeMemoryTrackingTests { private static final int K = 1024; private static final int M = 1024 * 1024; + private static final UnsignedWord GRANULARITY = VirtualMemoryProvider.get().getGranularity(); + private static final int COMMIT_SIZE = (int) GRANULARITY.rawValue(); + private static final int RESERVE_SIZE = COMMIT_SIZE * 8; @Test public void testMalloc() { @@ -117,4 +124,353 @@ public void testPeakTracking() { private static long getUsedMemory() { return NativeMemoryTracking.singleton().getUsedMemory(NmtCategory.Code); } + + @Test + public void testReserveAndCommitBasic() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + Pointer commitPtr = commit(reservePtr, COMMIT_SIZE); + assertEquals(RESERVE_SIZE, getReserved()); + assertEquals(COMMIT_SIZE, getCommitted()); + + uncommit(commitPtr, COMMIT_SIZE); + assertEquals(0, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testSmallAlignmentReserveAndCommit() { + assertEquals(0, getReserved()); + + Pointer reservePtr = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), GRANULARITY.unsignedDivide(2), false, + NmtCategory.Code); + assertEquals(RESERVE_SIZE, getReserved()); + assertEquals(0, getCommitted()); + + Pointer commitPtr = commit(reservePtr, COMMIT_SIZE * 3); + assertEquals(COMMIT_SIZE * 3, getCommitted()); + + uncommit(commitPtr, COMMIT_SIZE * 3); + assertEquals(0, getCommitted()); + + free(reservePtr, RESERVE_SIZE); + assertEquals(0, getReserved()); + } + + @Test + public void testReserveFree() { + Pointer reservePtr1 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), GRANULARITY, false, + NmtCategory.Code); + assertEquals(RESERVE_SIZE, getReserved()); + + Pointer reservePtr2 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), GRANULARITY, false, + NmtCategory.Code); + assertEquals(RESERVE_SIZE * 2, getReserved()); + + Pointer reservePtr3 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), GRANULARITY, false, + NmtCategory.Code); + assertEquals(RESERVE_SIZE * 3, getReserved()); + + free(reservePtr1, RESERVE_SIZE); + free(reservePtr2, RESERVE_SIZE); + free(reservePtr3, RESERVE_SIZE); + assertEquals(0, getReserved()); + } + + @Test + public void testAlternatingReserveFree() { + assertEquals(0, getReserved()); + + Pointer reservePtr1 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), HeapParameters.getAlignedHeapChunkSize(), false, + NmtCategory.Code); + free(reservePtr1, RESERVE_SIZE); + + Pointer reservePtr2 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), HeapParameters.getAlignedHeapChunkSize(), false, + NmtCategory.Code); + free(reservePtr2, RESERVE_SIZE); + + Pointer reservePtr3 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), HeapParameters.getAlignedHeapChunkSize(), false, + NmtCategory.Code); + free(reservePtr3, RESERVE_SIZE); + + assertEquals(0, getReserved()); + } + + @Test + public void testGappedCommits() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + Pointer commitPtr1 = commit(reservePtr, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + + /* Commit again leaving a big gap between the previous regions. */ + Pointer commitPtr2 = commit(commitPtr1.add(COMMIT_SIZE * 3), COMMIT_SIZE); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + + uncommit(commitPtr2, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + + uncommit(commitPtr1, COMMIT_SIZE); + assertEquals(0, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testCommitUncommitThroughFree() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + commit(reservePtr, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + + /* Commit again leaving a large gap between the previous regions. */ + commit(reservePtr.add(COMMIT_SIZE * 3), COMMIT_SIZE); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testAdjacentCommits() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + Pointer commitPtr1 = commit(reservePtr, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + + /* Commit again leaving a no gap between the previous regions. */ + Pointer commitPtr2 = commit(commitPtr1.add(COMMIT_SIZE), COMMIT_SIZE); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + + uncommit(commitPtr2, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + + uncommit(commitPtr1, COMMIT_SIZE); + assertEquals(0, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testAdjacentAndGappedCommits() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + Pointer commitPtr1 = commit(reservePtr, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + + /* Commit again adjacent to the previous committed region. */ + Pointer commitPtr2 = commit(commitPtr1.add(COMMIT_SIZE), COMMIT_SIZE); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + + /* Commit again leaving a big gap between the previous regions. */ + Pointer commitPtr3 = commit(commitPtr2.add(COMMIT_SIZE * 3), COMMIT_SIZE); + assertEquals(COMMIT_SIZE * 3, getCommitted()); + + uncommit(commitPtr3, COMMIT_SIZE); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + uncommit(commitPtr2, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + uncommit(commitPtr1, COMMIT_SIZE); + assertEquals(0, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testFullyOverlappingCommits() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + Pointer commitPtr1 = commit(reservePtr, COMMIT_SIZE); + long recordedCommittedSize1 = getCommitted(); + assertEquals(COMMIT_SIZE, recordedCommittedSize1); + + /* Commit again completely overlapping the previous committed region. */ + Pointer commitPtr2 = commit(commitPtr1, COMMIT_SIZE); + long recordedCommittedSize2 = getCommitted(); + assertEquals(COMMIT_SIZE, recordedCommittedSize2); + + uncommit(commitPtr2, COMMIT_SIZE); + assertEquals(0, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + /** + * Test committing regions with unordered start addresses. Then uncommit in an unordered way. + * |commitPtr1|commitPtr3| | | |commitPtr2| | | + */ + @Test + public void testUnorderedCommits() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + Pointer commitPtr1 = commit(reservePtr, COMMIT_SIZE); + long recordedCommittedSize1 = getCommitted(); + assertEquals(COMMIT_SIZE, recordedCommittedSize1); + + /* Commit again leaving a big gap between the previous regions. */ + Pointer commitPtr2 = commit(reservePtr.add(COMMIT_SIZE * 5), COMMIT_SIZE); + long recordedCommittedSize2 = getCommitted(); + assertEquals(COMMIT_SIZE * 2, recordedCommittedSize2); + + // Commit again adjacent to the first committed region + Pointer commitPtr3 = commit(commitPtr1.add(COMMIT_SIZE), COMMIT_SIZE); + long recordedCommittedSize3 = getCommitted(); + assertEquals(COMMIT_SIZE * 3, recordedCommittedSize3); + + // Uncommit the previously committed region + uncommit(commitPtr3, COMMIT_SIZE); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + uncommit(commitPtr1, COMMIT_SIZE); + assertEquals(COMMIT_SIZE, getCommitted()); + uncommit(commitPtr2, COMMIT_SIZE); + assertEquals(0, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testPartiallyOverlappingCommits() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + // Commit from within the middle of the reserved region + Pointer commitPtr1 = commit(reservePtr.add(COMMIT_SIZE), COMMIT_SIZE * 2); + long recordedCommittedSize1 = getCommitted(); + assertEquals(COMMIT_SIZE * 2, recordedCommittedSize1); + + // Commit again partially overlapping to the previous committed region + commit(commitPtr1.add(COMMIT_SIZE), COMMIT_SIZE * 2); + long recordedCommittedSize2 = getCommitted(); + assertEquals(COMMIT_SIZE * 3, recordedCommittedSize2); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testPartialUncommit() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + /* Commit from within the middle of the reserved region. */ + Pointer commitPtr1 = commit(reservePtr.add(COMMIT_SIZE), COMMIT_SIZE * 2); + long recordedCommittedSize1 = getCommitted(); + assertEquals(COMMIT_SIZE * 2, recordedCommittedSize1); + + /* + * Uncommit only half of the committed region. Half of the target region to uncommit is + * actually not committed. + */ + uncommit(commitPtr1.add(COMMIT_SIZE), COMMIT_SIZE * 2); + assertEquals(COMMIT_SIZE, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testPartialUncommitOverlappingMultipleRegions() { + Pointer reservePtr = beginVirtualMemoryTestAndReserve(); + + Pointer commitPtr1 = commit(reservePtr, COMMIT_SIZE * 2); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + + /* Commit again adjacent to the previous region */ + commit(commitPtr1.add(COMMIT_SIZE * 2), COMMIT_SIZE * 2); + assertEquals(COMMIT_SIZE * 4, getCommitted()); + + /* Uncommit a region overlapping both previously committed regions. */ + uncommit(commitPtr1.add(COMMIT_SIZE), COMMIT_SIZE * 2); + assertEquals(COMMIT_SIZE * 2, getCommitted()); + + endVirtualMemoryTestAndFree(reservePtr); + } + + @Test + public void testReservedPeak() { + assertEquals("Test should start with no memory already allocated in the test category.", 0, getReserved()); + assertEquals("Test should start with no memory already allocated in the test category.", 0, getCommitted()); + + long initialPeak = NativeMemoryTracking.singleton().getPeakReservedByCategory(NmtCategory.Code); + + Pointer reservePtr1 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(initialPeak), GRANULARITY, false, + NmtCategory.Code); + assertEquals(initialPeak, getReserved()); + + Pointer reservePtr2 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(initialPeak), GRANULARITY, false, + NmtCategory.Code); + long peakReserved = getReserved(); + assertEquals(peakReserved, NativeMemoryTracking.singleton().getPeakReservedByCategory(NmtCategory.Code)); + + free(reservePtr1, initialPeak); + free(reservePtr2, initialPeak); + + assertEquals(peakReserved, NativeMemoryTracking.singleton().getPeakReservedByCategory(NmtCategory.Code)); + assertEquals(0, getReserved()); + } + + @Test + public void testCommittedPeak() { + long initialPeak = NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code); + long largeReserveSize = initialPeak * 2; + int largeCommitSize = (int) initialPeak; + Pointer reservePtr = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(largeReserveSize), HeapParameters.getAlignedHeapChunkSize(), false, + NmtCategory.Code); + + Pointer commitPtr1 = commit(reservePtr, largeCommitSize); + assertEquals(getCommitted(), NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code)); + + Pointer commitPtr2 = commit(commitPtr1.add(largeCommitSize), largeCommitSize); + long recordedCommittedSize2 = getCommitted(); + assertEquals(recordedCommittedSize2, NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code)); + + uncommit(commitPtr1, largeCommitSize); + uncommit(commitPtr2, largeCommitSize); + assertEquals(0, getCommitted()); + assertEquals(recordedCommittedSize2, NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code)); + + free(reservePtr, largeReserveSize); + assertEquals(0, getReserved()); + assertEquals(0, getCommitted()); + } + + private static Pointer commit(PointerBase base, long size) { + Pointer result = VirtualMemoryProvider.get().commit(base, WordFactory.unsigned(size), 0, NmtCategory.Code); + assertTrue("Commit operation failed.", result.isNonNull()); + return result; + } + + private static void uncommit(PointerBase start, long nbytes) { + int result = VirtualMemoryProvider.get().uncommit(start, WordFactory.unsigned(nbytes)); + assertEquals("Uncommit operation failed.", 0, result); + } + + private static void free(PointerBase start, long nbytes) { + int result = VirtualMemoryProvider.get().free(start, WordFactory.unsigned(nbytes)); + assertEquals("Free operation failed.", 0, result); + } + + private static long getReserved() { + return NativeMemoryTracking.singleton().getReservedByCategory(NmtCategory.Code); + } + + private static long getCommitted() { + return NativeMemoryTracking.singleton().getCommittedByCategory(NmtCategory.Code); + } + + private static Pointer beginVirtualMemoryTestAndReserve() { + assertEquals("Test should start with no memory already allocated in the test category.", 0, getReserved()); + assertEquals("Test should start with no memory already allocated in the test category.", 0, getCommitted()); + + /* Reserve some memory. */ + Pointer reservePtr = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(RESERVE_SIZE), HeapParameters.getAlignedHeapChunkSize(), false, + NmtCategory.Code); + assertEquals(RESERVE_SIZE, getReserved()); + assertEquals(0, getCommitted()); + return reservePtr; + } + + private static void endVirtualMemoryTestAndFree(Pointer reservePtr) { + /* Free the reserved region, which should also uncommit contained committed regions. */ + free(reservePtr, RESERVE_SIZE); + assertEquals(0, getReserved()); + assertEquals(0, getCommitted()); + + } } From 555459b64f305f117ea77f4dbcb292128925cf75 Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Mon, 15 Jul 2024 12:03:10 -0400 Subject: [PATCH 2/3] windows instrumentation --- .../windows/WindowsVirtualMemoryProvider.java | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java index 989dfc6498df..05960d98b23d 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java @@ -47,7 +47,9 @@ import com.oracle.svm.core.c.function.CEntryPointActions; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.nmt.NativeMemoryTracking; import com.oracle.svm.core.nmt.NmtPreImageHeapData; +import com.oracle.svm.core.nmt.NmtPreImageHeapDataAccess; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.PointerUtils; import com.oracle.svm.core.util.UnsignedUtils; @@ -56,6 +58,7 @@ import com.oracle.svm.core.windows.headers.SysinfoAPI; import com.oracle.svm.core.windows.headers.WinBase; import com.oracle.svm.core.windows.headers.WinBase.HANDLE; +import com.oracle.svm.core.VMInspectionOptions; @AutomaticallyRegisteredImageSingleton(VirtualMemoryProvider.class) public class WindowsVirtualMemoryProvider implements VirtualMemoryProvider { @@ -160,6 +163,13 @@ private static Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boo * placeholders. This effectively makes the use of placeholders transparent. */ replacePlaceholder(reservedPlaceholder, nbytes); + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackReserve(reservedPlaceholder, nbytes, category); + } else { + NmtPreImageHeapDataAccess.enqueueReserve(nmtData, reservedPlaceholder, nbytes, category); + } + } return reservedPlaceholder; } @@ -174,7 +184,16 @@ private static Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boo if (reserved.isNull()) { return WordFactory.nullPointer(); } - return requiredAlignment.equal(UNALIGNED) ? reserved : PointerUtils.roundUp(reserved, requiredAlignment); + + Pointer addr = requiredAlignment.equal(UNALIGNED) ? reserved : PointerUtils.roundUp(reserved, requiredAlignment); + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackReserve(addr, nbytes.add(requiredAlignment), category); + } else { + NmtPreImageHeapDataAccess.enqueueReserve(nmtData, addr, nbytes.add(requiredAlignment), category); + } + } + return addr; } private static final int MEM_RESERVE_PLACEHOLDER = 0x00040000; @@ -338,6 +357,12 @@ private Pointer mapFile0(PointerBase start, UnsignedWord nbytes, WordBase fileHa if (fileView.isNull()) { /* Restore a normal allocation as the caller is unaware of placeholders. */ replacePlaceholder(start, nbytes); + } else if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackCommit(start, nbytes, category); + } else { + NmtPreImageHeapDataAccess.enqueueCommit(nmtData, start, nbytes, category); + } } return fileView; } @@ -398,7 +423,15 @@ private Pointer commit0(PointerBase start, UnsignedWord nbytes, int access, NmtP * VirtualAlloc only guarantees the zeroing for freshly committed pages (i.e., the content * of pages that were already committed earlier won't be touched). */ - return MemoryAPI.VirtualAlloc(start, nbytes, MemoryAPI.MEM_COMMIT(), accessAsProt(access)); + Pointer addr = MemoryAPI.VirtualAlloc(start, nbytes, MemoryAPI.MEM_COMMIT(), accessAsProt(access)); + if (addr.isNonNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + if (nmtData.isNull()) { + NativeMemoryTracking.singleton().trackCommit(start, nbytes, category); + } else { + NmtPreImageHeapDataAccess.enqueueCommit(nmtData, start, nbytes, category); + } + } + return addr; } @Override @@ -421,7 +454,13 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) { } int result = MemoryAPI.VirtualFree(start, nbytes, MemoryAPI.MEM_DECOMMIT()); - return (result != 0) ? 0 : -1; + if (result != 0) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().trackUncommit(start, nbytes); + } + return 0; + } + return -1; } @Override @@ -452,6 +491,10 @@ public int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtD return -1; } end = ((Pointer) memoryInfo.AllocationBase()).subtract(1); + + if (nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().trackFree(start); + } } return 0; } From 8f659164e8f5717c9bc85e42919b54f7ca2a978e Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Mon, 15 Jul 2024 15:14:34 -0400 Subject: [PATCH 3/3] Account for vmem in JFR events and NMT report. Improve NMT report format. --- .../EveryChunkNativePeriodicEvents.java | 12 +- .../svm/core/nmt/NativeMemoryTracking.java | 113 ++++++++++++------ .../svm/core/nmt/NmtVirtualMemoryInfo.java | 4 + .../test/nmt/NativeMemoryTrackingTests.java | 30 ++--- 4 files changed, 101 insertions(+), 58 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java index 85bdfceee5bd..a8ef37e78fb2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java @@ -124,20 +124,20 @@ private static void emitNativeMemoryTrackingEvents() { private static void emitNmtPeakEvents() { NativeMemoryUsageTotalPeakEvent nmtTotalPeakEvent = new NativeMemoryUsageTotalPeakEvent(); - long totalPeakUsed = NativeMemoryTracking.singleton().getPeakTotalUsedMemory(); + long totalPeakUsed = NativeMemoryTracking.singleton().getPeakTotalMallocMemory(); nmtTotalPeakEvent.peakCommitted = totalPeakUsed; nmtTotalPeakEvent.peakReserved = totalPeakUsed; - nmtTotalPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtTotalPeakUsage(); + nmtTotalPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtPeakTotalMallocMemory(); nmtTotalPeakEvent.commit(); for (NmtCategory nmtCategory : NmtCategory.values()) { NativeMemoryUsagePeakEvent nmtPeakEvent = new NativeMemoryUsagePeakEvent(); nmtPeakEvent.type = nmtCategory.getName(); - long peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(nmtCategory); + long peakUsed = NativeMemoryTracking.singleton().getPeakMallocMemory(nmtCategory); nmtPeakEvent.peakCommitted = peakUsed; nmtPeakEvent.peakReserved = peakUsed; - nmtPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtPeakUsage(nmtCategory); + nmtPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtPeakMallocMemory(nmtCategory); nmtPeakEvent.commit(); } } @@ -150,7 +150,7 @@ private static void emitJdkNmtEvents(NmtCategory[] nmtCategories) { if (JfrEvent.NativeMemoryUsage.shouldEmit()) { for (NmtCategory nmtCategory : nmtCategories) { - long usedMemory = NativeMemoryTracking.singleton().getUsedMemory(nmtCategory); + long usedMemory = NativeMemoryTracking.singleton().getMallocMemory(nmtCategory); JfrNativeEventWriter.beginSmallEvent(data, JfrEvent.NativeMemoryUsage); JfrNativeEventWriter.putLong(data, timestamp); @@ -162,7 +162,7 @@ private static void emitJdkNmtEvents(NmtCategory[] nmtCategories) { } if (JfrEvent.NativeMemoryUsageTotal.shouldEmit()) { - long totalUsedMemory = NativeMemoryTracking.singleton().getTotalUsedMemory(); + long totalUsedMemory = NativeMemoryTracking.singleton().getTotalMallocMemory(); JfrNativeEventWriter.beginSmallEvent(data, JfrEvent.NativeMemoryUsageTotal); JfrNativeEventWriter.putLong(data, timestamp); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java index 13187ca1b705..492edc7521f9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -57,6 +57,7 @@ public class NativeMemoryTracking { private static final UnsignedWord ALIGNMENT = WordFactory.unsigned(16); private static final int MAGIC = 0xF0F1F2F3; + private static final long KB = 1024; private final NmtMallocMemoryInfo[] mallocCategories; private final NmtVirtualMemoryInfo[] virtualCategories; @@ -208,54 +209,81 @@ void recordFree(UnsignedWord size, NmtCategory category) { } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getUsedMemory(NmtCategory category) { + public long getMallocMemory(NmtCategory category) { return getMallocInfo(category).getUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getPeakUsedMemory(NmtCategory category) { + public long getMallocCount(NmtCategory category) { + return getMallocInfo(category).getCount(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakMallocMemory(NmtCategory category) { return getMallocInfo(category).getPeakUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getCountAtPeakUsage(NmtCategory category) { + public long getCountAtPeakMallocMemory(NmtCategory category) { return getMallocInfo(category).getCountAtPeakUsage(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getTotalCount() { + public long getTotalMallocCount() { return mallocTotal.getCount(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getTotalUsedMemory() { + public long getTotalMallocMemory() { return mallocTotal.getUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getPeakTotalUsedMemory() { + public long getPeakTotalMallocMemory() { return mallocTotal.getPeakUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getCountAtTotalPeakUsage() { + public long getCountAtPeakTotalMallocMemory() { return mallocTotal.getCountAtPeakUsage(); } - public long getReservedByCategory(NmtCategory category) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getReservedVirtualMemory(NmtCategory category) { return NativeMemoryTracking.singleton().getVirtualInfo(category).getReservedSize(); } - public long getCommittedByCategory(NmtCategory category) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getCommittedVirtualMemory(NmtCategory category) { return getVirtualInfo(category).getCommittedSize(); } - public long getPeakCommittedByCategory(NmtCategory category) { + public long getPeakReservedVirtualMemory(NmtCategory category) { + return getVirtualInfo(category).getPeakReservedSize(); + } + + public long getPeakCommittedVirtualMemory(NmtCategory category) { return getVirtualInfo(category).getPeakCommittedSize(); } - public long getPeakReservedByCategory(NmtCategory category) { - return getVirtualInfo(category).getPeakReservedSize(); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getTotalReservedVirtualMemory() { + return virtualTotal.getReservedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getTotalCommittedVirtualMemory() { + return virtualTotal.getCommittedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakTotalReservedVirtualMemory() { + return virtualTotal.getPeakReservedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakTotalCommittedVirtualMemory() { + return virtualTotal.getPeakCommittedSize(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -289,32 +317,43 @@ public static RuntimeSupport.Hook shutdownHook() { private void printStatistics() { if (VMInspectionOptions.PrintNMTStatistics.getValue()) { - System.out.println(); - System.out.println("Native memory tracking"); - System.out.println(" Peak total used memory: " + getPeakTotalUsedMemory() + " bytes"); - System.out.println(" Peak Total committed memory: " + virtualTotal.getPeakCommittedSize() + " bytes"); - System.out.println(" Peak Total reserved memory: " + virtualTotal.getPeakReservedSize() + " bytes"); - System.out.println(" Total alive allocations at peak usage: " + getCountAtTotalPeakUsage()); - System.out.println(" Total used memory: " + getTotalUsedMemory() + " bytes"); - System.out.println(" Total alive allocations: " + getTotalCount()); - System.out.println(" Total committed memory: " + virtualTotal.getCommittedSize() + " bytes"); - System.out.println(" Total reserved memory: " + virtualTotal.getReservedSize() + " bytes"); - - for (int i = 0; i < NmtCategory.values().length; i++) { - String name = NmtCategory.values()[i].getName(); - NmtMallocMemoryInfo info = getMallocInfo(i); - NmtVirtualMemoryInfo vMemInfo = getVirtualInfo(i); - - System.out.println(" " + name + " peak used memory: " + info.getPeakUsed() + " bytes"); - System.out.println(" " + name + " alive allocations at peak: " + info.getCountAtPeakUsage()); - System.out.println(" " + name + " currently used memory: " + info.getUsed() + " bytes"); - System.out.println(" " + name + " currently alive allocations: " + info.getCount()); - System.out.println(" " + name + " committed memory: " + vMemInfo.getCommittedSize()); - System.out.println(" " + name + " reserved memory: " + vMemInfo.getReservedSize()); - System.out.println(" " + name + " peak committed memory: " + vMemInfo.getPeakCommittedSize()); - System.out.println(" " + name + " peak reserved memory: " + vMemInfo.getPeakReservedSize()); - } + System.out.println(generateReportString()); + } + } + + public String generateReportString() { + + StringBuilder stringBuilder = new StringBuilder(3000); + + stringBuilder.append("\n"); + stringBuilder.append("Native memory tracking").append("\n\n"); + + stringBuilder.append("Total").append("\n"); + long reservedTotal = (getTotalReservedVirtualMemory() + getTotalMallocMemory()) / KB; + long committedTotal = (getTotalCommittedVirtualMemory() + getTotalMallocMemory()) / KB; + stringBuilder.append("\t").append("(reserved=").append(reservedTotal).append("KB, committed=").append(committedTotal).append("KB)").append("\n"); + stringBuilder.append("\t").append("(malloc=").append(getTotalMallocMemory() / KB).append("KB, count=").append(getTotalMallocCount()).append(")").append("\n"); + stringBuilder.append("\t").append("(peak malloc=").append(getPeakTotalMallocMemory() / KB).append("KB, count at peak=").append(getCountAtPeakTotalMallocMemory()).append(")").append("\n"); + stringBuilder.append("\t").append("(mmap: reserved=").append(getTotalReservedVirtualMemory() / KB).append("KB, committed=").append(getTotalCommittedVirtualMemory() / KB).append("KB)") + .append("\n"); + stringBuilder.append("\t").append("(mmap: peak reserved=").append(getPeakTotalReservedVirtualMemory() / KB).append("KB, peak committed=").append(getPeakTotalCommittedVirtualMemory() / KB) + .append("KB)").append("\n"); + + for (int i = 0; i < NmtCategory.values().length; i++) { + NmtCategory category = NmtCategory.values()[i]; + stringBuilder.append(category.getName()).append("\n"); + long reserved = (getReservedVirtualMemory(category) + getMallocMemory(category)) / KB; + long committed = (getCommittedVirtualMemory(category) + getMallocMemory(category)) / KB; + stringBuilder.append("\t").append("(reserved=").append(reserved).append("KB, committed=").append(committed).append("KB)").append("\n"); + stringBuilder.append("\t").append("(malloc=").append(getMallocMemory(category) / KB).append("KB, count=").append(getMallocCount(category)).append(")").append("\n"); + stringBuilder.append("\t").append("(peak malloc=").append(getPeakMallocMemory(category) / KB).append("KB, count at peak=").append(getCountAtPeakMallocMemory(category)).append(")") + .append("\n"); + stringBuilder.append("\t").append("(mmap: reserved=").append(getReservedVirtualMemory(category) / KB).append("KB, committed=").append(getCommittedVirtualMemory(category) / KB) + .append("KB)").append("\n"); + stringBuilder.append("\t").append("(mmap: peak reserved=").append(getPeakReservedVirtualMemory(category) / KB).append("KB, peak committed=") + .append(getPeakCommittedVirtualMemory(category) / KB).append("KB)").append("\n"); } + return stringBuilder.toString(); } private static void teardown() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java index 5a1970f4de8b..2b5843009214 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java @@ -81,18 +81,22 @@ private static void updatePeak(long newSize, AtomicLong peakToUpdate) { } } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) long getReservedSize() { return reservedSize.get(); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) long getCommittedSize() { return committedSize.get(); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) long getPeakReservedSize() { return peakReservedSize.get(); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) long getPeakCommittedSize() { return peakCommittedSize.get(); } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java index c09a334105a1..0565073044c6 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java @@ -94,11 +94,11 @@ public void testPeakTracking() { assertEquals(0, getUsedMemory()); Pointer ptr1 = NativeMemory.malloc(M, NmtCategory.Code); - long peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code); + long peakUsed = NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code); assertEquals(M, peakUsed); Pointer ptr2 = NativeMemory.malloc(M, NmtCategory.Code); - peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code); + peakUsed = NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code); assertEquals(2 * M, peakUsed); NativeMemory.free(ptr1); @@ -108,21 +108,21 @@ public void testPeakTracking() { ptr2 = WordFactory.nullPointer(); assertEquals(0, getUsedMemory()); - assertEquals(2 * M, NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code)); + assertEquals(2 * M, NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code)); Pointer ptr3 = NativeMemory.malloc(3 * M, NmtCategory.Code); - peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code); + peakUsed = NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code); assertEquals(3 * M, peakUsed); NativeMemory.free(ptr3); ptr3 = WordFactory.nullPointer(); assertEquals(0, getUsedMemory()); - assertEquals(3 * M, NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code)); + assertEquals(3 * M, NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code)); } private static long getUsedMemory() { - return NativeMemoryTracking.singleton().getUsedMemory(NmtCategory.Code); + return NativeMemoryTracking.singleton().getMallocMemory(NmtCategory.Code); } @Test @@ -387,7 +387,7 @@ public void testReservedPeak() { assertEquals("Test should start with no memory already allocated in the test category.", 0, getReserved()); assertEquals("Test should start with no memory already allocated in the test category.", 0, getCommitted()); - long initialPeak = NativeMemoryTracking.singleton().getPeakReservedByCategory(NmtCategory.Code); + long initialPeak = NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(NmtCategory.Code); Pointer reservePtr1 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(initialPeak), GRANULARITY, false, NmtCategory.Code); @@ -396,34 +396,34 @@ public void testReservedPeak() { Pointer reservePtr2 = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(initialPeak), GRANULARITY, false, NmtCategory.Code); long peakReserved = getReserved(); - assertEquals(peakReserved, NativeMemoryTracking.singleton().getPeakReservedByCategory(NmtCategory.Code)); + assertEquals(peakReserved, NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(NmtCategory.Code)); free(reservePtr1, initialPeak); free(reservePtr2, initialPeak); - assertEquals(peakReserved, NativeMemoryTracking.singleton().getPeakReservedByCategory(NmtCategory.Code)); + assertEquals(peakReserved, NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(NmtCategory.Code)); assertEquals(0, getReserved()); } @Test public void testCommittedPeak() { - long initialPeak = NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code); + long initialPeak = NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.Code); long largeReserveSize = initialPeak * 2; int largeCommitSize = (int) initialPeak; Pointer reservePtr = VirtualMemoryProvider.get().reserve(WordFactory.unsigned(largeReserveSize), HeapParameters.getAlignedHeapChunkSize(), false, NmtCategory.Code); Pointer commitPtr1 = commit(reservePtr, largeCommitSize); - assertEquals(getCommitted(), NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code)); + assertEquals(getCommitted(), NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.Code)); Pointer commitPtr2 = commit(commitPtr1.add(largeCommitSize), largeCommitSize); long recordedCommittedSize2 = getCommitted(); - assertEquals(recordedCommittedSize2, NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code)); + assertEquals(recordedCommittedSize2, NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.Code)); uncommit(commitPtr1, largeCommitSize); uncommit(commitPtr2, largeCommitSize); assertEquals(0, getCommitted()); - assertEquals(recordedCommittedSize2, NativeMemoryTracking.singleton().getPeakCommittedByCategory(NmtCategory.Code)); + assertEquals(recordedCommittedSize2, NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.Code)); free(reservePtr, largeReserveSize); assertEquals(0, getReserved()); @@ -447,11 +447,11 @@ private static void free(PointerBase start, long nbytes) { } private static long getReserved() { - return NativeMemoryTracking.singleton().getReservedByCategory(NmtCategory.Code); + return NativeMemoryTracking.singleton().getReservedVirtualMemory(NmtCategory.Code); } private static long getCommitted() { - return NativeMemoryTracking.singleton().getCommittedByCategory(NmtCategory.Code); + return NativeMemoryTracking.singleton().getCommittedVirtualMemory(NmtCategory.Code); } private static Pointer beginVirtualMemoryTestAndReserve() {