Skip to content

[GR-56600] Track virtual memory with native memory tracking (NMT) #8630

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[GR-56600] Track virtual memory with native memory tracking (NMT) #8630

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7fc077f
add virtual memory tracking
roberttoyonaga Nov 27, 2023
555459b
windows instrumentation
roberttoyonaga Jul 15, 2024
8f65916
Account for vmem in JFR events and NMT report. Improve NMT report for…
roberttoyonaga Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...ratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.graalvm.word.UnsignedWord;
import org.graalvm.word.WordFactory;

import com.oracle.svm.core.nmt.NmtCategory;
import com.oracle.svm.core.os.VirtualMemoryProvider;
import com.oracle.svm.core.util.VMError;

Expand Down Expand Up @@ -111,7 +112,7 @@ private Pointer prepareTrampolines(PinnedObject mhsArray, PinnedObject stubsArra
VirtualMemoryProvider memoryProvider = VirtualMemoryProvider.get();
UnsignedWord pageSize = allocationSize();
/* We request a specific alignment to guarantee correctness of getAllocationBase */
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE);
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE, NmtCategory.Internal);
if (page.isNull()) {
throw new OutOfMemoryError("Could not allocate memory for trampolines.");
}
Expand Down
117 changes: 108 additions & 9 deletions ...c/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,15 @@
import com.oracle.svm.core.c.CGlobalDataFactory;
import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
import com.oracle.svm.core.feature.InternalFeature;
import com.oracle.svm.core.nmt.NativeMemoryTracking;
import com.oracle.svm.core.nmt.NmtCategory;
import com.oracle.svm.core.nmt.NmtPreImageHeapData;
import com.oracle.svm.core.nmt.NmtPreImageHeapDataAccess;
import com.oracle.svm.core.os.VirtualMemoryProvider;
import com.oracle.svm.core.posix.headers.Unistd;
import com.oracle.svm.core.util.PointerUtils;
import com.oracle.svm.core.util.UnsignedUtils;
import com.oracle.svm.core.VMInspectionOptions;

@AutomaticallyRegisteredFeature
class PosixVirtualMemoryProviderFeature implements InternalFeature {
Expand Down Expand Up @@ -109,7 +114,18 @@ public UnsignedWord getGranularity() {

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) {
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData) {
return reserve0(nbytes, alignment, executable, nmtData, NmtCategory.ImageHeap);
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtCategory category) {
return reserve0(nbytes, alignment, executable, WordFactory.nullPointer(), category);
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData, NmtCategory category) {
if (nbytes.equal(0)) {
return WordFactory.nullPointer();
}
Expand All @@ -128,25 +144,54 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec
return nullPointer();
}
if (!customAlignment) {
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackReserve(mappingBegin, mappingSize, category);
} else {
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, mappingBegin, mappingSize, category);
}
}
return mappingBegin;
}
UnsignedWord unmappedSize = WordFactory.zero();
Pointer begin = PointerUtils.roundUp(mappingBegin, alignment);
UnsignedWord clippedBegin = begin.subtract(mappingBegin);
if (clippedBegin.aboveOrEqual(granularity)) {
munmap(mappingBegin, UnsignedUtils.roundDown(clippedBegin, granularity));
UnsignedWord unmapSize = UnsignedUtils.roundDown(clippedBegin, granularity);
munmap(mappingBegin, unmapSize);
unmappedSize = unmappedSize.add(unmapSize);
}
Pointer mappingEnd = mappingBegin.add(mappingSize);
UnsignedWord clippedEnd = mappingEnd.subtract(begin.add(nbytes));
if (clippedEnd.aboveOrEqual(granularity)) {
UnsignedWord rounded = UnsignedUtils.roundDown(clippedEnd, granularity);
munmap(mappingEnd.subtract(rounded), rounded);
unmappedSize = unmappedSize.add(rounded);
}
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackReserve(begin, mappingSize.subtract(unmappedSize), category);
} else {
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, begin, mappingSize.subtract(unmappedSize), category);
}
}
return begin;
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) {
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData) {
return mapFile0(start, nbytes, fileHandle, offset, access, nmtData, NmtCategory.ImageHeap);
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtCategory category) {
return mapFile0(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer(), category);
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private Pointer mapFile0(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
return WordFactory.nullPointer();
}
Expand All @@ -157,12 +202,27 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand
}
int fd = (int) fileHandle.rawValue();
Pointer result = mmap(start, nbytes, accessAsProt(access), flags, fd, offset.rawValue());
return result.notEqual(MAP_FAILED()) ? result : WordFactory.nullPointer();
if (result.notEqual(MAP_FAILED())) {
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
return result;
}
return WordFactory.nullPointer();
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData) {
return commit0(start, nbytes, access, nmtData, NmtCategory.ImageHeap);
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtCategory category) {
return commit0(start, nbytes, access, WordFactory.nullPointer(), category);
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private Pointer commit0(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
return WordFactory.nullPointer();
}
Expand All @@ -177,7 +237,31 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
}
/* The memory returned by mmap is guaranteed to be zeroed. */
final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET);
return result.notEqual(MAP_FAILED()) ? result : nullPointer();
if (result.notEqual(MAP_FAILED())) {
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
return result;
}
return nullPointer();
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private static void trackCommitAndMaybeReserve(PointerBase baseAddr, PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData, NmtCategory category) {
if (!VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
return;
}
// Account for possible reserve before commit
if (start.isNull()) {
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackReserve(baseAddr, nbytes, category);
} else {
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, baseAddr, nbytes, category);
}
}
if (nmtData.isNull()) {
NativeMemoryTracking.singleton().trackCommit(baseAddr, nbytes, category);
} else {
NmtPreImageHeapDataAccess.enqueueCommit(nmtData, baseAddr, nbytes, category);
}
}

@Override
Expand All @@ -196,22 +280,37 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) {
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
return -1;
}

final Pointer result = mmap(start, nbytes, PROT_NONE(), MAP_FIXED() | MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET);
return result.notEqual(MAP_FAILED()) ? 0 : -1;
if (result.notEqual(MAP_FAILED())) {
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
NativeMemoryTracking.singleton().trackUncommit(start, nbytes);
}
return 0;
}
return -1;
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public int free(PointerBase start, UnsignedWord nbytes) {
return free(start, nbytes, WordFactory.nullPointer());
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData) {
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
return -1;
}

UnsignedWord granularity = getGranularity();
Pointer mappingBegin = PointerUtils.roundDown(start, granularity);
UnsignedWord mappingSize = UnsignedUtils.roundUp(nbytes, granularity);
return munmap(mappingBegin, mappingSize);
int ret = munmap(mappingBegin, mappingSize);
if (ret == 0 && nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
NativeMemoryTracking.singleton().trackFree(start);
}
return ret;
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
Expand Down
Loading