Skip to content

Commit a2f9c39

Browse files
roberttoyonagaroberttoyonaga
committed
add virtual memory tracking
1 parent e96f9a5 commit a2f9c39

25 files changed

+1850
-83
lines changed

‎substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.graalvm.word.WordFactory;
3838

3939
import com.oracle.svm.core.heap.OutOfMemoryUtil;
40+
import com.oracle.svm.core.nmt.NmtCategory;
4041
import com.oracle.svm.core.os.VirtualMemoryProvider;
4142
import com.oracle.svm.core.util.VMError;
4243

@@ -112,7 +113,7 @@ private Pointer prepareTrampolines(PinnedObject mhsArray, PinnedObject stubsArra
112113
VirtualMemoryProvider memoryProvider = VirtualMemoryProvider.get();
113114
UnsignedWord pageSize = allocationSize();
114115
/* We request a specific alignment to guarantee correctness of getAllocationBase */
115-
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE);
116+
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE, NmtCategory.Internal);
116117
if (page.isNull()) {
117118
throw OutOfMemoryUtil.reportOutOfMemoryError(new OutOfMemoryError("Could not allocate memory for trampolines."));
118119
}

‎substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java‎

Lines changed: 108 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,15 @@
5454
import com.oracle.svm.core.c.CGlobalDataFactory;
5555
import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
5656
import com.oracle.svm.core.feature.InternalFeature;
57+
import com.oracle.svm.core.nmt.NativeMemoryTracking;
58+
import com.oracle.svm.core.nmt.NmtCategory;
59+
import com.oracle.svm.core.nmt.NmtPreImageHeapData;
60+
import com.oracle.svm.core.nmt.NmtPreImageHeapDataAccess;
5761
import com.oracle.svm.core.os.VirtualMemoryProvider;
5862
import com.oracle.svm.core.posix.headers.Unistd;
5963
import com.oracle.svm.core.util.PointerUtils;
6064
import com.oracle.svm.core.util.UnsignedUtils;
65+
import com.oracle.svm.core.VMInspectionOptions;
6166

6267
@AutomaticallyRegisteredFeature
6368
class PosixVirtualMemoryProviderFeature implements InternalFeature {
@@ -109,7 +114,18 @@ public UnsignedWord getGranularity() {
109114

110115
@Override
111116
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
112-
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) {
117+
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData) {
118+
return reserve0(nbytes, alignment, executable, nmtData, NmtCategory.ImageHeap);
119+
}
120+
121+
@Override
122+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
123+
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtCategory category) {
124+
return reserve0(nbytes, alignment, executable, WordFactory.nullPointer(), category);
125+
}
126+
127+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
128+
private Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData, NmtCategory category) {
113129
if (nbytes.equal(0)) {
114130
return WordFactory.nullPointer();
115131
}
@@ -128,25 +144,54 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec
128144
return nullPointer();
129145
}
130146
if (!customAlignment) {
147+
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
148+
if (nmtData.isNull()) {
149+
NativeMemoryTracking.singleton().trackReserve(mappingBegin, mappingSize, category);
150+
} else {
151+
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, mappingBegin, mappingSize, category);
152+
}
153+
}
131154
return mappingBegin;
132155
}
156+
UnsignedWord unmappedSize = WordFactory.zero();
133157
Pointer begin = PointerUtils.roundUp(mappingBegin, alignment);
134158
UnsignedWord clippedBegin = begin.subtract(mappingBegin);
135159
if (clippedBegin.aboveOrEqual(granularity)) {
136-
munmap(mappingBegin, UnsignedUtils.roundDown(clippedBegin, granularity));
160+
UnsignedWord unmapSize = UnsignedUtils.roundDown(clippedBegin, granularity);
161+
munmap(mappingBegin, unmapSize);
162+
unmappedSize = unmappedSize.add(unmapSize);
137163
}
138164
Pointer mappingEnd = mappingBegin.add(mappingSize);
139165
UnsignedWord clippedEnd = mappingEnd.subtract(begin.add(nbytes));
140166
if (clippedEnd.aboveOrEqual(granularity)) {
141167
UnsignedWord rounded = UnsignedUtils.roundDown(clippedEnd, granularity);
142168
munmap(mappingEnd.subtract(rounded), rounded);
169+
unmappedSize = unmappedSize.add(rounded);
170+
}
171+
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
172+
if (nmtData.isNull()) {
173+
NativeMemoryTracking.singleton().trackReserve(begin, mappingSize.subtract(unmappedSize), category);
174+
} else {
175+
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, begin, mappingSize.subtract(unmappedSize), category);
176+
}
143177
}
144178
return begin;
145179
}
146180

147181
@Override
148182
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
149-
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) {
183+
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData) {
184+
return mapFile0(start, nbytes, fileHandle, offset, access, nmtData, NmtCategory.ImageHeap);
185+
}
186+
187+
@Override
188+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
189+
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtCategory category) {
190+
return mapFile0(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer(), category);
191+
}
192+
193+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
194+
private Pointer mapFile0(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
150195
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
151196
return WordFactory.nullPointer();
152197
}
@@ -157,12 +202,27 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand
157202
}
158203
int fd = (int) fileHandle.rawValue();
159204
Pointer result = mmap(start, nbytes, accessAsProt(access), flags, fd, offset.rawValue());
160-
return result.notEqual(MAP_FAILED()) ? result : WordFactory.nullPointer();
205+
if (result.notEqual(MAP_FAILED())) {
206+
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
207+
return result;
208+
}
209+
return WordFactory.nullPointer();
161210
}
162211

163212
@Override
164213
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
165-
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
214+
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData) {
215+
return commit0(start, nbytes, access, nmtData, NmtCategory.ImageHeap);
216+
}
217+
218+
@Override
219+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
220+
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtCategory category) {
221+
return commit0(start, nbytes, access, WordFactory.nullPointer(), category);
222+
}
223+
224+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
225+
private Pointer commit0(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
166226
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
167227
return WordFactory.nullPointer();
168228
}
@@ -177,7 +237,31 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
177237
}
178238
/* The memory returned by mmap is guaranteed to be zeroed. */
179239
final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET);
180-
return result.notEqual(MAP_FAILED()) ? result : nullPointer();
240+
if (result.notEqual(MAP_FAILED())) {
241+
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
242+
return result;
243+
}
244+
return nullPointer();
245+
}
246+
247+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
248+
private static void trackCommitAndMaybeReserve(PointerBase baseAddr, PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData, NmtCategory category) {
249+
if (!VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
250+
return;
251+
}
252+
// Account for possible reserve before commit
253+
if (start.isNull()) {
254+
if (nmtData.isNull()) {
255+
NativeMemoryTracking.singleton().trackReserve(baseAddr, nbytes, category);
256+
} else {
257+
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, baseAddr, nbytes, category);
258+
}
259+
}
260+
if (nmtData.isNull()) {
261+
NativeMemoryTracking.singleton().trackCommit(baseAddr, nbytes, category);
262+
} else {
263+
NmtPreImageHeapDataAccess.enqueueCommit(nmtData, baseAddr, nbytes, category);
264+
}
181265
}
182266

183267
@Override
@@ -196,22 +280,37 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) {
196280
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
197281
return -1;
198282
}
199-
200283
final Pointer result = mmap(start, nbytes, PROT_NONE(), MAP_FIXED() | MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET);
201-
return result.notEqual(MAP_FAILED()) ? 0 : -1;
284+
if (result.notEqual(MAP_FAILED())) {
285+
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
286+
NativeMemoryTracking.singleton().trackUncommit(start, nbytes);
287+
}
288+
return 0;
289+
}
290+
return -1;
202291
}
203292

204293
@Override
205294
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
206295
public int free(PointerBase start, UnsignedWord nbytes) {
296+
return free(start, nbytes, WordFactory.nullPointer());
297+
}
298+
299+
@Override
300+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
301+
public int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData) {
207302
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
208303
return -1;
209304
}
210305

211306
UnsignedWord granularity = getGranularity();
212307
Pointer mappingBegin = PointerUtils.roundDown(start, granularity);
213308
UnsignedWord mappingSize = UnsignedUtils.roundUp(nbytes, granularity);
214-
return munmap(mappingBegin, mappingSize);
309+
int ret = munmap(mappingBegin, mappingSize);
310+
if (ret == 0 && nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
311+
NativeMemoryTracking.singleton().trackFree(start);
312+
}
313+
return ret;
215314
}
216315

217316
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)

‎substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import com.oracle.svm.core.config.ConfigurationValues;
5858
import com.oracle.svm.core.headers.LibC;
5959
import com.oracle.svm.core.heap.Heap;
60+
import com.oracle.svm.core.nmt.NmtPreImageHeapData;
6061
import com.oracle.svm.core.os.AbstractImageHeapProvider;
6162
import com.oracle.svm.core.os.VirtualMemoryProvider;
6263
import com.oracle.svm.core.os.VirtualMemoryProvider.Access;
@@ -98,12 +99,12 @@ public class LinuxImageHeapProvider extends AbstractImageHeapProvider {
9899

99100
@Override
100101
@Uninterruptible(reason = "Called during isolate initialization.")
101-
public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) {
102+
public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtPreImageHeapData nmtData) {
102103
Pointer selfReservedMemory = WordFactory.nullPointer();
103104
UnsignedWord requiredSize = getTotalRequiredAddressSpaceSize();
104105
if (reservedAddressSpace.isNull()) {
105106
UnsignedWord alignment = WordFactory.unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment());
106-
selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false);
107+
selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false, nmtData);
107108
if (selfReservedMemory.isNull()) {
108109
return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED;
109110
}
@@ -127,13 +128,13 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W
127128

128129
int error = DynamicMethodAddressResolutionHeapSupport.get().initialize();
129130
if (error != CEntryPointErrors.NO_ERROR) {
130-
freeImageHeap(selfReservedHeapBase);
131+
freeImageHeap(selfReservedHeapBase, nmtData);
131132
return error;
132133
}
133134

134135
error = DynamicMethodAddressResolutionHeapSupport.get().install(heapBase);
135136
if (error != CEntryPointErrors.NO_ERROR) {
136-
freeImageHeap(selfReservedHeapBase);
137+
freeImageHeap(selfReservedHeapBase, nmtData);
137138
return error;
138139
}
139140
} else {
@@ -149,16 +150,17 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W
149150
CACHED_IMAGE_FD.get(), CACHED_IMAGE_HEAP_OFFSET_IN_FILE.get(), MAGIC.get(),
150151
IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get(),
151152
IMAGE_HEAP_RELOCATABLE_BEGIN.get(), IMAGE_HEAP_A_RELOCATABLE_POINTER.get(), IMAGE_HEAP_RELOCATABLE_END.get(),
152-
IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get());
153+
IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get(), nmtData);
153154
if (result != CEntryPointErrors.NO_ERROR) {
154-
freeImageHeap(selfReservedHeapBase);
155+
freeImageHeap(selfReservedHeapBase, nmtData);
155156
}
156157
return result;
157158
}
158159

159160
@Uninterruptible(reason = "Called during isolate initialization.")
160161
private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedSize, WordPointer endPointer, WordPointer cachedFd, WordPointer cachedOffsetInFile,
161-
Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) {
162+
Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym,
163+
NmtPreImageHeapData nmtData) {
162164
assert heapBeginSym.belowOrEqual(heapWritableSym) && heapWritableSym.belowOrEqual(heapWritableEndSym) && heapWritableEndSym.belowOrEqual(heapEndSym);
163165
assert heapBeginSym.belowOrEqual(heapRelocsSym) && heapRelocsSym.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.belowOrEqual(heapEndSym);
164166
assert heapAnyRelocPointer.isNull() || (heapRelocsSym.belowOrEqual(heapAnyRelocPointer) && heapAnyRelocPointer.belowThan(heapRelocsEndSym));
@@ -195,12 +197,12 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
195197
* heap must be in pristine condition for that).
196198
*/
197199
if (fd.equal(CANNOT_OPEN_FD)) {
198-
return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym);
200+
return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym, nmtData);
199201
}
200202

201203
// Create memory mappings from the image file.
202204
UnsignedWord fileOffset = cachedOffsetInFile.read();
203-
Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ);
205+
Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ, nmtData);
204206
if (mappedImageHeap.isNull() || mappedImageHeap != imageHeap) {
205207
return CEntryPointErrors.MAP_HEAP_FAILED;
206208
}
@@ -221,7 +223,7 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
221223
* be part of a chunk with writable objects, in which case the chunk header must
222224
* also be writable, and all the chunk's pages will be unprotected below.
223225
*/
224-
Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE);
226+
Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE, nmtData);
225227
if (committedRelocsBegin.isNull() || committedRelocsBegin != relocsBoundary) {
226228
return CEntryPointErrors.PROTECT_HEAP_FAILED;
227229
}
@@ -249,8 +251,9 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
249251
}
250252

251253
@Uninterruptible(reason = "Called during isolate initialization.")
252-
private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym) {
253-
Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE);
254+
private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym,
255+
NmtPreImageHeapData nmtData) {
256+
Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE, nmtData);
254257
if (committedBegin.isNull()) {
255258
return CEntryPointErrors.MAP_HEAP_FAILED;
256259
}
@@ -343,6 +346,11 @@ private static boolean checkImageFileMagic(int mapfd, int imagefd, CCharPointer
343346
@Override
344347
@Uninterruptible(reason = "Called during isolate tear-down.")
345348
public int freeImageHeap(PointerBase heapBase) {
349+
return freeImageHeap(heapBase, WordFactory.nullPointer());
350+
}
351+
352+
@Uninterruptible(reason = "Called during isolate tear-down.")
353+
private int freeImageHeap(PointerBase heapBase, NmtPreImageHeapData nmtData) {
346354
if (heapBase.isNull()) { // no memory allocated
347355
return CEntryPointErrors.NO_ERROR;
348356
}
@@ -352,7 +360,7 @@ public int freeImageHeap(PointerBase heapBase) {
352360
if (DynamicMethodAddressResolutionHeapSupport.isEnabled()) {
353361
addressSpaceStart = addressSpaceStart.subtract(getPreHeapAlignedSizeForDynamicMethodAddressResolver());
354362
}
355-
if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize()) != 0) {
363+
if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize(), nmtData) != 0) {
356364
return CEntryPointErrors.FREE_IMAGE_HEAP_FAILED;
357365
}
358366
return CEntryPointErrors.NO_ERROR;

0 commit comments

Comments
 (0)