Skip to content

Commit 896c258

Browse files
roberttoyonagaroberttoyonaga
committed
add virtual memory tracking
1 parent e96f9a5 commit 896c258

25 files changed

+1845
-83
lines changed

‎substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.graalvm.word.WordFactory;
3838

3939
import com.oracle.svm.core.heap.OutOfMemoryUtil;
40+
import com.oracle.svm.core.nmt.NmtCategory;
4041
import com.oracle.svm.core.os.VirtualMemoryProvider;
4142
import com.oracle.svm.core.util.VMError;
4243

@@ -112,7 +113,7 @@ private Pointer prepareTrampolines(PinnedObject mhsArray, PinnedObject stubsArra
112113
VirtualMemoryProvider memoryProvider = VirtualMemoryProvider.get();
113114
UnsignedWord pageSize = allocationSize();
114115
/* We request a specific alignment to guarantee correctness of getAllocationBase */
115-
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE);
116+
Pointer page = memoryProvider.commit(WordFactory.nullPointer(), pageSize, VirtualMemoryProvider.Access.WRITE | VirtualMemoryProvider.Access.FUTURE_EXECUTE, NmtCategory.Internal);
116117
if (page.isNull()) {
117118
throw OutOfMemoryUtil.reportOutOfMemoryError(new OutOfMemoryError("Could not allocate memory for trampolines."));
118119
}

‎substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java‎

Lines changed: 101 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,15 @@
5454
import com.oracle.svm.core.c.CGlobalDataFactory;
5555
import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
5656
import com.oracle.svm.core.feature.InternalFeature;
57+
import com.oracle.svm.core.nmt.NativeMemoryTracking;
58+
import com.oracle.svm.core.nmt.NmtCategory;
59+
import com.oracle.svm.core.nmt.NmtPreImageHeapData;
60+
import com.oracle.svm.core.nmt.NmtPreImageHeapDataAccess;
5761
import com.oracle.svm.core.os.VirtualMemoryProvider;
5862
import com.oracle.svm.core.posix.headers.Unistd;
5963
import com.oracle.svm.core.util.PointerUtils;
6064
import com.oracle.svm.core.util.UnsignedUtils;
65+
import com.oracle.svm.core.VMInspectionOptions;
6166

6267
@AutomaticallyRegisteredFeature
6368
class PosixVirtualMemoryProviderFeature implements InternalFeature {
@@ -109,7 +114,18 @@ public UnsignedWord getGranularity() {
109114

110115
@Override
111116
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
112-
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) {
117+
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData) {
118+
return reserve0(nbytes, alignment, executable, nmtData, NmtCategory.ImageHeap);
119+
}
120+
121+
@Override
122+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
123+
public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtCategory category) {
124+
return reserve0(nbytes, alignment, executable, WordFactory.nullPointer(), category);
125+
}
126+
127+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
128+
private Pointer reserve0(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtPreImageHeapData nmtData, NmtCategory category) {
113129
if (nbytes.equal(0)) {
114130
return WordFactory.nullPointer();
115131
}
@@ -128,25 +144,50 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec
128144
return nullPointer();
129145
}
130146
if (!customAlignment) {
147+
if (nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
148+
NativeMemoryTracking.singleton().trackReserve(mappingBegin, mappingSize, category);
149+
} else {
150+
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, mappingBegin, mappingSize, category);
151+
}
131152
return mappingBegin;
132153
}
154+
UnsignedWord unmappedSize = WordFactory.zero();
133155
Pointer begin = PointerUtils.roundUp(mappingBegin, alignment);
134156
UnsignedWord clippedBegin = begin.subtract(mappingBegin);
135157
if (clippedBegin.aboveOrEqual(granularity)) {
136-
munmap(mappingBegin, UnsignedUtils.roundDown(clippedBegin, granularity));
158+
UnsignedWord unmapSize = UnsignedUtils.roundDown(clippedBegin, granularity);
159+
munmap(mappingBegin, unmapSize);
160+
unmappedSize = unmappedSize.add(unmapSize);
137161
}
138162
Pointer mappingEnd = mappingBegin.add(mappingSize);
139163
UnsignedWord clippedEnd = mappingEnd.subtract(begin.add(nbytes));
140164
if (clippedEnd.aboveOrEqual(granularity)) {
141165
UnsignedWord rounded = UnsignedUtils.roundDown(clippedEnd, granularity);
142166
munmap(mappingEnd.subtract(rounded), rounded);
167+
unmappedSize = unmappedSize.add(rounded);
168+
}
169+
if (nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
170+
NativeMemoryTracking.singleton().trackReserve(begin, mappingSize.subtract(unmappedSize), category);
171+
} else {
172+
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, begin, mappingSize.subtract(unmappedSize), category);
143173
}
144174
return begin;
145175
}
146176

147177
@Override
148178
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
149-
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) {
179+
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData) {
180+
return mapFile0(start, nbytes, fileHandle, offset, access, nmtData, NmtCategory.ImageHeap);
181+
}
182+
183+
@Override
184+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
185+
public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtCategory category) {
186+
return mapFile0(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer(), category);
187+
}
188+
189+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
190+
private Pointer mapFile0(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
150191
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
151192
return WordFactory.nullPointer();
152193
}
@@ -157,12 +198,27 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand
157198
}
158199
int fd = (int) fileHandle.rawValue();
159200
Pointer result = mmap(start, nbytes, accessAsProt(access), flags, fd, offset.rawValue());
160-
return result.notEqual(MAP_FAILED()) ? result : WordFactory.nullPointer();
201+
if (result.notEqual(MAP_FAILED())) {
202+
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
203+
return result;
204+
}
205+
return WordFactory.nullPointer();
206+
}
207+
208+
@Override
209+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
210+
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData) {
211+
return commit0(start, nbytes, access, nmtData, NmtCategory.ImageHeap);
161212
}
162213

163214
@Override
164215
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
165-
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
216+
public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtCategory category) {
217+
return commit0(start, nbytes, access, WordFactory.nullPointer(), category);
218+
}
219+
220+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
221+
private Pointer commit0(PointerBase start, UnsignedWord nbytes, int access, NmtPreImageHeapData nmtData, NmtCategory category) {
166222
if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) {
167223
return WordFactory.nullPointer();
168224
}
@@ -177,7 +233,28 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
177233
}
178234
/* The memory returned by mmap is guaranteed to be zeroed. */
179235
final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET);
180-
return result.notEqual(MAP_FAILED()) ? result : nullPointer();
236+
if (result.notEqual(MAP_FAILED())) {
237+
trackCommitAndMaybeReserve(result, start, nbytes, nmtData, category);
238+
return result;
239+
}
240+
return nullPointer();
241+
}
242+
243+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
244+
private static void trackCommitAndMaybeReserve(PointerBase baseAddr, PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData, NmtCategory category) {
245+
// Account for possible reserve before commit
246+
if (start.isNull()) {
247+
if (nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
248+
NativeMemoryTracking.singleton().trackReserve(baseAddr, nbytes, category);
249+
} else {
250+
NmtPreImageHeapDataAccess.enqueueReserve(nmtData, baseAddr, nbytes, category);
251+
}
252+
}
253+
if (nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
254+
NativeMemoryTracking.singleton().trackCommit(baseAddr, nbytes, category);
255+
} else {
256+
NmtPreImageHeapDataAccess.enqueueCommit(nmtData, baseAddr, nbytes, category);
257+
}
181258
}
182259

183260
@Override
@@ -196,22 +273,37 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) {
196273
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
197274
return -1;
198275
}
199-
200276
final Pointer result = mmap(start, nbytes, PROT_NONE(), MAP_FIXED() | MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET);
201-
return result.notEqual(MAP_FAILED()) ? 0 : -1;
277+
if (result.notEqual(MAP_FAILED())) {
278+
if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
279+
NativeMemoryTracking.singleton().trackUncommit(start, nbytes);
280+
}
281+
return 0;
282+
}
283+
return -1;
202284
}
203285

204286
@Override
205287
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
206288
public int free(PointerBase start, UnsignedWord nbytes) {
289+
return free(start, nbytes, WordFactory.nullPointer());
290+
}
291+
292+
@Override
293+
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
294+
public int free(PointerBase start, UnsignedWord nbytes, NmtPreImageHeapData nmtData) {
207295
if (start.isNull() || !isAligned(start) || nbytes.equal(0)) {
208296
return -1;
209297
}
210298

211299
UnsignedWord granularity = getGranularity();
212300
Pointer mappingBegin = PointerUtils.roundDown(start, granularity);
213301
UnsignedWord mappingSize = UnsignedUtils.roundUp(nbytes, granularity);
214-
return munmap(mappingBegin, mappingSize);
302+
int ret = munmap(mappingBegin, mappingSize);
303+
if (ret == 0 && nmtData.isNull() && VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
304+
NativeMemoryTracking.singleton().trackFree(start);
305+
}
306+
return ret;
215307
}
216308

217309
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)

‎substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import com.oracle.svm.core.config.ConfigurationValues;
5858
import com.oracle.svm.core.headers.LibC;
5959
import com.oracle.svm.core.heap.Heap;
60+
import com.oracle.svm.core.nmt.NmtPreImageHeapData;
6061
import com.oracle.svm.core.os.AbstractImageHeapProvider;
6162
import com.oracle.svm.core.os.VirtualMemoryProvider;
6263
import com.oracle.svm.core.os.VirtualMemoryProvider.Access;
@@ -98,12 +99,12 @@ public class LinuxImageHeapProvider extends AbstractImageHeapProvider {
9899

99100
@Override
100101
@Uninterruptible(reason = "Called during isolate initialization.")
101-
public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) {
102+
public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtPreImageHeapData nmtData) {
102103
Pointer selfReservedMemory = WordFactory.nullPointer();
103104
UnsignedWord requiredSize = getTotalRequiredAddressSpaceSize();
104105
if (reservedAddressSpace.isNull()) {
105106
UnsignedWord alignment = WordFactory.unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment());
106-
selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false);
107+
selfReservedMemory = VirtualMemoryProvider.get().reserve(requiredSize, alignment, false, nmtData);
107108
if (selfReservedMemory.isNull()) {
108109
return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED;
109110
}
@@ -127,13 +128,13 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W
127128

128129
int error = DynamicMethodAddressResolutionHeapSupport.get().initialize();
129130
if (error != CEntryPointErrors.NO_ERROR) {
130-
freeImageHeap(selfReservedHeapBase);
131+
freeImageHeap(selfReservedHeapBase, nmtData);
131132
return error;
132133
}
133134

134135
error = DynamicMethodAddressResolutionHeapSupport.get().install(heapBase);
135136
if (error != CEntryPointErrors.NO_ERROR) {
136-
freeImageHeap(selfReservedHeapBase);
137+
freeImageHeap(selfReservedHeapBase, nmtData);
137138
return error;
138139
}
139140
} else {
@@ -149,16 +150,17 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W
149150
CACHED_IMAGE_FD.get(), CACHED_IMAGE_HEAP_OFFSET_IN_FILE.get(), MAGIC.get(),
150151
IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get(),
151152
IMAGE_HEAP_RELOCATABLE_BEGIN.get(), IMAGE_HEAP_A_RELOCATABLE_POINTER.get(), IMAGE_HEAP_RELOCATABLE_END.get(),
152-
IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get());
153+
IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get(), nmtData);
153154
if (result != CEntryPointErrors.NO_ERROR) {
154-
freeImageHeap(selfReservedHeapBase);
155+
freeImageHeap(selfReservedHeapBase, nmtData);
155156
}
156157
return result;
157158
}
158159

159160
@Uninterruptible(reason = "Called during isolate initialization.")
160161
private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedSize, WordPointer endPointer, WordPointer cachedFd, WordPointer cachedOffsetInFile,
161-
Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) {
162+
Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym,
163+
NmtPreImageHeapData nmtData) {
162164
assert heapBeginSym.belowOrEqual(heapWritableSym) && heapWritableSym.belowOrEqual(heapWritableEndSym) && heapWritableEndSym.belowOrEqual(heapEndSym);
163165
assert heapBeginSym.belowOrEqual(heapRelocsSym) && heapRelocsSym.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.belowOrEqual(heapEndSym);
164166
assert heapAnyRelocPointer.isNull() || (heapRelocsSym.belowOrEqual(heapAnyRelocPointer) && heapAnyRelocPointer.belowThan(heapRelocsEndSym));
@@ -195,12 +197,12 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
195197
* heap must be in pristine condition for that).
196198
*/
197199
if (fd.equal(CANNOT_OPEN_FD)) {
198-
return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym);
200+
return initializeImageHeapByCopying(imageHeap, imageHeapSize, pageSize, heapBeginSym, heapWritableSym, heapWritableEndSym, nmtData);
199201
}
200202

201203
// Create memory mappings from the image file.
202204
UnsignedWord fileOffset = cachedOffsetInFile.read();
203-
Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ);
205+
Pointer mappedImageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSize, fd, fileOffset, Access.READ, nmtData);
204206
if (mappedImageHeap.isNull() || mappedImageHeap != imageHeap) {
205207
return CEntryPointErrors.MAP_HEAP_FAILED;
206208
}
@@ -221,7 +223,7 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
221223
* be part of a chunk with writable objects, in which case the chunk header must
222224
* also be writable, and all the chunk's pages will be unprotected below.
223225
*/
224-
Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE);
226+
Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE, nmtData);
225227
if (committedRelocsBegin.isNull() || committedRelocsBegin != relocsBoundary) {
226228
return CEntryPointErrors.PROTECT_HEAP_FAILED;
227229
}
@@ -249,8 +251,9 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS
249251
}
250252

251253
@Uninterruptible(reason = "Called during isolate initialization.")
252-
private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym) {
253-
Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE);
254+
private static int initializeImageHeapByCopying(Pointer imageHeap, UnsignedWord imageHeapSize, UnsignedWord pageSize, Word heapBeginSym, Word heapWritableSym, Word heapWritableEndSym,
255+
NmtPreImageHeapData nmtData) {
256+
Pointer committedBegin = VirtualMemoryProvider.get().commit(imageHeap, imageHeapSize, Access.READ | Access.WRITE, nmtData);
254257
if (committedBegin.isNull()) {
255258
return CEntryPointErrors.MAP_HEAP_FAILED;
256259
}
@@ -343,6 +346,11 @@ private static boolean checkImageFileMagic(int mapfd, int imagefd, CCharPointer
343346
@Override
344347
@Uninterruptible(reason = "Called during isolate tear-down.")
345348
public int freeImageHeap(PointerBase heapBase) {
349+
return freeImageHeap(heapBase, WordFactory.nullPointer());
350+
}
351+
352+
@Uninterruptible(reason = "Called during isolate tear-down.")
353+
private int freeImageHeap(PointerBase heapBase, NmtPreImageHeapData nmtData) {
346354
if (heapBase.isNull()) { // no memory allocated
347355
return CEntryPointErrors.NO_ERROR;
348356
}
@@ -352,7 +360,7 @@ public int freeImageHeap(PointerBase heapBase) {
352360
if (DynamicMethodAddressResolutionHeapSupport.isEnabled()) {
353361
addressSpaceStart = addressSpaceStart.subtract(getPreHeapAlignedSizeForDynamicMethodAddressResolver());
354362
}
355-
if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize()) != 0) {
363+
if (VirtualMemoryProvider.get().free(addressSpaceStart, getTotalRequiredAddressSpaceSize(), nmtData) != 0) {
356364
return CEntryPointErrors.FREE_IMAGE_HEAP_FAILED;
357365
}
358366
return CEntryPointErrors.NO_ERROR;

0 commit comments

Comments
 (0)