diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java index a9d475d4ed0f..09a3509e8c0d 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java @@ -76,6 +76,14 @@ public static T malloc(UnsignedWord size) { return result; } + public static T malloc(UnsignedWord size, int flag) { + T result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size, flag); + if (result.isNull()) { + throw new OutOfMemoryError("malloc of unmanaged memory"); + } + return result; + } + /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is undefined. *

@@ -138,6 +146,14 @@ public static T realloc(T ptr, UnsignedWord size) { return result; } + public static T realloc(T ptr, UnsignedWord size, int flag) { + T result = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, size, flag); + if (result.isNull()) { + throw new OutOfMemoryError("realloc of unmanaged memory"); + } + return result; + } + /** * Frees unmanaged memory that was previously allocated using methods of this class. * @@ -146,4 +162,13 @@ public static T realloc(T ptr, UnsignedWord size) { public static void free(PointerBase ptr) { ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); } + + /** + * Will not attempt to perform any NMT operations. This is crucial for releasing memory + * allocated by C libraries which will not have NMT "malloc headers". If + * {@link UnmanagedMemory#free(PointerBase)} is used instead, a segfault will occur. + */ + public static void untrackedFree(PointerBase ptr) { + ImageSingletons.lookup(UnmanagedMemorySupport.class).untrackedFree(ptr); + } } diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java index c18ab87631e8..4dca2cabd080 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java @@ -47,9 +47,17 @@ public interface UnmanagedMemorySupport { T malloc(UnsignedWord size); + T malloc(UnsignedWord size, int flag); + T calloc(UnsignedWord size); + T calloc(UnsignedWord size, int flag); + T realloc(T ptr, UnsignedWord size); + T realloc(T ptr, UnsignedWord size, int flag); + void free(PointerBase ptr); + + void untrackedFree(PointerBase ptr); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java index f3d2b586e41e..dfe66b33245d 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java @@ -36,12 +36,12 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.BaseProcessPropertiesSupport; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.posix.headers.Dlfcn; import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Stdlib; @@ -103,7 +103,7 @@ public String getObjectFile(PointerBase symbolAddress) { try { return CTypeConversion.toJavaString(realpath); } finally { - LibC.free(realpath); + UnmanagedMemory.untrackedFree(realpath); } } @@ -164,7 +164,7 @@ protected static String realpath(String path) { } else { /* Success */ final String result = CTypeConversion.toJavaString(realpathPointer); - LibC.free(realpathPointer); + UnmanagedMemory.untrackedFree(realpathPointer); return result; } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java index 6c56453a7466..8f03ad222df9 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java @@ -24,38 +24,112 @@ */ package com.oracle.svm.core.posix; +import jdk.graal.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; + import org.graalvm.word.PointerBase; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import org.graalvm.nativeimage.StackValue; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.headers.LibCSupport; +import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.ReturnAddress; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class UnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return LibC.malloc(size); + return malloc(size, NmtFlag.mtNone.ordinal()); + } + + @Override + @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T malloc(UnsignedWord size, int flag) { + ReturnAddress ra = StackValue.get(ReturnAddress.class); + if (NativeMemoryTracking.handlePreInitMallocs(size, flag, ra)) { + return (T) ra.get(); + } + Pointer outerPointer = libc().malloc(size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return LibC.calloc(WordFactory.unsigned(1), size); + return calloc(size, NmtFlag.mtNone.ordinal()); + } + + @Override + @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T calloc(UnsignedWord size, int flag) { + ReturnAddress ra = StackValue.get(ReturnAddress.class); + if (NativeMemoryTracking.handlePreInitCallocs(size, flag, ra)) { + return (T) ra.get(); + } + Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return LibC.realloc(ptr, size); + return realloc(ptr, size, NmtFlag.mtNone.ordinal()); + } + + @Override + @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T realloc(T ptr, UnsignedWord size, int flag) { + + ReturnAddress ra = StackValue.get(ReturnAddress.class); + if (NativeMemoryTracking.handlePreInitReallocs((Pointer) ptr, size, flag, ra)) { + return (T) ra.get(); + } + + // Retrieve necessary data from the old block + Pointer oldOuterPointer = ((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize()); + long oldSize = NativeMemoryTracking.getAllocationSize(oldOuterPointer); + int oldCategory = NativeMemoryTracking.getAllocationCategory(oldOuterPointer); + + // Perform the realloc + Pointer newOuterPointer = libc().realloc(oldOuterPointer, size.add(NativeMemoryTracking.getHeaderSize())); + + // Only deaccount the old block, if we were successful. + if (newOuterPointer.isNonNull()) { + NativeMemoryTracking.deaccountMalloc(oldSize, oldCategory); + } + + // Account the new block and overwrite the header with the new tracking data + return (T) NativeMemoryTracking.recordMalloc(newOuterPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { - LibC.free(ptr); + if (NativeMemoryTracking.handlePreInitFrees((Pointer) ptr)) { + return; + } + NativeMemoryTracking.deaccountMalloc(ptr); + libc().free(((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize())); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void untrackedFree(PointerBase ptr) { + libc().free(ptr); + } + + @Fold + static LibCSupport libc() { + return ImageSingletons.lookup(LibCSupport.class); } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java index fe0e9ccbbc9a..88dc4551d9e3 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java @@ -31,13 +31,13 @@ import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; import org.graalvm.nativeimage.impl.RuntimeSystemPropertiesSupport; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.SystemPropertiesSupport; import com.oracle.svm.core.posix.PosixSystemPropertiesSupport; import com.oracle.svm.core.posix.headers.Limits; @@ -105,7 +105,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatform(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - LibC.free(osVersionStr); + UnmanagedMemory.untrackedFree(osVersionStr); return osVersionValue; } } else { @@ -120,7 +120,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatformFallback(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - LibC.free(osVersionStr); + UnmanagedMemory.untrackedFree(osVersionStr); return osVersionValue; } return osVersionValue = "Unknown"; diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java index 353855263cba..2bff667d1bf9 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java @@ -38,6 +38,7 @@ import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.RuntimeSystemPropertiesSupport; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -45,7 +46,6 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.SystemPropertiesSupport; import com.oracle.svm.core.util.VMError; import com.oracle.svm.core.windows.headers.FileAPI; @@ -245,29 +245,30 @@ public Pair getOsNameAndVersion() { break; } - VoidPointer versionInfo = LibC.malloc(WordFactory.unsigned(versionSize)); + VoidPointer versionInfo = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(versionSize)); if (versionInfo.isNull()) { break; } + try { - if (WinVer.GetFileVersionInfoW(kernel32Path, 0, versionSize, versionInfo) == 0) { - LibC.free(versionInfo); - break; - } + if (WinVer.GetFileVersionInfoW(kernel32Path, 0, versionSize, versionInfo) == 0) { + break; + } - WindowsLibC.WCharPointer rootPath = NonmovableArrays.addressOf(NonmovableArrays.fromImageHeap(ROOT_PATH), 0); - WordPointer fileInfoPointer = UnsafeStackValue.get(WordPointer.class); - CIntPointer lengthPointer = UnsafeStackValue.get(CIntPointer.class); - if (WinVer.VerQueryValueW(versionInfo, rootPath, fileInfoPointer, lengthPointer) == 0) { - LibC.free(versionInfo); - break; - } + WindowsLibC.WCharPointer rootPath = NonmovableArrays.addressOf(NonmovableArrays.fromImageHeap(ROOT_PATH), 0); + WordPointer fileInfoPointer = UnsafeStackValue.get(WordPointer.class); + CIntPointer lengthPointer = UnsafeStackValue.get(CIntPointer.class); + if (WinVer.VerQueryValueW(versionInfo, rootPath, fileInfoPointer, lengthPointer) == 0) { + break; + } - VerRsrc.VS_FIXEDFILEINFO fileInfo = fileInfoPointer.read(); - majorVersion = (short) (fileInfo.dwProductVersionMS() >> 16); // HIWORD - minorVersion = (short) fileInfo.dwProductVersionMS(); // LOWORD - buildNumber = (short) (fileInfo.dwProductVersionLS() >> 16); // HIWORD - LibC.free(versionInfo); + VerRsrc.VS_FIXEDFILEINFO fileInfo = fileInfoPointer.read(); + majorVersion = (short) (fileInfo.dwProductVersionMS() >> 16); // HIWORD + minorVersion = (short) fileInfo.dwProductVersionMS(); // LOWORD + buildNumber = (short) (fileInfo.dwProductVersionLS() >> 16); // HIWORD + } finally { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(versionInfo); + } } while (false); String osVersion = majorVersion + "." + minorVersion; diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java index 7ee93b64d4e7..3f9d85915fea 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.core.windows; +import jdk.graal.compiler.api.replacements.Fold; + +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -31,31 +34,56 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.headers.LibCSupport; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class WindowsUnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return LibC.malloc(size); + return libc().malloc(size); + } + + @Override + public T malloc(UnsignedWord size, int flag) { + return libc().malloc(size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return LibC.calloc(WordFactory.unsigned(1), size); + return libc().calloc(WordFactory.unsigned(1), size); + } + + @Override + public T calloc(UnsignedWord size, int flag) { + return libc().calloc(WordFactory.unsigned(1), size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return LibC.realloc(ptr, size); + return libc().realloc(ptr, size); + } + + @Override + public T realloc(T ptr, UnsignedWord size, int flag) { + return libc().realloc(ptr, size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { - LibC.free(ptr); + libc().free(ptr); + } + + @Override + public void untrackedFree(PointerBase ptr) { + libc().free(ptr); + } + + @Fold + static LibCSupport libc() { + return ImageSingletons.lookup(LibCSupport.class); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 465ac470c8b1..4298e1455c8d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -912,6 +912,9 @@ public Boolean getValue(OptionValues values) { @Option(help = "file:doc-files/FlightRecorderLoggingHelp.txt")// public static final RuntimeOptionKey FlightRecorderLogging = new RuntimeOptionKey<>("all=warning", Immutable); + @Option(help = "Enable native memory tracking")// + public static final RuntimeOptionKey NativeMemoryTracking = new RuntimeOptionKey<>(false); + public static String reportsPath() { Path reportsPath = ImageSingletons.lookup(ReportingSupport.class).reportsPath; if (reportsPath.isAbsolute()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index bf1d2a4e5946..9e5f61a768b1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -57,11 +57,12 @@ public final class VMInspectionOptions { private static final String MONITORING_JMXCLIENT_NAME = "jmxclient"; private static final String MONITORING_JMXSERVER_NAME = "jmxserver"; private static final String MONITORING_THREADDUMP_NAME = "threaddump"; + private static final String MONITORING_NMT_NAME = "nmt"; private static final List MONITORING_ALL_VALUES = List.of(MONITORING_HEAPDUMP_NAME, MONITORING_JFR_NAME, MONITORING_JVMSTAT_NAME, MONITORING_JMXCLIENT_NAME, MONITORING_JMXSERVER_NAME, - MONITORING_THREADDUMP_NAME, MONITORING_ALL_NAME, MONITORING_DEFAULT_NAME); + MONITORING_THREADDUMP_NAME, MONITORING_NMT_NAME, MONITORING_ALL_NAME, MONITORING_DEFAULT_NAME); private static final String MONITORING_ALLOWED_VALUES_TEXT = "'" + MONITORING_HEAPDUMP_NAME + "', '" + MONITORING_JFR_NAME + "', '" + MONITORING_JVMSTAT_NAME + "', '" + MONITORING_JMXSERVER_NAME + - "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', or '" + MONITORING_ALL_NAME + + "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', '" + MONITORING_NMT_NAME + "', or '" + MONITORING_ALL_NAME + "' (deprecated behavior: defaults to '" + MONITORING_ALL_NAME + "' if no argument is provided)"; static { @@ -163,6 +164,11 @@ public static boolean hasThreadDumpSupport() { return hasAllOrKeywordMonitoringSupport(MONITORING_THREADDUMP_NAME) || DeprecatedOptions.DumpThreadStacksOnSignal.getValue(); } + @Fold + public static boolean hasNmtSupport() { + return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME) && !Platform.includedIn(WINDOWS.class); + } + @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // public static final HostedOptionKey DumpRuntimeCompilationOnSignal = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java index 8eb7cd7b84ac..8166fa91416a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java @@ -42,7 +42,7 @@ public abstract class AbstractUninterruptibleHashtable implements Uninterruptibl private static final int DEFAULT_TABLE_LENGTH = 2053; private final UninterruptibleEntry[] table; - private int size; + protected int size; @Platforms(Platform.HOSTED_ONLY.class) public AbstractUninterruptibleHashtable() { @@ -95,6 +95,26 @@ protected UninterruptibleEntry insertEntry(UninterruptibleEntry valueOnStack) { return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void remove(UninterruptibleEntry valueOnStack) { + int index = Integer.remainderUnsigned(valueOnStack.getHash(), DEFAULT_TABLE_LENGTH); + UninterruptibleEntry entry = table[index]; + UninterruptibleEntry prev = WordFactory.nullPointer(); + while (entry.isNonNull()) { + if (isEqual(valueOnStack, entry)) { + if (prev.isNull()) { + table[index] = entry.getNext(); + } else { + prev.setNext(entry.getNext()); + } + free(entry); + return; + } + prev = entry; + entry = entry.getNext(); + } + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getSize() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java index b72147d4a989..f23f371b99e1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java @@ -67,26 +67,6 @@ public static T memset(T s, SignedWord c, UnsignedWord n return libc().memset(s, c, n); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T malloc(UnsignedWord size) { - return libc().malloc(size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T calloc(UnsignedWord nmemb, UnsignedWord size) { - return libc().calloc(nmemb, size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T realloc(PointerBase ptr, UnsignedWord size) { - return libc().realloc(ptr, size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void free(PointerBase ptr) { - libc().free(ptr); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void exit(int status) { libc().exit(status); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java index a20bee43c883..ed8e8891c086 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java @@ -46,6 +46,7 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.hub.PredefinedClassesSupport; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.VMError; @@ -55,12 +56,12 @@ final class Target_jdk_internal_misc_Unsafe_Core { @Substitute private long allocateMemory0(long bytes) { - return UnmanagedMemory.malloc(WordFactory.unsigned(bytes)).rawValue(); + return UnmanagedMemory.malloc(WordFactory.unsigned(bytes), NmtFlag.mtOther.ordinal()).rawValue(); } @Substitute private long reallocateMemory0(long address, long bytes) { - return UnmanagedMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes)).rawValue(); + return UnmanagedMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes), NmtFlag.mtOther.ordinal()).rawValue(); } @Substitute diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java index 6875c685d0c2..6f3fb6cf8741 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java @@ -100,7 +100,7 @@ private static String getSystemTimeZoneID(String javaHome) { CCharPointer tzId = LibCHelper.SVM_FindJavaTZmd(tzMappingsPtr, contentLen); String result = CTypeConversion.toJavaString(tzId); // SVM_FindJavaTZmd returns a newly allocated string - UnmanagedMemory.free(tzId); + UnmanagedMemory.untrackedFree(tzId); return result; } finally { if (refContent != null) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java index 31da845e8aa1..f83bd41dd1d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java @@ -34,6 +34,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; @@ -61,7 +62,7 @@ public static JfrBuffer allocate(JfrBufferType bufferType) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBuffer allocate(UnsignedWord dataSize, JfrBufferType bufferType) { UnsignedWord headerSize = JfrBufferAccess.getHeaderSize(); - JfrBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize)); + JfrBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize), NmtFlag.mtTracing.ordinal()); if (result.isNonNull()) { result.setSize(dataSize); result.setBufferType(bufferType); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java index 759db4b69ce4..9102ec67abbb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java @@ -34,6 +34,7 @@ import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.thread.NativeSpinLockUtils; import com.oracle.svm.core.thread.VMOperation; @@ -47,7 +48,7 @@ private JfrBufferNodeAccess() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBufferNode allocate(JfrBuffer buffer) { - JfrBufferNode node = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(JfrBufferNode.class)); + JfrBufferNode node = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(JfrBufferNode.class), NmtFlag.mtTracing.ordinal()); if (node.isNonNull()) { node.setBuffer(buffer); node.setNext(WordFactory.nullPointer()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java index 689c3c37f9a0..ee885c2a0c0c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java @@ -51,6 +51,7 @@ import com.oracle.svm.core.jfr.traceid.JfrTraceIdEpoch; import com.oracle.svm.core.jfr.utils.JfrVisited; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.sampler.SamplerSampleWriter; import com.oracle.svm.core.sampler.SamplerSampleWriterData; import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess; @@ -193,7 +194,7 @@ private JfrStackTraceTableEntry getOrPutStackTrace0(Pointer start, UnsignedWord * the thread-local buffer to the C heap because the thread-local buffer will be * overwritten or freed at some point. */ - Pointer to = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + Pointer to = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size, NmtFlag.mtTracing.ordinal()); if (to.isNonNull()) { UnmanagedMemoryUtil.copy(start, to, size); entry.setRawStackTrace(to); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java new file mode 100644 index 000000000000..19607542b0a0 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import java.util.function.BooleanSupplier; +import jdk.graal.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; + +public class HasNmtSupport implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return get(); + } + + @Fold + public static boolean get() { + return ImageSingletons.contains(NativeMemoryTracking.class); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java new file mode 100644 index 000000000000..f0b6929fd15b --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; + +/** + * A "malloc header" is used to cache metadata about the native allocation (calloc/realloc/malloc). + * To do this, a small amount of additional space is requested contiguous to the user allocation. + * This metadata is later used to update the memory tracking once the block is freed. + */ +@RawStructure +public interface MallocHeader extends PointerBase { + @RawField + long getSize(); + + @RawField + void setSize(long value); + + @RawField + int getFlag(); + + @RawField + void setFlag(int value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java new file mode 100644 index 000000000000..c2159078129a --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; + +import com.oracle.svm.core.Uninterruptible; +import org.graalvm.word.UnsignedWord; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.Platform; + +class MallocMemoryInfo { + private AtomicLong count; + private AtomicLong size; + + @Platforms(Platform.HOSTED_ONLY.class) + MallocMemoryInfo() { + count = new AtomicLong(0); + size = new AtomicLong(0); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void recordMalloc(UnsignedWord allocationSize) { + count.incrementAndGet(); + size.addAndGet(allocationSize.rawValue()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void deaccountMalloc(long allocationSize) { + long lastCount = count.decrementAndGet(); + long lastSize = size.addAndGet(-allocationSize); + assert lastSize >= 0 && lastCount >= 0; + } + + long getSize() { + return size.get(); + } + + long getCount() { + return count.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java new file mode 100644 index 000000000000..07336bf784fd --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.Platform; + +public class MallocMemorySnapshot { + private MallocMemoryInfo[] categories; + private MallocMemoryInfo total; + + @Platforms(Platform.HOSTED_ONLY.class) + MallocMemorySnapshot() { + total = new MallocMemoryInfo(); + categories = new MallocMemoryInfo[NmtFlag.values().length]; + for (int i = 0; i < categories.length; i++) { + categories[i] = new MallocMemoryInfo(); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + MallocMemoryInfo getInfoByCategory(int flag) { + assert flag < categories.length; + return categories[flag]; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public MallocMemoryInfo getTotalInfo() { + return total; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java new file mode 100644 index 000000000000..984731c961be --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -0,0 +1,497 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.headers.LibCSupport; +import com.oracle.svm.core.jdk.RuntimeSupport; +import com.oracle.svm.core.thread.JavaSpinLockUtils; +import com.oracle.svm.core.util.UnsignedUtils; +import com.oracle.svm.util.LogUtils; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.internal.misc.Unsafe; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +/** + * This is the class that handles native memory tracking (NMT). There are two components to NMT: + * tracking malloc/realloc/calloc, and tracking virtual memory usage. + *

+ *

+ *

+ * Malloc/realloc/calloc: Malloc headers ({@link MallocHeader}) are used for caching data about the + * allocation. This is necessary because that data is needed to deaccount the tracked memory when it + * is freed. There are some scenarios where an implementation dependent on malloc header requires + * special precautions: + *

+ * + *
    + *
  1. When native memory is allocated outside of java. This might be in a C library for example. + * When it comes time to free the memory, it is important that the NMT infrastructure, if enabled, + * does not assume a malloc header exists and attempt to access it. + * {@link UnmanagedMemorySupport#untrackedFree(PointerBase)} must be used in this case. + *
  2. When raw {@link LibCSupport} allocation methods are called directly. This is problematic + * because such allocated blocks will not contain malloc headers. If NMT is enabled it will assume + * headers exist and access incorrect memory locations. This must be avoided. + *
  3. During the phase pre-NMT initialization. Before runtime options are parsed, it is unknown + * whether NMT should be enabled/disabled. During this time, it is assumed that NMT will later be + * enabled, so malloc headers are allocated. This increases the accuracy of the tracking. However, + * if NMT is later disabled upon initialization, there will be a mix of allocated blocks that + * have headers and do some that do not . This is a problem when it comes time to free the + * blocks. For this reason, there are methods in this class that handle pre-init allocations using a + * look-up table to cache addresses seen before initialization. This is the same reason NMT cannot + * be enabled/disabled after initialization. + *
  4. If native allocations are done before the image heap is mapped. This is not strictly a + * problem related to malloc headers, but must still be avoided. If such situations are unavoidable, + * NMT code must be bypassed using raw {@link LibCSupport} methods for both allocation and + * deallocation. + *
+ */ +public class NativeMemoryTracking { + private static final Unsafe U = Unsafe.getUnsafe(); + private static final long LOCK_OFFSET = U.objectFieldOffset(NativeMemoryTracking.class, "preInitLock"); + + /** + * Can't use VmMutex because it tracks owners and this class may be used by unattached threads + * (calloc). This lock is only used during the pre-init phase. + */ + @SuppressWarnings("unused") private volatile int preInitLock; + + private final PreInitTable preInitTable; + private volatile boolean enabled; + private volatile boolean initialized; + + private MallocMemorySnapshot mallocMemorySnapshot; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeMemoryTracking() { + enabled = true; + initialized = false; + mallocMemorySnapshot = new MallocMemorySnapshot(); + preInitTable = new PreInitTable(); + } + + @Fold + static UnsignedWord getHeaderSize0() { + return UnsignedUtils.roundUp(SizeOf.unsigned(MallocHeader.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + } + + @Fold + static LibCSupport libc() { + return ImageSingletons.lookup(LibCSupport.class); + } + + @Fold + static NativeMemoryTracking get() { + return ImageSingletons.lookup(NativeMemoryTracking.class); + } + + /** This must be called after the image heap is mapped. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void initialize(boolean enabled) { + if (HasNmtSupport.get()) { + get().initialize0(enabled); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void initialize0(boolean enabled) { + assert !initialized; + // Block until all in-progress pre-init operations are completed + lockNoTransition(); + try { + this.enabled = enabled; + // After this point, the preInit table becomes read only. + initialized = true; + } finally { + unlock(); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getHeaderSize() { + if (HasNmtSupport.get() && get().enabled) { + return getHeaderSize0(); + } + return WordFactory.zero(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static long getAllocationSize(Pointer outerPointer) { + if (HasNmtSupport.get() && get().enabled) { + MallocHeader header = (MallocHeader) outerPointer; + return header.getSize(); + } + return -1; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static int getAllocationCategory(Pointer outerPointer) { + if (HasNmtSupport.get() && get().enabled) { + MallocHeader header = (MallocHeader) outerPointer; + return header.getFlag(); + } + return -1; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer recordMalloc(Pointer outerPointer, UnsignedWord size, int flag) { + if (HasNmtSupport.get() && outerPointer.isNonNull()) { + return get().recordMalloc0(outerPointer, size, flag); + } + return outerPointer; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer recordMalloc0(Pointer outerPointer, UnsignedWord size, int flag) { + if (enabled) { + UnsignedWord headerSize = getHeaderSize0(); + mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); + mallocMemorySnapshot.getInfoByCategory(NmtFlag.mtNMT.ordinal()).recordMalloc(headerSize); + mallocMemorySnapshot.getTotalInfo().recordMalloc(size.add(headerSize)); + return initializeHeader(outerPointer, size, flag); + } + return outerPointer; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer initializeHeader(Pointer outerPointer, UnsignedWord size, int flag) { + MallocHeader mallocHeader = (MallocHeader) outerPointer; + mallocHeader.setSize(size.rawValue()); + mallocHeader.setFlag(flag); + return ((Pointer) mallocHeader).add(NativeMemoryTracking.getHeaderSize0()); + } + + /** + * This is only needed for {@link PreInitTable} since we must use raw malloc/free there but + * still want to track its allocations. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void recordMallocWithoutHeader(UnsignedWord size, int flag) { + if (HasNmtSupport.get()) { + get().recordMallocWithoutHeader0(size, flag); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void recordMallocWithoutHeader0(UnsignedWord size, int flag) { + if (enabled) { + mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); + mallocMemorySnapshot.getTotalInfo().recordMalloc(size); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void deaccountMalloc(PointerBase innerPtr) { + if (HasNmtSupport.get() && get().enabled) { + MallocHeader header = (MallocHeader) ((Pointer) innerPtr).subtract(getHeaderSize0()); + get().deaccountMalloc0(header.getSize(), header.getFlag()); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void deaccountMalloc(long size, int flag) { + if (HasNmtSupport.get() && size > 0 && flag >= 0) { + get().deaccountMalloc0(size, flag); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void deaccountMalloc0(long size, int flag) { + if (enabled) { + mallocMemorySnapshot.getInfoByCategory(flag).deaccountMalloc(size); + mallocMemorySnapshot.getInfoByCategory(NmtFlag.mtNMT.ordinal()).deaccountMalloc(getHeaderSize0().rawValue()); + mallocMemorySnapshot.getTotalInfo().deaccountMalloc(size + getHeaderSize0().rawValue()); + } + } + + /** + * This is only needed for {@link PreInitTable} since we must use raw malloc/free there but + * still want to track its allocations. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void deaccountMallocWithoutHeader(long size, int flag) { + if (HasNmtSupport.get() && size > 0 && flag >= 0) { + get().deaccountMallocWithoutHeader0(size, flag); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void deaccountMallocWithoutHeader0(long size, int flag) { + if (enabled) { + mallocMemorySnapshot.getInfoByCategory(flag).deaccountMalloc(size); + mallocMemorySnapshot.getTotalInfo().deaccountMalloc(size); + } + } + + public static long getMallocByCategory(NmtFlag flag) { + if (HasNmtSupport.get()) { + return get().mallocMemorySnapshot.getInfoByCategory(flag.ordinal()).getSize(); + } + return -1; + } + + /** Prints stats contained in the current snapshot. */ + public static void printStats() { + if (HasNmtSupport.get()) { + get().printStats0(); + } + } + + private void printStats0() { + if (!enabled) { + return; + } + LogUtils.info("Total current malloc size:" + mallocMemorySnapshot.getTotalInfo().getSize() + "B"); + LogUtils.info("Total current malloc count:" + mallocMemorySnapshot.getTotalInfo().getCount()); + + for (int i = 0; i < NmtFlag.values().length; i++) { + LogUtils.info(NmtFlag.values()[i].getName() + " current malloc size:" + mallocMemorySnapshot.getInfoByCategory(i).getSize() + "B"); + LogUtils.info(NmtFlag.values()[i].getName() + " current malloc count:" + mallocMemorySnapshot.getInfoByCategory(i).getCount()); + } + + } + + /** + * This method is needed because locking is required to make the allocation, LUT insertion, and + * initialization check atomic. If the malloc was handled here, returnAddress holds the location + * of the malloc block payload. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitMallocs(UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (HasNmtSupport.get()) { + return get().handlePreInitMallocs0(size, flag, returnAddress); + } + return false; + } + + /** + * Special handling is needed here becuase we need ot check initialization state and add an + * address to the LUT. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitMallocs0(UnsignedWord size, int flag, ReturnAddress returnAddress) { + // For speed, check if initialized before acquiring lock + if (!initialized) { + lockNoTransition(); + try { + // Double check after acquiring lock. If it's since been initialized, proceed + // normally. + if (initialized) { + return false; + } + // Still uninitialized. Allocate and add to LUT. + Pointer outerPointer = libc().malloc(size.add(getHeaderSize0())); + recordPreInitAlloc(outerPointer, size, flag, returnAddress); + return true; + } finally { + unlock(); + } + } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitCallocs(UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (HasNmtSupport.get()) { + return get().handlePreInitCallocs0(size, flag, returnAddress); + } + return false; + } + + /** + * Similar to + * {@link NativeMemoryTracking#handlePreInitMallocs0(UnsignedWord, int, ReturnAddress)}. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitCallocs0(UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (!initialized) { + lockNoTransition(); + try { + if (initialized) { + return false; + } + Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(getHeaderSize())); + recordPreInitAlloc(outerPointer, size, flag, returnAddress); + return true; + } finally { + unlock(); + } + } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitReallocs(Pointer oldInnerPtr, UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (HasNmtSupport.get()) { + return get().handlePreInitReallocs0(oldInnerPtr, size, flag, returnAddress); + } + return false; + } + + /** + * This method is needed because locking is required to make the allocation, LUT insertion, and + * initialization check atomic. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitReallocs0(Pointer oldInnerPtr, UnsignedWord size, int flag, ReturnAddress returnAddress) { + // For speed, check if initialized before acquiring lock + if (!initialized) { + lockNoTransition(); + try { + // Double check after acquiring lock. If still uninitialized, perform realloc and do + // LUT updates. + if (!initialized) { + // Retrieve necessary data from the old block + Pointer oldOuterPointer = oldInnerPtr.subtract(NativeMemoryTracking.getHeaderSize0()); + long oldSize = getAllocationSize(oldOuterPointer); + int oldCategory = getAllocationCategory(oldOuterPointer); + + // Perform the realloc + Pointer newOuterPointer = libc().realloc(oldOuterPointer, size.add(NativeMemoryTracking.getHeaderSize0())); + + if (newOuterPointer.isNonNull()) { + // Deaccount old block. Order matters here in case old/new have same + // address. + deaccountMalloc0(oldSize, oldCategory); + preInitTable.remove(oldInnerPtr); + + // Account new block + recordPreInitAlloc(newOuterPointer, size, flag, returnAddress); + } else { + returnAddress.set(WordFactory.nullPointer()); + } + return true; + } + } finally { + unlock(); + } + } + + // Post init. + if (!enabled && preInitTable.get(oldInnerPtr).isNonNull()) { + // There is a header from pre-init time, but tracking has since been disabled. + + // Malloc a new block with the given size and no header + Pointer newBlock = libc().malloc(size); + returnAddress.set(newBlock); + if (newBlock.isNonNull()) { + // Copy payload from original block to new block + Pointer oldOuterPointer = oldInnerPtr.subtract(getHeaderSize0()); + UnsignedWord oldSize = WordFactory.unsigned(getAllocationSize(oldOuterPointer)); + UnsignedWord amountToCopy = size.belowThan(oldSize) ? size : oldSize; + libc().memcpy(newBlock, oldInnerPtr, amountToCopy); + // Don't raw free the old block to avoid libc returning the same address later. + } + return true; + } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void recordPreInitAlloc(Pointer newOuterPointer, UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (newOuterPointer.isNull()) { + returnAddress.set(WordFactory.nullPointer()); + return; + } + Pointer newInnerPtr = recordMalloc0(newOuterPointer, size, flag); + preInitTable.putIfAbsent(newInnerPtr); + returnAddress.set(newInnerPtr); + } + + /** Special handling is needed for pre-init mallocs if NMT is disabled post-init. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitFrees(Pointer innerPtr) { + if (HasNmtSupport.get()) { + return get().handlePreInitFrees0(innerPtr); + } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitFrees0(Pointer innerPtr) { + if (!initialized) { + lockNoTransition(); + try { + if (!initialized) { + deaccountMalloc(innerPtr); + preInitTable.remove(innerPtr); + libc().free(innerPtr.subtract(getHeaderSize0())); + return true; + } + } finally { + unlock(); + } + } + + // Post init. + if (!enabled && preInitTable.get(innerPtr).isNonNull()) { + // If NMT is now disabled and we're dealing with a pre-init block. + // There is a header from pre-init time, but tracking has since been disabled. + // Do nothing more. Don't Raw free the old block to avoid libc returning the same + // address later. + return true; + + } + // Either NMT is still enabled or we're dealing with a block allocated after initialization. + // We can proceed with tracking normally. + return false; + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) + private void lockNoTransition() { + JavaSpinLockUtils.lockNoTransition(this, LOCK_OFFSET); + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) + private void unlock() { + JavaSpinLockUtils.unlock(this, LOCK_OFFSET); + } + + public static RuntimeSupport.Hook shutdownHook() { + return isFirstIsolate -> { + printStats(); + }; + } + + public static RuntimeSupport.Hook startupHook() { + return isFirstIsolate -> { + initialize(SubstrateOptions.NativeMemoryTracking.getValue()); + }; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java new file mode 100644 index 000000000000..ca6d96b244a6 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.VMInspectionOptions; +import org.graalvm.nativeimage.ImageSingletons; + +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.jdk.RuntimeSupport; + +@AutomaticallyRegisteredFeature +public class NmtFeature implements InternalFeature { + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return isInConfiguration(); + } + + public static boolean isInConfiguration() { + return VMInspectionOptions.hasNmtSupport(); + } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(NativeMemoryTracking.class, new NativeMemoryTracking()); + } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + RuntimeSupport runtime = RuntimeSupport.getRuntimeSupport(); + + runtime.addShutdownHook(NativeMemoryTracking.shutdownHook()); + runtime.addStartupHook(NativeMemoryTracking.startupHook()); + } + +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java new file mode 100644 index 000000000000..7f3a371117ca --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; + +/** These category flag names match their counterparts in Hotspot. */ +public enum NmtFlag { + mtJavaHeap("Java Heap"), + mtThread("Thread"), + mtThreadStack("Thread Stack"), + mtServiceability("Serviceability"), + mtGC("GC"), + mtInternal("Internal"), // Memory used by VM, outside other categories + mtCode("Code"), + mtOther("Other"), // Memory not used by VM (Unsafe) + mtNMT("Native Memory Tracking"), // Memory used by NMT itself + mtTest("Test"), // Test type for verifying NMT + mtTracing("Tracing"), // JFR + mtNone("Unknown"); // This is the default category + + private final String name; + + NmtFlag(String name) { + this.name = name; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public String getName() { + return name; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java new file mode 100644 index 000000000000..cedbdf21a0b3 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.nativeimage.ImageSingletons; + +/** This feature is required to omit the startup and shutdown hooks. */ +class NmtTestFeature implements Feature { + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(NativeMemoryTracking.class, new NativeMemoryTracking()); + } + + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return !NmtFeature.isInConfiguration(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java new file mode 100644 index 000000000000..8b4d909cc8a5 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.collections.UninterruptibleEntry; +import com.oracle.svm.core.collections.AbstractUninterruptibleHashtable; +import com.oracle.svm.core.headers.LibCSupport; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.StackValue; +import com.oracle.svm.core.jdk.UninterruptibleUtils; + +/** + * This table stores addresses of NMT pre-init malloc blocks. These blocks will always contain + * headers. It is important that this class only uses raw malloc/free to avoid recursive behaviour. + * This table is read-only after NMT is initialized to mitigate the cost of synchronization. + */ +class PreInitTable extends AbstractUninterruptibleHashtable { + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected UninterruptibleEntry[] createTable(int length) { + return new UninterruptibleEntry[length]; + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected boolean isEqual(UninterruptibleEntry a, UninterruptibleEntry b) { + return a.getHash() == b.getHash(); + } + + /** This override is necessary to use LibC directly. */ + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected UninterruptibleEntry copyToHeap(UninterruptibleEntry valueOnStack) { + UnsignedWord size = SizeOf.unsigned(UninterruptibleEntry.class); + UninterruptibleEntry pointerOnHeap = ImageSingletons.lookup(LibCSupport.class).malloc(size); + if (pointerOnHeap.isNonNull()) { + NativeMemoryTracking.recordMallocWithoutHeader(size, NmtFlag.mtNMT.ordinal()); + UnmanagedMemoryUtil.copy((Pointer) valueOnStack, (Pointer) pointerOnHeap, SizeOf.unsigned(UninterruptibleEntry.class)); + return pointerOnHeap; + } + return WordFactory.nullPointer(); + } + + /** + * Since these native memory hashtable nodes don't have headers, this is necessary to avoid + * re-entering NMT code. Use LibC directly. + */ + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected void free(UninterruptibleEntry entry) { + size--; + ImageSingletons.lookup(LibCSupport.class).free(entry); + NativeMemoryTracking.deaccountMallocWithoutHeader(SizeOf.unsigned(UninterruptibleEntry.class).rawValue(), NmtFlag.mtNMT.ordinal()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void remove(Pointer ptr) { + UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); + entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); + remove(entry); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean putIfAbsent(Pointer ptr) { + UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); + entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); + return putIfAbsent(entry); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public UninterruptibleEntry get(Pointer ptr) { + UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); + entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); + return get(entry); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java new file mode 100644 index 000000000000..e8820ecc8e57 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; + +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; + +@RawStructure +public interface ReturnAddress extends PointerBase { + @RawField + Pointer get(); + + @RawField + void set(Pointer value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index f7557013a637..66cb3663e655 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -56,6 +56,7 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; import com.oracle.svm.core.threadlocal.FastThreadLocal; @@ -240,7 +241,7 @@ public IsolateThread allocateIsolateThread(int isolateThreadSize) { UnsignedWord alignment = WordFactory.unsigned(64); UnsignedWord memorySize = WordFactory.unsigned(isolateThreadSize).add(alignment); - Pointer memory = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(memorySize); + Pointer memory = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(memorySize, NmtFlag.mtThread.ordinal()); if (memory.isNull()) { return WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties index 095bac6acbb2..c72a60578e4a 100644 --- a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties +++ b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties @@ -6,6 +6,7 @@ Args= \ --features=com.oracle.svm.test.NoProviderConstructorServiceLoaderTest$TestFeature \ --features=com.oracle.svm.test.NativeImageResourceUtils$TestFeature \ --features=com.oracle.svm.test.jfr.JfrTestFeature \ + --features=com.oracle.svm.core.nmt.NmtTestFeature \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-exports=org.graalvm.nativeimage.base/com.oracle.svm.util=ALL-UNNAMED \ --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient \ diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java new file mode 100644 index 000000000000..4ae6149fe5cd --- /dev/null +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.test.nmt; + +import com.oracle.svm.core.nmt.NmtFlag; + +import org.junit.BeforeClass; +import org.junit.Test; + +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.WordFactory; +import org.graalvm.word.Pointer; +import static org.junit.Assert.assertTrue; + +public class TestBasic { + private static final int ALLOCATION_SIZE = 1024 * 16; + private static final int RESERVE_SIZE = ALLOCATION_SIZE; + private static final int REALLOC_SIZE = ALLOCATION_SIZE / 2; + private static final int COMMIT_SIZE = RESERVE_SIZE / 2; + + /** + * This both initializes NMT and does some basic checks to verify pre-init allocations are + * handled. + */ + @BeforeClass + public static void setup() { + // Allocate some memory and check it's being tracked. + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + + // Realloc previously allocated memory and check NMT has tracked it correctly + Pointer reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); + + // Free the memory and ensure the tracking is now zeroed. + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + + // Allocate a new block that will live across the initialization boundary + ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + + // We must initialize NMT here so that other tests can use it. + NativeMemoryTracking.initialize(true); + + // Reallocate a block that has lived across the initialization boundary. Verify its been + // recorded. + reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); + + // Free the memory and verify we're back at zero. + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } + + @Test + public void testMalloc() throws Throwable { + assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + + assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } + + @Test + public void testCalloc() throws Throwable { + assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + + assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } + + @Test + public void testRealloc() throws Throwable { + assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); + + Pointer reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); + + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); + assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } +}