diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java index a9d475d4ed0f..3551cf7f486a 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -62,8 +62,7 @@ private UnmanagedMemory() { /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is undefined. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -79,8 +78,7 @@ public static T malloc(UnsignedWord size) { /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is undefined. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -92,8 +90,7 @@ public static T malloc(int size) { /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is set to 0. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -109,8 +106,7 @@ public static T calloc(UnsignedWord size) { /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is set to 0. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -124,9 +120,8 @@ public static T calloc(int size) { * If the new size is larger than the old size, the content of the additional memory is * undefined. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when - * allocation fails. + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. In that case, the old data is not deallocated and remains unchanged. * * @since 19.0 */ @@ -139,7 +134,8 @@ public static T realloc(T ptr, UnsignedWord size) { } /** - * Frees unmanaged memory that was previously allocated using methods of this class. + * Frees unmanaged memory that was previously allocated using methods of this class. This method + * is a no-op if the given pointer is {@code null}. * * @since 19.0 */ diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java index c18ab87631e8..dc013c8aa826 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -43,7 +43,7 @@ import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; -/** Implemented by operating-system specific code. */ +/** Implemented by platform or operating-system specific code. */ public interface UnmanagedMemorySupport { T malloc(UnsignedWord size); diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index ba7c99edda4e..f61bbea50cf3 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -6,6 +6,7 @@ This changelog summarizes major changes to GraalVM Native Image. * (GR-51106) Fields that are accessed via a `VarHandle` or `MethodHandle` are no longer marked as "unsafe accessed" when the `VarHandle`/`MethodHandle` can be fully intrinsified. * (GR-49996) Ensure explicitly set image name (e.g., via `-o imagename`) is not accidentally overwritten by `-jar jarfile` option. * (GR-48683) Together with Red Hat, we added partial support for the JFR event `OldObjectSample`. +* (GR-51851) Together with Red Hat, we added initial support for native memory tracking (`--enable-monitoring=nmt`). * (GR-47109) Together with Red Hat, we added support for JFR event throttling and the event `ObjectAllocationSample`. * (GR-52030) Add a stable name for `Proxy` types in Native Image. The name `$Proxy[id]` is replaced by `$Proxy.s[hashCode]` where `hashCode` is computed using the names of the `Proxy` interfaces, the name of the class loader and the name of the module if it is not a dynamic module. diff --git a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java index ee11c2f4a8b3..5e72875f8fa1 100644 --- a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java +++ b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java @@ -61,9 +61,6 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; -import com.oracle.svm.core.jni.headers.JNIMode; -import jdk.graal.compiler.core.common.NumUtil; -import jdk.graal.compiler.java.LambdaUtils; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CEntryPoint; @@ -88,6 +85,7 @@ import com.oracle.svm.core.jni.headers.JNIEnvironment; import com.oracle.svm.core.jni.headers.JNIFieldId; import com.oracle.svm.core.jni.headers.JNIMethodId; +import com.oracle.svm.core.jni.headers.JNIMode; import com.oracle.svm.core.jni.headers.JNINativeMethod; import com.oracle.svm.core.jni.headers.JNIObjectHandle; import com.oracle.svm.core.jni.headers.JNIValue; @@ -106,6 +104,9 @@ import com.oracle.svm.jvmtiagentbase.jvmti.JvmtiInterface; import com.oracle.svm.jvmtiagentbase.jvmti.JvmtiLocationFormat; +import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.java.LambdaUtils; + /** * Intercepts events of interest via breakpoints in Java code. *

diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java index f3d2b586e41e..0a8ef5e06db5 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java @@ -41,7 +41,7 @@ import com.oracle.svm.core.BaseProcessPropertiesSupport; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.posix.headers.Dlfcn; import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Stdlib; @@ -103,7 +103,7 @@ public String getObjectFile(PointerBase symbolAddress) { try { return CTypeConversion.toJavaString(realpath); } finally { - LibC.free(realpath); + UntrackedNullableNativeMemory.free(realpath); } } @@ -157,14 +157,14 @@ protected static String realpath(String path) { * pointer to it, so I have to free it. */ try (CCharPointerHolder pathHolder = CTypeConversion.toCString(path)) { - final CCharPointer realpathPointer = Stdlib.realpath(pathHolder.get(), WordFactory.nullPointer()); - if (realpathPointer.isNull()) { + CCharPointer realpath = Stdlib.realpath(pathHolder.get(), WordFactory.nullPointer()); + if (realpath.isNull()) { /* Failure to find a real path. */ return null; } else { /* Success */ - final String result = CTypeConversion.toJavaString(realpathPointer); - LibC.free(realpathPointer); + String result = CTypeConversion.toJavaString(realpath); + UntrackedNullableNativeMemory.free(realpath); return result; } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java index b318b1b93cd9..af5da227d5fd 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java @@ -32,7 +32,6 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.SignedWord; import org.graalvm.word.UnsignedWord; @@ -42,6 +41,7 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.os.AbstractRawFileOperationSupport; import com.oracle.svm.core.os.AbstractRawFileOperationSupport.RawFileOperationSupportHolder; import com.oracle.svm.core.posix.headers.Errno; @@ -58,7 +58,7 @@ public PosixRawFileOperationSupport(boolean useNativeByteOrder) { @Override public CCharPointer allocateCPath(String path) { byte[] data = path.getBytes(); - CCharPointer filename = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(data.length + 1)); + CCharPointer filename = UntrackedNullableNativeMemory.malloc(WordFactory.unsigned(data.length + 1)); if (filename.isNull()) { return WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java index fe0e9ccbbc9a..283f30947ae1 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java @@ -37,8 +37,8 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.SystemPropertiesSupport; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.posix.PosixSystemPropertiesSupport; import com.oracle.svm.core.posix.headers.Limits; import com.oracle.svm.core.posix.headers.Stdlib; @@ -105,7 +105,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatform(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - LibC.free(osVersionStr); + UntrackedNullableNativeMemory.free(osVersionStr); return osVersionValue; } } else { @@ -120,7 +120,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatformFallback(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - LibC.free(osVersionStr); + UntrackedNullableNativeMemory.free(osVersionStr); return osVersionValue; } return osVersionValue = "Unknown"; diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java index 1942cde8ee0d..fffe6ebe1034 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java @@ -41,15 +41,12 @@ import java.nio.ByteBuffer; -import jdk.graal.compiler.core.common.NumUtil; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -64,6 +61,8 @@ import com.oracle.svm.core.jvmstat.PerfManager; import com.oracle.svm.core.jvmstat.PerfMemoryPrologue; import com.oracle.svm.core.jvmstat.PerfMemoryProvider; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.RawFileOperationSupport; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; import com.oracle.svm.core.os.VirtualMemoryProvider; @@ -80,6 +79,8 @@ import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Unistd; +import jdk.graal.compiler.core.common.NumUtil; + /** * This class uses high-level JDK features at the moment. In the future, we will need to rewrite * this code so that it can be executed during the isolate startup (i.e., in uninterruptible code), @@ -157,7 +158,7 @@ private static String getUserName(int uid) { } /* Retrieve the username and copy it to a String object. */ - CCharPointer pwBuf = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(bufSize)); + CCharPointer pwBuf = NullableNativeMemory.malloc(WordFactory.unsigned(bufSize), NmtCategory.JvmStat); if (pwBuf.isNull()) { return null; } @@ -182,7 +183,7 @@ private static String getUserName(int uid) { return CTypeConversion.toJavaString(pwName); } finally { - UnmanagedMemory.free(pwBuf); + NullableNativeMemory.free(pwBuf); } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index ef05aa4fb2b4..93d375f39bf1 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -24,19 +24,16 @@ */ package com.oracle.svm.core.posix.thread; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platform.HOSTED_ONLY; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -50,6 +47,8 @@ import com.oracle.svm.core.annotate.TargetClass; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.posix.PosixUtils; import com.oracle.svm.core.posix.headers.Errno; import com.oracle.svm.core.posix.headers.Pthread; @@ -321,7 +320,7 @@ final class PosixParker extends Parker { // Allocate mutex and condition in a single step so that they are adjacent in memory. UnsignedWord mutexSize = SizeOf.unsigned(pthread_mutex_t.class); UnsignedWord condSize = SizeOf.unsigned(pthread_cond_t.class); - Pointer memory = UnmanagedMemory.malloc(mutexSize.add(condSize.multiply(2))); + Pointer memory = NativeMemory.malloc(mutexSize.add(condSize.multiply(2)), NmtCategory.Threading); mutex = (pthread_mutex_t) memory; relativeCond = (pthread_cond_t) memory.add(mutexSize); absoluteCond = (pthread_cond_t) memory.add(mutexSize).add(condSize); @@ -437,7 +436,7 @@ protected void release() { status = Pthread.pthread_mutex_destroy(mutex); assert status == 0; - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(mutex); + NativeMemory.free(mutex); mutex = WordFactory.nullPointer(); } } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java index fc6e0012e97e..10a35b294b30 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java @@ -45,8 +45,9 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.SystemPropertiesSupport; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.VMError; import com.oracle.svm.core.windows.headers.FileAPI; import com.oracle.svm.core.windows.headers.LibLoaderAPI; @@ -245,29 +246,29 @@ public Pair getOsNameAndVersion() { break; } - VoidPointer versionInfo = LibC.malloc(WordFactory.unsigned(versionSize)); + VoidPointer versionInfo = NullableNativeMemory.malloc(WordFactory.unsigned(versionSize), NmtCategory.Internal); if (versionInfo.isNull()) { break; } + try { + if (WinVer.GetFileVersionInfoW(kernel32Path, 0, versionSize, versionInfo) == 0) { + break; + } - if (WinVer.GetFileVersionInfoW(kernel32Path, 0, versionSize, versionInfo) == 0) { - LibC.free(versionInfo); - break; - } + WindowsLibC.WCharPointer rootPath = NonmovableArrays.addressOf(NonmovableArrays.fromImageHeap(ROOT_PATH), 0); + WordPointer fileInfoPointer = UnsafeStackValue.get(WordPointer.class); + CIntPointer lengthPointer = UnsafeStackValue.get(CIntPointer.class); + if (WinVer.VerQueryValueW(versionInfo, rootPath, fileInfoPointer, lengthPointer) == 0) { + break; + } - WindowsLibC.WCharPointer rootPath = NonmovableArrays.addressOf(NonmovableArrays.fromImageHeap(ROOT_PATH), 0); - WordPointer fileInfoPointer = UnsafeStackValue.get(WordPointer.class); - CIntPointer lengthPointer = UnsafeStackValue.get(CIntPointer.class); - if (WinVer.VerQueryValueW(versionInfo, rootPath, fileInfoPointer, lengthPointer) == 0) { - LibC.free(versionInfo); - break; + VerRsrc.VS_FIXEDFILEINFO fileInfo = fileInfoPointer.read(); + majorVersion = (short) (fileInfo.dwProductVersionMS() >> 16); // HIWORD + minorVersion = (short) fileInfo.dwProductVersionMS(); // LOWORD + buildNumber = (short) (fileInfo.dwProductVersionLS() >> 16); // HIWORD + } finally { + NullableNativeMemory.free(versionInfo); } - - VerRsrc.VS_FIXEDFILEINFO fileInfo = fileInfoPointer.read(); - majorVersion = (short) (fileInfo.dwProductVersionMS() >> 16); // HIWORD - minorVersion = (short) fileInfo.dwProductVersionMS(); // LOWORD - buildNumber = (short) (fileInfo.dwProductVersionLS() >> 16); // HIWORD - LibC.free(versionInfo); } while (false); String osVersion = majorVersion + "." + minorVersion; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index bf1d2a4e5946..55c96fa1234a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -30,10 +30,6 @@ import java.util.Set; import org.graalvm.collections.EconomicMap; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionKey; -import jdk.graal.compiler.options.OptionType; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platform.WINDOWS; import org.graalvm.nativeimage.Platforms; @@ -43,10 +39,16 @@ import com.oracle.svm.core.option.APIOption; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; +import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.LogUtils; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionType; + public final class VMInspectionOptions { private static final String ENABLE_MONITORING_OPTION = "enable-monitoring"; private static final String MONITORING_DEFAULT_NAME = ""; @@ -57,12 +59,13 @@ public final class VMInspectionOptions { private static final String MONITORING_JMXCLIENT_NAME = "jmxclient"; private static final String MONITORING_JMXSERVER_NAME = "jmxserver"; private static final String MONITORING_THREADDUMP_NAME = "threaddump"; + private static final String MONITORING_NMT_NAME = "nmt"; private static final List MONITORING_ALL_VALUES = List.of(MONITORING_HEAPDUMP_NAME, MONITORING_JFR_NAME, MONITORING_JVMSTAT_NAME, MONITORING_JMXCLIENT_NAME, MONITORING_JMXSERVER_NAME, - MONITORING_THREADDUMP_NAME, MONITORING_ALL_NAME, MONITORING_DEFAULT_NAME); + MONITORING_THREADDUMP_NAME, MONITORING_NMT_NAME, MONITORING_ALL_NAME, MONITORING_DEFAULT_NAME); private static final String MONITORING_ALLOWED_VALUES_TEXT = "'" + MONITORING_HEAPDUMP_NAME + "', '" + MONITORING_JFR_NAME + "', '" + MONITORING_JVMSTAT_NAME + "', '" + MONITORING_JMXSERVER_NAME + - "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', or '" + MONITORING_ALL_NAME + - "' (deprecated behavior: defaults to '" + MONITORING_ALL_NAME + "' if no argument is provided)"; + "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', '" + MONITORING_NMT_NAME + "' (experimental), or '" + + MONITORING_ALL_NAME + "' (deprecated behavior: defaults to '" + MONITORING_ALL_NAME + "' if no argument is provided)"; static { assert MONITORING_ALL_VALUES.stream().allMatch(v -> MONITORING_DEFAULT_NAME.equals(v) || MONITORING_ALLOWED_VALUES_TEXT.contains(v)) : "A value is missing in the user-facing help text"; @@ -74,6 +77,12 @@ public final class VMInspectionOptions { public static final HostedOptionKey EnableMonitoringFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter(), VMInspectionOptions::validateEnableMonitoringFeatures); + @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // + public static final HostedOptionKey DumpRuntimeCompilationOnSignal = new HostedOptionKey<>(false); + + @Option(help = "Print native memory tracking statistics on shutdown if native memory tracking is enabled.", type = OptionType.User) // + public static final RuntimeOptionKey PrintNMTStatistics = new RuntimeOptionKey<>(false); + @Platforms(Platform.HOSTED_ONLY.class) public static void validateEnableMonitoringFeatures(@SuppressWarnings("unused") OptionKey optionKey) { Set enabledFeatures = getEnabledMonitoringFeatures(); @@ -163,8 +172,10 @@ public static boolean hasThreadDumpSupport() { return hasAllOrKeywordMonitoringSupport(MONITORING_THREADDUMP_NAME) || DeprecatedOptions.DumpThreadStacksOnSignal.getValue(); } - @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // - public static final HostedOptionKey DumpRuntimeCompilationOnSignal = new HostedOptionKey<>(false); + @Fold + public static boolean hasNativeMemoryTrackingSupport() { + return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME); + } static class DeprecatedOptions { @Option(help = "Enables features that allow the VM to be inspected during run time.", type = OptionType.User, // diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java index 1b765e534da2..b0390c59a113 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java @@ -28,11 +28,9 @@ import java.nio.ByteBuffer; import java.util.Arrays; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.type.CTypeConversion; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -51,6 +49,8 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.util.VMError; @@ -84,7 +84,7 @@ public final class NonmovableArrays { @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static > T createArray(int length, Class arrayType) { + private static > T createArray(int length, Class arrayType, NmtCategory nmtCategory) { if (SubstrateUtil.HOSTED) { Class componentType = arrayType.getComponentType(); Object array = Array.newInstance(componentType, length); @@ -93,7 +93,7 @@ private static > T createArray(int length, Class DynamicHub hub = SubstrateUtil.cast(arrayType, DynamicHub.class); assert LayoutEncoding.isArray(hub.getLayoutEncoding()); UnsignedWord size = LayoutEncoding.getArrayAllocationSize(hub.getLayoutEncoding(), length); - Pointer array = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(size); + Pointer array = NullableNativeMemory.calloc(size, nmtCategory); if (array.isNull()) { throw OUT_OF_MEMORY_ERROR; } @@ -181,8 +181,8 @@ public static > T nullArray() { * array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableArray createByteArray(int nbytes) { - return createArray(nbytes, byte[].class); + public static NonmovableArray createByteArray(int nbytes, NmtCategory nmtCategory) { + return createArray(nbytes, byte[].class, nmtCategory); } /** @@ -193,8 +193,8 @@ public static NonmovableArray createByteArray(int nbytes) { * of a Java array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableArray createIntArray(int length) { - return createArray(length, int[].class); + public static NonmovableArray createIntArray(int length, NmtCategory nmtCategory) { + return createArray(length, int[].class, nmtCategory); } /** @@ -205,8 +205,8 @@ public static NonmovableArray createIntArray(int length) { * of a Java array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableArray createWordArray(int length) { - return createArray(length, WordBase[].class); + public static NonmovableArray createWordArray(int length, NmtCategory nmtCategory) { + return createArray(length, WordBase[].class, nmtCategory); } /** @@ -220,16 +220,16 @@ public static NonmovableArray createWordArray(int length * that of a Java array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableObjectArray createObjectArray(Class arrayType, int length) { + public static NonmovableObjectArray createObjectArray(Class arrayType, int length, NmtCategory nmtCategory) { assert (SubstrateUtil.HOSTED ? (arrayType.isArray() && !arrayType.getComponentType().isPrimitive()) : LayoutEncoding.isObjectArray(SubstrateUtil.cast(arrayType, DynamicHub.class).getLayoutEncoding())) : "must be an object array type"; - return createArray(length, arrayType); + return createArray(length, arrayType, nmtCategory); } /** @see java.util.Arrays#copyOf */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableObjectArray copyOfObjectArray(T[] source, int newLength) { - NonmovableObjectArray array = createArray(newLength, source.getClass()); + public static NonmovableObjectArray copyOfObjectArray(T[] source, int newLength, NmtCategory nmtCategory) { + NonmovableObjectArray array = createArray(newLength, source.getClass(), nmtCategory); int copyLength = (source.length < newLength) ? source.length : newLength; for (int i = 0; i < copyLength; i++) { setObject(array, i, source[i]); @@ -239,8 +239,8 @@ public static NonmovableObjectArray copyOfObjectArray(T[] source, int new /** Same as {@link #copyOfObjectArray} with a {@code newLength} of the array length. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableObjectArray copyOfObjectArray(T[] source) { - return copyOfObjectArray(source, source.length); + public static NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory) { + return copyOfObjectArray(source, source.length, nmtCategory); } public static byte[] heapCopyOfByteArray(NonmovableArray source) { @@ -299,7 +299,7 @@ private static T arraycopyToHeap(NonmovableArray src, int srcPos, T dest, /** Releases an array created at runtime. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void releaseUnmanagedArray(NonmovableArray array) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(array); + NullableNativeMemory.free(array); untrackUnmanagedArray(array); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java index a23112ad2cce..251b914c27ec 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java @@ -24,6 +24,11 @@ */ package com.oracle.svm.core.c; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + import com.oracle.svm.core.JavaMemoryUtil; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.Uninterruptible; @@ -32,17 +37,13 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.nodes.java.ArrayLengthNode; import jdk.graal.compiler.word.Word; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; -import org.graalvm.word.Pointer; -import org.graalvm.word.PointerBase; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; /** * Support for allocating and accessing primitive element arrays created in unmanaged memory. They @@ -57,11 +58,11 @@ public final class UnmanagedPrimitiveArrays { @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static > T createArray(int length, Class arrayType) { + public static > T createArray(int length, Class arrayType, NmtCategory nmtCategory) { DynamicHub hub = SubstrateUtil.cast(arrayType, DynamicHub.class); VMError.guarantee(LayoutEncoding.isPrimitiveArray(hub.getLayoutEncoding())); UnsignedWord size = WordFactory.unsigned(length).shiftLeft(LayoutEncoding.getArrayIndexShift(hub.getLayoutEncoding())); - Pointer array = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(size); + Pointer array = NullableNativeMemory.calloc(size, nmtCategory); if (array.isNull()) { throw OUT_OF_MEMORY_ERROR; } @@ -88,7 +89,7 @@ public static void untrackUnmanagedArray(UnmanagedPrimitiveArray array) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void releaseUnmanagedArray(UnmanagedPrimitiveArray array) { untrackUnmanagedArray(array); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(array); + NullableNativeMemory.free(array); } /** Returns a pointer to the address of the given index of an array. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java index 5a47755a8c91..a64bc2ac4104 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java @@ -31,7 +31,6 @@ import org.graalvm.nativeimage.Isolates.CreateIsolateParameters; import org.graalvm.nativeimage.Isolates.IsolateException; import org.graalvm.nativeimage.Isolates.ProtectionDomain; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CCharPointerPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; @@ -42,6 +41,8 @@ import com.oracle.svm.core.c.function.CEntryPointNativeFunctions.IsolateThreadPointer; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.os.MemoryProtectionProvider; import com.oracle.svm.core.os.MemoryProtectionProvider.UnsupportedDomainException; @@ -89,7 +90,7 @@ public static IsolateThread createIsolate(CreateIsolateParameters parameters, bo // Internally, we use C-style arguments, i.e., the first argument is reserved for // the name of the binary. We use null when isolates are created manually. argc = isolateArgCount + 1; - argv = UnmanagedMemory.malloc(SizeOf.unsigned(CCharPointerPointer.class).multiply(argc)); + argv = NativeMemory.malloc(SizeOf.unsigned(CCharPointerPointer.class).multiply(argc), NmtCategory.Internal); argv.write(0, WordFactory.nullPointer()); pointerHolders = new CTypeConversion.CCharPointerHolder[isolateArgCount]; @@ -121,7 +122,7 @@ public static IsolateThread createIsolate(CreateIsolateParameters parameters, bo for (CTypeConversion.CCharPointerHolder ph : pointerHolders) { ph.close(); } - UnmanagedMemory.free(params.getArgv()); + NativeMemory.free(params.getArgv()); } throwOnError(result); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java index c58ab2d74eda..3d3cfb28896e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java @@ -58,6 +58,7 @@ import com.oracle.svm.core.meta.SharedField; import com.oracle.svm.core.meta.SharedMethod; import com.oracle.svm.core.meta.SharedType; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.util.ByteArrayReader; import com.oracle.svm.core.util.Counter; @@ -129,12 +130,10 @@ private void encodeAllAndInstall(CodeInfo target, ReferenceAdjuster adjuster) { } @Uninterruptible(reason = "Nonmovable object arrays are not visible to GC until installed in target.") - private static void install(CodeInfo target, JavaConstant[] objectConstantsArray, Class[] sourceClassesArray, - String[] sourceMethodNamesArray, ReferenceAdjuster adjuster) { - - NonmovableObjectArray frameInfoObjectConstants = adjuster.copyOfObjectConstantArray(objectConstantsArray); - NonmovableObjectArray> frameInfoSourceClasses = (sourceClassesArray != null) ? adjuster.copyOfObjectArray(sourceClassesArray) : NonmovableArrays.nullArray(); - NonmovableObjectArray frameInfoSourceMethodNames = (sourceMethodNamesArray != null) ? adjuster.copyOfObjectArray(sourceMethodNamesArray) : NonmovableArrays.nullArray(); + private static void install(CodeInfo target, JavaConstant[] objectConstants, Class[] sourceClasses, String[] sourceMethodNames, ReferenceAdjuster adjuster) { + NonmovableObjectArray frameInfoObjectConstants = adjuster.copyOfObjectConstantArray(objectConstants, NmtCategory.Code); + NonmovableObjectArray> frameInfoSourceClasses = (sourceClasses != null) ? adjuster.copyOfObjectArray(sourceClasses, NmtCategory.Code) : NonmovableArrays.nullArray(); + NonmovableObjectArray frameInfoSourceMethodNames = (sourceMethodNames != null) ? adjuster.copyOfObjectArray(sourceMethodNames, NmtCategory.Code) : NonmovableArrays.nullArray(); CodeInfoAccess.setEncodings(target, frameInfoObjectConstants, frameInfoSourceClasses, frameInfoSourceMethodNames); } @@ -339,9 +338,9 @@ private void encodeIPData() { writeEncodedFrameInfo(encodingBuffer, data, entryFlags); } - codeInfoIndex = NonmovableArrays.createByteArray(TypeConversion.asU4(indexBuffer.getBytesWritten())); + codeInfoIndex = NonmovableArrays.createByteArray(TypeConversion.asU4(indexBuffer.getBytesWritten()), NmtCategory.Code); indexBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(codeInfoIndex)); - codeInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asU4(encodingBuffer.getBytesWritten())); + codeInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asU4(encodingBuffer.getBytesWritten()), NmtCategory.Code); encodingBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(codeInfoEncodings)); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java index 5c77dc245957..19908dd5523f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java @@ -29,17 +29,19 @@ import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; import org.graalvm.collections.Equivalence; -import jdk.graal.compiler.core.common.util.FrequencyEncoder; -import jdk.graal.compiler.core.common.util.TypeConversion; -import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; -import jdk.graal.compiler.graph.NodeSourcePosition; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.c.NonmovableArrays; import com.oracle.svm.core.c.NonmovableObjectArray; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.ByteArrayReader; +import jdk.graal.compiler.core.common.util.FrequencyEncoder; +import jdk.graal.compiler.core.common.util.TypeConversion; +import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; +import jdk.graal.compiler.graph.NodeSourcePosition; + public class DeoptimizationSourcePositionEncoder { private final FrequencyEncoder objectConstants; @@ -54,10 +56,10 @@ public void encodeAndInstall(List deoptSourcePositions, Code UnsafeArrayTypeWriter encodingBuffer = UnsafeArrayTypeWriter.create(ByteArrayReader.supportsUnalignedMemoryAccess()); EconomicMap sourcePositionStartOffsets = EconomicMap.create(Equivalence.IDENTITY_WITH_SYSTEM_HASHCODE); - NonmovableArray deoptStartOffsets = NonmovableArrays.createIntArray(deoptSourcePositions.size()); + NonmovableArray deoptStartOffsets = NonmovableArrays.createIntArray(deoptSourcePositions.size(), NmtCategory.Code); encodeSourcePositions(deoptSourcePositions, sourcePositionStartOffsets, deoptStartOffsets, encodingBuffer); - NonmovableArray deoptEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten())); + NonmovableArray deoptEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten()), NmtCategory.Code); encodingBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(deoptEncodings)); install(target, deoptStartOffsets, deoptEncodings, encodedObjectConstants, deoptSourcePositions, adjuster); @@ -67,7 +69,7 @@ public void encodeAndInstall(List deoptSourcePositions, Code private static void install(CodeInfo target, NonmovableArray deoptStartOffsets, NonmovableArray deoptEncodings, Object[] encodedObjectConstants, List deoptSourcePositions, ReferenceAdjuster adjuster) { - NonmovableObjectArray deoptObjectConstants = adjuster.copyOfObjectArray(encodedObjectConstants); + NonmovableObjectArray deoptObjectConstants = adjuster.copyOfObjectArray(encodedObjectConstants, NmtCategory.Code); RuntimeCodeInfoAccess.setDeoptimizationMetadata(target, deoptStartOffsets, deoptEncodings, deoptObjectConstants); afterInstallation(deoptStartOffsets, deoptEncodings, deoptSourcePositions, deoptObjectConstants, adjuster); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java index 5e06b89ba6ca..1227003e734a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java @@ -56,6 +56,7 @@ import com.oracle.svm.core.meta.SharedField; import com.oracle.svm.core.meta.SharedMethod; import com.oracle.svm.core.meta.SharedType; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.sampler.CallStackFrameMethodData; import com.oracle.svm.core.sampler.CallStackFrameMethodInfo; import com.oracle.svm.core.util.ByteArrayReader; @@ -909,7 +910,7 @@ private NonmovableArray encodeFrameDatas() { assert frameMetadata.writeFrameVerificationInfo(data, encoders); } } - NonmovableArray frameInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten())); + NonmovableArray frameInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten()), NmtCategory.Code); encodingBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(frameInfoEncodings)); return frameInfoEncodings; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java index fec3fde15470..4c8140249588 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.c.NonmovableObjectArray; import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.heap.UnknownPrimitiveField; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.word.Word; @@ -71,7 +72,7 @@ public class ImageCodeInfo { @Platforms(Platform.HOSTED_ONLY.class) ImageCodeInfo() { - NonmovableObjectArray objfields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT); + NonmovableObjectArray objfields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT, NmtCategory.Code); NonmovableArrays.setObject(objfields, CodeInfoImpl.NAME_OBJFIELD, CODE_INFO_NAME); // The image code info is never invalidated, so we consider it as always tethered. NonmovableArrays.setObject(objfields, CodeInfoImpl.TETHER_OBJFIELD, new CodeInfoTether(true)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java index f2c7f21523a7..63a53f6691ca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java @@ -27,8 +27,6 @@ import java.util.ArrayList; import java.util.List; -import jdk.graal.compiler.code.CompilationResult; -import jdk.graal.compiler.debug.DebugContext; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -40,6 +38,10 @@ import com.oracle.svm.core.code.InstalledCodeObserver.InstalledCodeObserverHandle; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.meta.SharedMethod; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.code.CompilationResult; +import jdk.graal.compiler.debug.DebugContext; @AutomaticallyRegisteredImageSingleton public final class InstalledCodeObserverSupport { @@ -74,7 +76,7 @@ public static NonmovableArray installObservers(Inst if (observers.length == 0) { return NonmovableArrays.nullArray(); } - NonmovableArray observerHandles = NonmovableArrays.createWordArray(observers.length); + NonmovableArray observerHandles = NonmovableArrays.createWordArray(observers.length, NmtCategory.Code); for (int i = 0; i < observers.length; i++) { InstalledCodeObserverHandle handle = observers[i].install(); NonmovableArrays.setWord(observerHandles, i, handle); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java index bc577142320c..c7bafc4dfe7c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java @@ -31,6 +31,7 @@ import com.oracle.svm.core.c.NonmovableArrays; import com.oracle.svm.core.c.NonmovableObjectArray; import com.oracle.svm.core.meta.DirectSubstrateObjectConstant; +import com.oracle.svm.core.nmt.NmtCategory; import jdk.vm.ci.meta.JavaConstant; @@ -59,8 +60,8 @@ public void setConstantTargetAt(PointerBase address, int length, JavaConstant co @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public NonmovableObjectArray copyOfObjectArray(T[] source) { - return NonmovableArrays.copyOfObjectArray(source); + public NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory) { + return NonmovableArrays.copyOfObjectArray(source, nmtCategory); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java index 55c69b6fc7f3..124e31840391 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java @@ -26,7 +26,6 @@ import java.nio.ByteOrder; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -35,7 +34,9 @@ import com.oracle.svm.core.c.NonmovableObjectArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.nmt.NmtCategory; +import jdk.graal.compiler.api.replacements.Fold; import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.JavaConstant; @@ -53,8 +54,8 @@ public interface ReferenceAdjuster { void setConstantTargetAt(PointerBase address, int length, JavaConstant constant); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - default NonmovableObjectArray copyOfObjectConstantArray(T[] constants) { - NonmovableObjectArray objects = NonmovableArrays.createObjectArray(Object[].class, constants.length); + default NonmovableObjectArray copyOfObjectConstantArray(T[] constants, NmtCategory nmtCategory) { + NonmovableObjectArray objects = NonmovableArrays.createObjectArray(Object[].class, constants.length, nmtCategory); for (int i = 0; i < constants.length; i++) { setConstantTargetInArray(objects, i, (JavaConstant) constants[i]); } @@ -62,7 +63,7 @@ default NonmovableObjectArray copyOfObjectConstantA } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - NonmovableObjectArray copyOfObjectArray(T[] source); + NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory); /** Indicates whether all object references have been written. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java index 50a389d42d1e..6d2af1753d59 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java @@ -46,6 +46,7 @@ import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.deopt.SubstrateInstalledCode; import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.StackFrameVisitor; @@ -192,7 +193,7 @@ private void enlargeTable() { if (newTableSize < INITIAL_TABLE_SIZE) { newTableSize = INITIAL_TABLE_SIZE; } - NonmovableArray newCodeInfos = NonmovableArrays.createWordArray(newTableSize); + NonmovableArray newCodeInfos = NonmovableArrays.createWordArray(newTableSize, NmtCategory.Code); if (codeInfos.isNonNull()) { NonmovableArrays.arraycopy(codeInfos, 0, newCodeInfos, 0, NonmovableArrays.lengthOf(codeInfos)); NonmovableArrays.releaseUnmanagedArray(codeInfos); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java index 334672f9fc6a..4b5d4b0a5d13 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java @@ -26,10 +26,7 @@ import java.util.EnumSet; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CodePointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -45,6 +42,9 @@ import com.oracle.svm.core.heap.CodeReferenceMapDecoder; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectReferenceVisitor; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.VMError; @@ -203,12 +203,12 @@ public static boolean walkObjectFields(CodeInfo info, ObjectReferenceVisitor vis } public static CodeInfo allocateMethodInfo() { - NonmovableObjectArray objectFields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT); + NonmovableObjectArray objectFields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT, NmtCategory.Code); return allocateMethodInfo(objectFields); } public static CodeInfo allocateMethodInfo(NonmovableObjectArray objectData) { - CodeInfoImpl info = UnmanagedMemory.calloc(CodeInfoAccess.getSizeOfCodeInfo()); + CodeInfoImpl info = NativeMemory.calloc(CodeInfoAccess.getSizeOfCodeInfo(), NmtCategory.Code); assert objectData.isNonNull() && NonmovableArrays.lengthOf(objectData) == CodeInfoImpl.OBJFIELDS_COUNT; info.setObjectFields(objectData); @@ -279,7 +279,7 @@ public static void free(CodeInfo info) { } impl.setState(CodeInfo.STATE_FREED); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(info); + NullableNativeMemory.free(info); } private static final NonmovableArrayAction GUARANTEE_ALL_OBJECTS_IN_IMAGE_HEAP_ACTION = new NonmovableArrayAction() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index 677a26df2e17..761481723d48 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -42,6 +42,7 @@ import com.oracle.svm.core.deopt.SubstrateInstalledCode; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.VMError; @@ -235,7 +236,7 @@ public boolean removeDuringGC(CodeInfo info) { private void add0(CodeInfo info) { addToSizeCounters(info); if (table.isNull()) { - table = NonmovableArrays.createWordArray(32); + table = NonmovableArrays.createWordArray(32, NmtCategory.Code); } int index; boolean resized; @@ -271,7 +272,7 @@ private boolean resize(int newLength) { return false; } NonmovableArray oldTable = table; - table = NonmovableArrays.createWordArray(newLength); + table = NonmovableArrays.createWordArray(newLength, NmtCategory.Code); for (int i = 0; i < oldLength; i++) { UntetheredCodeInfo tag = NonmovableArrays.getWord(oldTable, i); if (tag.isNonNull()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java index 8eb7cd7b84ac..f5b915281843 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java @@ -24,16 +24,16 @@ */ package com.oracle.svm.core.collections; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; /** * An uninterruptible hashtable with a fixed size that uses chaining in case of a collision. @@ -41,17 +41,19 @@ public abstract class AbstractUninterruptibleHashtable implements UninterruptibleHashtable { private static final int DEFAULT_TABLE_LENGTH = 2053; + private final NmtCategory nmtCategory; private final UninterruptibleEntry[] table; private int size; @Platforms(Platform.HOSTED_ONLY.class) - public AbstractUninterruptibleHashtable() { - this(DEFAULT_TABLE_LENGTH); + public AbstractUninterruptibleHashtable(NmtCategory nmtCategory) { + this(nmtCategory, DEFAULT_TABLE_LENGTH); } @Platforms(Platform.HOSTED_ONLY.class) @SuppressWarnings("this-escape") - public AbstractUninterruptibleHashtable(int primeLength) { + public AbstractUninterruptibleHashtable(NmtCategory nmtCategory, int primeLength) { + this.nmtCategory = nmtCategory; this.table = createTable(primeLength); this.size = 0; } @@ -67,7 +69,7 @@ public AbstractUninterruptibleHashtable(int primeLength) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected UninterruptibleEntry copyToHeap(UninterruptibleEntry pointerOnStack, UnsignedWord sizeToAlloc) { - UninterruptibleEntry pointerOnHeap = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(sizeToAlloc); + UninterruptibleEntry pointerOnHeap = NullableNativeMemory.malloc(sizeToAlloc, nmtCategory); if (pointerOnHeap.isNonNull()) { UnmanagedMemoryUtil.copy((Pointer) pointerOnStack, (Pointer) pointerOnHeap, sizeToAlloc); return pointerOnHeap; @@ -78,7 +80,7 @@ protected UninterruptibleEntry copyToHeap(UninterruptibleEntry pointerOnStack, U @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected void free(UninterruptibleEntry entry) { size--; - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(entry); + NullableNativeMemory.free(entry); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -95,6 +97,27 @@ protected UninterruptibleEntry insertEntry(UninterruptibleEntry valueOnStack) { return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean remove(UninterruptibleEntry valueOnStack) { + int index = Integer.remainderUnsigned(valueOnStack.getHash(), DEFAULT_TABLE_LENGTH); + UninterruptibleEntry entry = table[index]; + UninterruptibleEntry prev = WordFactory.nullPointer(); + while (entry.isNonNull()) { + if (isEqual(valueOnStack, entry)) { + if (prev.isNull()) { + table[index] = entry.getNext(); + } else { + prev.setNext(entry.getNext()); + } + free(entry); + return true; + } + prev = entry; + entry = entry.getNext(); + } + return false; + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getSize() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java index f9d7e0e3e337..6b3142d625bd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java @@ -24,16 +24,17 @@ */ package com.oracle.svm.core.collections; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.type.WordPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; public class GrowableWordArrayAccess { private static final int INITIAL_CAPACITY = 10; @@ -49,8 +50,8 @@ public static Word get(GrowableWordArray array, int i) { return array.getData().addressOf(i).read(); } - public static boolean add(GrowableWordArray array, Word element) { - if (array.getSize() == array.getCapacity() && !grow(array)) { + public static boolean add(GrowableWordArray array, Word element, NmtCategory nmtCategory) { + if (array.getSize() == array.getCapacity() && !grow(array, nmtCategory)) { return false; } @@ -61,14 +62,14 @@ public static boolean add(GrowableWordArray array, Word element) { public static void freeData(GrowableWordArray array) { if (array.isNonNull()) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(array.getData()); + NullableNativeMemory.free(array.getData()); array.setData(WordFactory.nullPointer()); array.setSize(0); array.setCapacity(0); } } - private static boolean grow(GrowableWordArray array) { + private static boolean grow(GrowableWordArray array, NmtCategory nmtCategory) { int newCapacity = computeNewCapacity(array); if (newCapacity < 0) { /* Overflow. */ @@ -77,13 +78,13 @@ private static boolean grow(GrowableWordArray array) { assert newCapacity >= INITIAL_CAPACITY; WordPointer oldData = array.getData(); - WordPointer newData = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(newCapacity).multiply(wordSize())); + WordPointer newData = NullableNativeMemory.malloc(WordFactory.unsigned(newCapacity).multiply(wordSize()), nmtCategory); if (newData.isNull()) { return false; } UnmanagedMemoryUtil.copyForward((Pointer) oldData, (Pointer) newData, WordFactory.unsigned(array.getSize()).multiply(wordSize())); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(oldData); + NullableNativeMemory.free(oldData); array.setData(newData); array.setCapacity(newCapacity); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java index 663a97993d84..65681458609d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java @@ -35,6 +35,7 @@ import jdk.graal.compiler.api.replacements.Fold; +/** Platform-independent LibC support. */ public class LibC { public static final int EXIT_CODE_ABORT = 99; @@ -68,26 +69,6 @@ public static T memset(T s, SignedWord c, UnsignedWord n return libc().memset(s, c, n); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T malloc(UnsignedWord size) { - return libc().malloc(size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T calloc(UnsignedWord nmemb, UnsignedWord size) { - return libc().calloc(nmemb, size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T realloc(PointerBase ptr, UnsignedWord size) { - return libc().realloc(ptr, size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void free(PointerBase ptr) { - libc().free(ptr); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void exit(int status) { libc().exit(status); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java index 6965ee9b356d..b7ed0b46fdae 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java @@ -31,7 +31,9 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.NativeMemory; +/** Platform-independent LibC support. Don't use this class directly, use {@link LibC} instead. */ public interface LibCSupport { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int errno(); @@ -51,15 +53,19 @@ public interface LibCSupport { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T memset(T s, SignedWord c, UnsignedWord n); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T malloc(UnsignedWord size); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T calloc(UnsignedWord nmemb, UnsignedWord size); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T realloc(PointerBase ptr, UnsignedWord size); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void free(PointerBase ptr); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java index e1f987790055..edaf84a07a39 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java @@ -33,13 +33,14 @@ import java.util.PrimitiveIterator; import java.util.Set; -import jdk.graal.compiler.core.common.util.TypeConversion; -import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; - import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.c.NonmovableArrays; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.ByteArrayReader; +import jdk.graal.compiler.core.common.util.TypeConversion; +import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; + public abstract class ReferenceMapEncoder { public interface OffsetIterator extends PrimitiveIterator.OfInt { @Override @@ -99,7 +100,7 @@ public NonmovableArray encodeAll() { encodeAll(sortedEntries); int length = TypeConversion.asS4(writeBuffer.getBytesWritten()); - NonmovableArray array = NonmovableArrays.createByteArray(length); + NonmovableArray array = NonmovableArrays.createByteArray(length, NmtCategory.Code); writeBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(array)); return array; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java index 04ef357f2423..942812612c05 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.heap.dump; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -34,7 +33,6 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -47,11 +45,15 @@ import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.coder.ByteStream; import com.oracle.svm.core.util.coder.ByteStreamAccess; import com.oracle.svm.core.util.coder.NativeCoder; import com.oracle.svm.core.util.coder.Pack200Coder; +import jdk.graal.compiler.api.replacements.Fold; + /** * Provides access to the encoded heap dump metadata that was prepared at image build-time. * @@ -140,19 +142,19 @@ public boolean initialize() { * structures so that we can abort right away in case that an allocation fails. */ UnsignedWord classInfosSize = WordFactory.unsigned(classInfoCount).multiply(SizeOf.get(ClassInfo.class)); - classInfos = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(classInfosSize); + classInfos = NullableNativeMemory.calloc(classInfosSize, NmtCategory.HeapDump); if (classInfos.isNull()) { return false; } UnsignedWord fieldStartsSize = WordFactory.unsigned(totalFieldCount).multiply(SizeOf.get(FieldInfoPointer.class)); - fieldInfoTable = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(fieldStartsSize); + fieldInfoTable = NullableNativeMemory.calloc(fieldStartsSize, NmtCategory.HeapDump); if (fieldInfoTable.isNull()) { return false; } UnsignedWord fieldNameTableSize = WordFactory.unsigned(fieldNameCount).multiply(SizeOf.get(FieldNamePointer.class)); - fieldNameTable = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(fieldNameTableSize); + fieldNameTable = NullableNativeMemory.calloc(fieldNameTableSize, NmtCategory.HeapDump); if (fieldNameTable.isNull()) { return false; } @@ -214,13 +216,13 @@ public boolean initialize() { * Must always be called, regardless if {@link #initialize} returned true or false. */ public void teardown() { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(classInfos); + NullableNativeMemory.free(classInfos); classInfos = WordFactory.nullPointer(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(fieldInfoTable); + NullableNativeMemory.free(fieldInfoTable); fieldInfoTable = WordFactory.nullPointer(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(fieldNameTable); + NullableNativeMemory.free(fieldNameTable); fieldNameTable = WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java index 95cffbb587bd..4f6db2db3570 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java @@ -28,8 +28,6 @@ import java.io.IOException; -import jdk.graal.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; @@ -37,7 +35,6 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -47,6 +44,7 @@ import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.os.RawFileOperationSupport; import com.oracle.svm.core.os.RawFileOperationSupport.FileCreationMode; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; @@ -55,6 +53,8 @@ import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.TimeUtils; +import jdk.graal.compiler.api.replacements.Fold; + public class HeapDumpSupportImpl extends HeapDumping { private final HeapDumpWriter writer; private final HeapDumpOperation heapDumpOperation; @@ -77,7 +77,7 @@ public void initializeDumpHeapOnOutOfMemoryError() { @Override public void teardownDumpHeapOnOutOfMemoryError() { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(outOfMemoryHeapDumpPath); + UntrackedNullableNativeMemory.free(outOfMemoryHeapDumpPath); outOfMemoryHeapDumpPath = WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java index 63338d5f2ee2..30fb293d6c67 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java @@ -71,6 +71,7 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.BufferedFileOperationSupport; import com.oracle.svm.core.os.BufferedFileOperationSupport.BufferedFile; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; @@ -443,7 +444,7 @@ private boolean dumpHeap0(RawFileDescriptor fd) { private boolean initialize(RawFileDescriptor fd) { assert topLevelRecordBegin == -1 && subRecordBegin == -1 && !error; - this.f = file().allocate(fd); + this.f = file().allocate(fd, NmtCategory.HeapDump); if (f.isNull()) { return false; } @@ -1340,7 +1341,7 @@ public void initialize(GrowableWordArray largeObjects) { @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") public boolean visitObject(Object obj) { if (isLarge(obj)) { - boolean added = GrowableWordArrayAccess.add(largeObjects, Word.objectToUntrackedPointer(obj)); + boolean added = GrowableWordArrayAccess.add(largeObjects, Word.objectToUntrackedPointer(obj), NmtCategory.HeapDump); if (!added) { Log.log().string("Failed to add an element to the large object list. Heap dump will be incomplete.").newline(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java index a20bee43c883..0d8a683fcc28 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java @@ -29,9 +29,7 @@ import java.lang.reflect.Method; import java.security.ProtectionDomain; -import jdk.graal.compiler.nodes.extended.MembarNode; import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.impl.UnsafeMemorySupport; import org.graalvm.word.WordFactory; @@ -46,26 +44,30 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.hub.PredefinedClassesSupport; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.nodes.extended.MembarNode; + @TargetClass(className = "jdk.internal.misc.Unsafe") @SuppressWarnings({"static-method", "unused"}) final class Target_jdk_internal_misc_Unsafe_Core { @Substitute private long allocateMemory0(long bytes) { - return UnmanagedMemory.malloc(WordFactory.unsigned(bytes)).rawValue(); + return NativeMemory.malloc(WordFactory.unsigned(bytes), NmtCategory.Unsafe).rawValue(); } @Substitute private long reallocateMemory0(long address, long bytes) { - return UnmanagedMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes)).rawValue(); + return NativeMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes), NmtCategory.Unsafe).rawValue(); } @Substitute private void freeMemory0(long address) { - UnmanagedMemory.free(WordFactory.unsigned(address)); + NativeMemory.free(WordFactory.unsigned(address)); } @Substitute diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java index 6875c685d0c2..925f43e24b0f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java @@ -32,11 +32,8 @@ import java.util.TimeZone; import org.graalvm.collections.EconomicMap; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionKey; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.impl.InternalPlatform; @@ -51,9 +48,13 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.handles.PrimitiveArrayView; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; + /** * The following classes aim to provide full support for time zones for native-image. This * substitution is necessary due to the reliance on JAVA_HOME in the JDK. @@ -100,7 +101,7 @@ private static String getSystemTimeZoneID(String javaHome) { CCharPointer tzId = LibCHelper.SVM_FindJavaTZmd(tzMappingsPtr, contentLen); String result = CTypeConversion.toJavaString(tzId); // SVM_FindJavaTZmd returns a newly allocated string - UnmanagedMemory.free(tzId); + UntrackedNullableNativeMemory.free(tzId); return result; } finally { if (refContent != null) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java index 31da845e8aa1..8c3a4389bbf8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java @@ -24,19 +24,20 @@ */ package com.oracle.svm.core.jfr; -import jdk.graal.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.api.replacements.Fold; + /** * Used to access the raw memory of a {@link JfrBuffer}. */ @@ -61,7 +62,7 @@ public static JfrBuffer allocate(JfrBufferType bufferType) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBuffer allocate(UnsignedWord dataSize, JfrBufferType bufferType) { UnsignedWord headerSize = JfrBufferAccess.getHeaderSize(); - JfrBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize)); + JfrBuffer result = NullableNativeMemory.malloc(headerSize.add(dataSize), NmtCategory.JFR); if (result.isNonNull()) { result.setSize(dataSize); result.setBufferType(bufferType); @@ -74,7 +75,7 @@ public static JfrBuffer allocate(UnsignedWord dataSize, JfrBufferType bufferType @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void free(JfrBuffer buffer) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + NullableNativeMemory.free(buffer); } @Uninterruptible(reason = "Prevent safepoints as those could change the flushed position.") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java index 17c202d7693a..320a0922a66c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java @@ -26,10 +26,8 @@ package com.oracle.svm.core.jfr; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; @@ -75,7 +73,7 @@ public void teardown() { } JfrBufferNode next = node.getNext(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(node); + JfrBufferNodeAccess.free(node); node = next; } head = WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java index 759db4b69ce4..a72e5118208c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java @@ -27,14 +27,14 @@ package com.oracle.svm.core.jfr; import org.graalvm.nativeimage.CurrentIsolate; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CIntPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.thread.NativeSpinLockUtils; import com.oracle.svm.core.thread.VMOperation; @@ -47,7 +47,7 @@ private JfrBufferNodeAccess() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBufferNode allocate(JfrBuffer buffer) { - JfrBufferNode node = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(JfrBufferNode.class)); + JfrBufferNode node = NullableNativeMemory.malloc(SizeOf.unsigned(JfrBufferNode.class), NmtCategory.JFR); if (node.isNonNull()) { node.setBuffer(buffer); node.setNext(WordFactory.nullPointer()); @@ -59,7 +59,7 @@ public static JfrBufferNode allocate(JfrBuffer buffer) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void free(JfrBufferNode node) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(node); + NullableNativeMemory.free(node); } /** Should be used instead of {@link JfrBufferNode#getBuffer}. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java index 689c3c37f9a0..c8b98185ca65 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java @@ -33,7 +33,6 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CIntPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -51,6 +50,8 @@ import com.oracle.svm.core.jfr.traceid.JfrTraceIdEpoch; import com.oracle.svm.core.jfr.utils.JfrVisited; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.sampler.SamplerSampleWriter; import com.oracle.svm.core.sampler.SamplerSampleWriterData; import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess; @@ -193,7 +194,7 @@ private JfrStackTraceTableEntry getOrPutStackTrace0(Pointer start, UnsignedWord * the thread-local buffer to the C heap because the thread-local buffer will be * overwritten or freed at some point. */ - Pointer to = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + Pointer to = NullableNativeMemory.malloc(size, NmtCategory.JFR); if (to.isNonNull()) { UnmanagedMemoryUtil.copy(start, to, size); entry.setRawStackTrace(to); @@ -205,7 +206,8 @@ private JfrStackTraceTableEntry getOrPutStackTrace0(Pointer start, UnsignedWord } /* Hashtable entry allocation failed. */ - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(to); + NullableNativeMemory.free(to); + to = WordFactory.nullPointer(); } /* Some allocation failed. */ @@ -303,6 +305,11 @@ public interface JfrStackTraceTableEntry extends JfrVisited { public static final class JfrStackTraceTable extends AbstractUninterruptibleHashtable { private static long nextId; + @Platforms(Platform.HOSTED_ONLY.class) + JfrStackTraceTable() { + super(NmtCategory.JFR); + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected JfrStackTraceTableEntry[] createTable(int size) { @@ -333,7 +340,7 @@ protected UninterruptibleEntry copyToHeap(UninterruptibleEntry valueOnStack) { protected void free(UninterruptibleEntry entry) { JfrStackTraceTableEntry stackTraceEntry = (JfrStackTraceTableEntry) entry; /* The base method will free only the entry itself, not the pointer with stacktrace. */ - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(stackTraceEntry.getRawStackTrace()); + NullableNativeMemory.free(stackTraceEntry.getRawStackTrace()); super.free(entry); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java index 8949a12c50cb..a5b6f6bee8e9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.jfr; -import jdk.graal.compiler.core.common.SuppressFBWarnings; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; @@ -43,6 +41,10 @@ import com.oracle.svm.core.jdk.UninterruptibleUtils.CharReplacer; import com.oracle.svm.core.jfr.traceid.JfrTraceIdEpoch; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.core.common.SuppressFBWarnings; +import jdk.graal.compiler.word.Word; /** * In Native Image, we use {@link java.lang.String} objects that live in the image heap as symbols. @@ -179,6 +181,11 @@ private interface JfrSymbol extends UninterruptibleEntry { private static class JfrSymbolHashtable extends AbstractUninterruptibleHashtable { private static long nextId; + @Platforms(Platform.HOSTED_ONLY.class) + JfrSymbolHashtable() { + super(NmtCategory.JFR); + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected JfrSymbol[] createTable(int size) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java index 0aa3e41ff71c..593c54d2d56f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java @@ -25,13 +25,20 @@ package com.oracle.svm.core.jfr.utils; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.collections.AbstractUninterruptibleHashtable; import com.oracle.svm.core.collections.UninterruptibleEntry; +import com.oracle.svm.core.nmt.NmtCategory; public final class JfrVisitedTable extends AbstractUninterruptibleHashtable { + @Platforms(Platform.HOSTED_ONLY.class) + public JfrVisitedTable() { + super(NmtCategory.JFR); + } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java index 24131c95471d..8966b02ff48c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java @@ -29,8 +29,6 @@ import static org.graalvm.word.WordFactory.unsigned; import static org.graalvm.word.WordFactory.zero; -import jdk.graal.compiler.word.Word; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CIntPointer; import org.graalvm.nativeimage.c.type.WordPointer; @@ -42,6 +40,10 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.jni.headers.JNIJavaVM; import com.oracle.svm.core.jni.headers.JNIJavaVMPointer; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.word.Word; /** * A process-global, lock-free list of JavaVM pointers. Implemented as arrays in native memory which @@ -76,7 +78,7 @@ public static void addJavaVM(JNIJavaVM newEntry) { Pointer p = nextPointer.readWord(0); if (p.isNull()) { // No empty slots, create new array UnsignedWord newCapacity = capacity.notEqual(0) ? capacity.multiply(2) : INITIAL_CAPACITY; - Pointer newArray = UnmanagedMemory.calloc(newCapacity.add(2 /* capacity and next */).multiply(wordSize)); + Pointer newArray = NativeMemory.calloc(newCapacity.add(2 /* capacity and next */).multiply(wordSize), NmtCategory.JNI); newArray.writeWord(0, newCapacity); newArray.writeWord(wordSize, newEntry); p = nextPointer.compareAndSwapWord(0, nullPointer(), newArray, ANY_LOCATION); @@ -84,7 +86,7 @@ public static void addJavaVM(JNIJavaVM newEntry) { return; } // Another thread already created and linked a new array, continue in that array - UnmanagedMemory.free(newArray); + NativeMemory.free(newArray); } capacity = p.readWord(0); p = p.add(wordSize); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java index 60d6d3b04e1e..730742e77080 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.jni.functions; -import jdk.graal.compiler.serviceprovider.IsolateUtil; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Isolate; import org.graalvm.nativeimage.LogHandler; import org.graalvm.nativeimage.StackValue; @@ -36,7 +34,6 @@ import org.graalvm.nativeimage.c.type.CCharPointerPointer; import org.graalvm.nativeimage.c.type.CIntPointer; import org.graalvm.nativeimage.c.type.WordPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -73,6 +70,7 @@ import com.oracle.svm.core.jni.headers.JNIJavaVMPointer; import com.oracle.svm.core.jni.headers.JNIVersion; import com.oracle.svm.core.log.FunctionPointerLogHandler; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.monitor.MonitorInflationCause; import com.oracle.svm.core.monitor.MonitorSupport; import com.oracle.svm.core.snippets.ImplicitExceptions; @@ -80,6 +78,8 @@ import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.util.Utf8; +import jdk.graal.compiler.serviceprovider.IsolateUtil; + /** * Implementation of the JNI invocation API for interacting with a Java VM without having an * existing context and without linking against the Java VM library beforehand. @@ -128,7 +128,7 @@ static int enter(JNIJavaVMPointer vmBuf, JNIEnvironmentPointer penv, JNIJavaVMIn int vmArgc = vmArgs.getNOptions(); if (vmArgc > 0) { UnsignedWord size = SizeOf.unsigned(CCharPointerPointer.class).multiply(vmArgc + 1); - CCharPointerPointer argv = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + CCharPointerPointer argv = UntrackedNullableNativeMemory.malloc(size); if (argv.isNull()) { return JNIErrors.JNI_ENOMEM(); } @@ -169,7 +169,7 @@ static int enter(JNIJavaVMPointer vmBuf, JNIEnvironmentPointer penv, JNIJavaVMIn int code = CEntryPointActions.enterCreateIsolate(params); if (params.isNonNull()) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(params.getArgv()); + UntrackedNullableNativeMemory.free(params.getArgv()); params = WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java index d7ba95251947..ac1417b0e57b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java @@ -26,11 +26,12 @@ import java.nio.ByteBuffer; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; import com.oracle.svm.core.jdk.DirectByteBufferUtil; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; /** * Allocates a buffer with a minimal size that only contains the performance data header (see @@ -42,14 +43,14 @@ public class CHeapPerfMemoryProvider implements PerfMemoryProvider { @Override public ByteBuffer create() { int size = PerfMemoryPrologue.getPrologueSize(); - memory = UnmanagedMemory.calloc(size); + memory = NativeMemory.calloc(size, NmtCategory.JvmStat); return DirectByteBufferUtil.allocate(memory.rawValue(), size); } @Override public void teardown() { if (memory.isNonNull()) { - UnmanagedMemory.free(memory); + NativeMemory.free(memory); memory = WordFactory.nullPointer(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java index 223f2481a1a3..75b4dfb96d96 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java @@ -29,12 +29,10 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -45,6 +43,10 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.jdk.DirectByteBufferUtil; import com.oracle.svm.core.jdk.Target_java_nio_Buffer; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.word.Word; /** * Provides access to the underlying OS-specific memory that stores the performance data. @@ -148,7 +150,7 @@ public synchronized Word allocate(int size) { if (used + size >= capacity) { PerfMemoryPrologue.addOverflow(rawMemory, size); // Always return a valid pointer (external tools won't see this memory though). - Word result = UnmanagedMemory.calloc(size); + Word result = NativeMemory.calloc(size, NmtCategory.JvmStat); addOverflowMemory(result); return result; } @@ -198,7 +200,7 @@ public void teardown() { if (overflowMemory != null) { for (int i = 0; i < overflowMemory.length; i++) { - UnmanagedMemory.free(overflowMemory[i]); + NativeMemory.free(overflowMemory[i]); } overflowMemory = null; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NativeMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NativeMemory.java new file mode 100644 index 000000000000..31929d623cd1 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NativeMemory.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.memory; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.nmt.NmtCategory; + +/** + * Internal API for managing native memory. This class supports native memory tracking (NMT) and is + * therefore preferred over the public API class {@link UnmanagedMemory} and its + * {@link UnmanagedMemorySupport implementations}. + * + * All methods that allocate native memory throw an {@link OutOfMemoryError} if the memory + * allocation fails. If native memory needs to be allocated from uninterruptible code, use + * {@link NullableNativeMemory} instead. + * + * Note that NMT may cause segfaults in certain scenarios: + *
    + *
  • Native memory that was allocated outside of Java (e.g., in a C library) or via some API that + * does not support NMT (e.g., {@link UnmanagedMemory}) may only be freed with + * {@link UntrackedNullableNativeMemory#free}. This is necessary because NMT assumes that each + * allocation has a custom header, which isn't true for such memory.
    • + *
  • NMT accesses the image heap. If native memory needs to be allocated before the image heap is + * mapped or after it was unmapped, {@link UntrackedNullableNativeMemory} must be used.
    • + *
+ */ +public class NativeMemory { + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method never returns a null pointer but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T malloc(UnsignedWord size, NmtCategory category) { + T result = NullableNativeMemory.malloc(size, category); + if (result.isNull()) { + throw new OutOfMemoryError("Memory allocation failed: malloc returned null."); + } + return result; + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method never returns a null pointer but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T malloc(int size, NmtCategory category) { + assert size >= 0; + return malloc(WordFactory.unsigned(size), category); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T calloc(UnsignedWord size, NmtCategory category) { + T result = NullableNativeMemory.calloc(size, category); + if (result.isNull()) { + throw new OutOfMemoryError("Memory allocation failed: calloc returned null."); + } + return result; + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T calloc(int size, NmtCategory category) { + assert size >= 0; + return calloc(WordFactory.unsigned(size), category); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. In that case, the old data is not deallocated and remains unchanged. + */ + public static T realloc(T ptr, UnsignedWord size, NmtCategory category) { + T result = NullableNativeMemory.realloc(ptr, size, category); + if (result.isNull()) { + throw new OutOfMemoryError("Memory allocation failed: realloc returned null."); + } + return result; + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. In that case, the old data is not deallocated and remains unchanged. + */ + public static T realloc(T ptr, int size, NmtCategory category) { + assert size >= 0; + return realloc(ptr, WordFactory.unsigned(size), category); + } + + /** + * Frees native memory that was previously allocated using methods of this class. This method is + * a no-op if the given pointer is {@code null}. + *

+ * Note that this method must NOT be used to free memory that was allocated via other + * classes (e.g., {@link UnmanagedMemorySupport}) or outside of Native Image code (e.g., in a C + * library). + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void free(PointerBase ptr) { + NullableNativeMemory.free(ptr); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java new file mode 100644 index 000000000000..7fa50b7615f6 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.memory; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.nmt.NmtMallocHeader; + +/** + * Internal API for managing native memory. This class supports native memory tracking (NMT) and is + * therefore preferred over the public API class {@link UnmanagedMemory} and its + * {@link UnmanagedMemorySupport implementations}. + * + * All methods that allocate native memory return {@code null} if the memory allocation fails. + */ +public class NullableNativeMemory { + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(UnsignedWord size, NmtCategory category) { + T outerPointer = UntrackedNullableNativeMemory.malloc(getAllocationSize(size)); + return track(outerPointer, size, category); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(int size, NmtCategory category) { + assert size >= 0; + return malloc(WordFactory.unsigned(size), category); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(UnsignedWord size, NmtCategory category) { + T outerPointer = UntrackedNullableNativeMemory.calloc(getAllocationSize(size)); + return track(outerPointer, size, category); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(int size, NmtCategory category) { + assert size >= 0; + return calloc(WordFactory.unsigned(size), category); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @SuppressWarnings("unchecked") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, UnsignedWord size, NmtCategory category) { + if (ptr.isNull()) { + return malloc(size, category); + } else if (!VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + return UntrackedNullableNativeMemory.realloc(ptr, getAllocationSize(size)); + } + + /* Query the NMT information for the old allocation. */ + NmtMallocHeader header = NativeMemoryTracking.getHeader(ptr); + T oldOuterPointer = (T) header; + int oldCategory = header.getCategory(); + UnsignedWord oldSize = header.getAllocationSize(); + + /* Try to realloc. */ + T newOuterPointer = UntrackedNullableNativeMemory.realloc(oldOuterPointer, getAllocationSize(size)); + if (newOuterPointer.isNull()) { + return WordFactory.nullPointer(); + } + + oldOuterPointer = WordFactory.nullPointer(); + + /* Only untrack the old block if the allocation was successful. */ + NativeMemoryTracking.singleton().untrack(oldSize, oldCategory); + return track(newOuterPointer, size, category); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, int size, NmtCategory category) { + assert size >= 0; + return realloc(ptr, WordFactory.unsigned(size), category); + } + + /** + * Frees native memory that was previously allocated using methods of this class. This method is + * a no-op if the given pointer is {@code null}. + *

+ * Note that this method must NOT be used to free memory that was allocated via other + * classes (e.g., {@link UnmanagedMemorySupport}) or outside of Native Image code (e.g., in a C + * library). + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void free(PointerBase ptr) { + if (ptr.isNull()) { + return; + } + + PointerBase outerPtr = untrack(ptr); + UntrackedNullableNativeMemory.free(outerPtr); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord getAllocationSize(UnsignedWord size) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + return size.add(NativeMemoryTracking.sizeOfNmtHeader()); + } + return size; + } + + @SuppressWarnings("unchecked") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static T track(T outerPtr, UnsignedWord size, NmtCategory category) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport() && outerPtr.isNonNull()) { + T innerPtr = (T) NativeMemoryTracking.singleton().initializeHeader(outerPtr, size, category); + NativeMemoryTracking.singleton().track(innerPtr); + return innerPtr; + } + return outerPtr; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static PointerBase untrack(PointerBase innerPtr) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + return NativeMemoryTracking.singleton().untrack(innerPtr); + } + return innerPtr; + } +} diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UnmanagedMemorySupportImpl.java similarity index 59% rename from substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UnmanagedMemorySupportImpl.java index 6c56453a7466..9c41d7ce1b7c 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UnmanagedMemorySupportImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,40 +22,68 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package com.oracle.svm.core.posix; +package com.oracle.svm.core.memory; +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.headers.LibCSupport; +import com.oracle.svm.core.util.UnsignedUtils; -@AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) +import jdk.graal.compiler.api.replacements.Fold; + +/** + * Delegates to the libc-specific memory management functions. Some platforms use a different + * implementation. + */ class UnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return LibC.malloc(size); + /* + * Some libc implementations may return a nullptr when the size is 0. We use a minimum size + * of 1 to ensure that we always get a unique pointer if the allocation succeeds. + */ + UnsignedWord allocationSize = UnsignedUtils.max(size, WordFactory.unsigned(1)); + return libc().malloc(allocationSize); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return LibC.calloc(WordFactory.unsigned(1), size); + return libc().calloc(WordFactory.unsigned(1), size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return LibC.realloc(ptr, size); + return libc().realloc(ptr, size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { - LibC.free(ptr); + libc().free(ptr); + } + + @Fold + static LibCSupport libc() { + return ImageSingletons.lookup(LibCSupport.class); + } +} + +@AutomaticallyRegisteredFeature +class UnmanagedMemorySupportFeature implements InternalFeature { + @Override + public void duringSetup(DuringSetupAccess access) { + if (!ImageSingletons.contains(UnmanagedMemorySupport.class)) { + ImageSingletons.add(UnmanagedMemorySupport.class, new UnmanagedMemorySupportImpl()); + } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UntrackedNullableNativeMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UntrackedNullableNativeMemory.java new file mode 100644 index 000000000000..d05fa8041444 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UntrackedNullableNativeMemory.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.memory; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; + +import jdk.graal.compiler.api.replacements.Fold; + +/** + * Manages native memory. This class explicitly does NOT support native memory tracking + * (NMT). It can therefore be used for the following use cases: + *

    + *
  • Allocate native memory that is later freed outside of Native Image code (e.g., in a C + * library).
    • + *
  • Free native memory that was allocated outside of Native Image code (e.g., in a C + * library).
    • + *
+ */ +public class UntrackedNullableNativeMemory { + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(UnsignedWord size) { + return memory().malloc(size); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(int size) { + assert size >= 0; + return malloc(WordFactory.unsigned(size)); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(UnsignedWord size) { + return memory().calloc(size); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(int size) { + assert size >= 0; + return calloc(WordFactory.unsigned(size)); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, UnsignedWord size) { + return memory().realloc(ptr, size); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, int size) { + assert size >= 0; + return realloc(ptr, WordFactory.unsigned(size)); + } + + /** + * Frees native memory that was previously allocated using methods of this class or that was + * allocated outside of Java (e.g., in a C library). This method is a no-op if the given pointer + * is {@code null}. + *

+ * Note that this method must NOT be used to free memory that was allocated via classes + * that support native memory tracking (e.g., {@link NativeMemory} or + * {@link NullableNativeMemory}). + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void free(PointerBase ptr) { + memory().free(ptr); + } + + @Fold + static UnmanagedMemorySupport memory() { + return ImageSingletons.lookup(UnmanagedMemorySupport.class); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java new file mode 100644 index 000000000000..616cbe1944d0 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.jdk.RuntimeSupport; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.util.UnsignedUtils; + +import jdk.graal.compiler.api.replacements.Fold; + +/** + * This class implements native memory tracking (NMT). There are two components to NMT: tracking + * memory allocations (malloc/realloc/calloc), and tracking virtual memory usage (not supported + * yet). + * + * For tracking memory allocations, we have an internal API (see {@link NativeMemory}) that adds a + * custom {@link NmtMallocHeader header} to each allocation if NMT is enabled. This header stores + * data that is needed to properly untrack the memory when it is freed. + */ +public class NativeMemoryTracking { + private static final UnsignedWord ALIGNMENT = WordFactory.unsigned(16); + private static final int MAGIC = 0xF0F1F2F3; + private final NmtMallocMemorySnapshot mallocMemorySnapshot = new NmtMallocMemorySnapshot(); + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeMemoryTracking() { + } + + @Fold + public static NativeMemoryTracking singleton() { + return ImageSingletons.lookup(NativeMemoryTracking.class); + } + + @Fold + public static UnsignedWord sizeOfNmtHeader() { + /* + * Align the header to 16 bytes to preserve platform-specific malloc alignments up to 16 + * bytes (i.e., the allocation payload is aligned to 16 bytes if the platform-specific + * malloc implementation returns a pointer that is aligned to at least 16 bytes). + */ + return UnsignedUtils.roundUp(SizeOf.unsigned(NmtMallocHeader.class), ALIGNMENT); + } + + /** + * Initializes the NMT header and returns a pointer to the allocation payload (i.e., the inner + * pointer). + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @SuppressWarnings("static-method") + public Pointer initializeHeader(PointerBase outerPtr, UnsignedWord size, NmtCategory category) { + NmtMallocHeader mallocHeader = (NmtMallocHeader) outerPtr; + mallocHeader.setAllocationSize(size); + mallocHeader.setCategory(category.ordinal()); + assert setMagic(mallocHeader); + return getInnerPointer(mallocHeader); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean setMagic(NmtMallocHeader mallocHeader) { + mallocHeader.setMagic(MAGIC); + return true; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void track(PointerBase innerPtr) { + if (innerPtr.isNull()) { + return; + } + + NmtMallocHeader header = getHeader(innerPtr); + UnsignedWord nmtHeaderSize = sizeOfNmtHeader(); + UnsignedWord allocationSize = header.getAllocationSize(); + UnsignedWord totalSize = allocationSize.add(nmtHeaderSize); + + mallocMemorySnapshot.getInfoByCategory(header.getCategory()).track(allocationSize); + mallocMemorySnapshot.getInfoByCategory(NmtCategory.NMT).track(nmtHeaderSize); + mallocMemorySnapshot.getTotalInfo().track(totalSize); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public PointerBase untrack(PointerBase innerPtr) { + if (innerPtr.isNull()) { + return WordFactory.nullPointer(); + } + + NmtMallocHeader header = getHeader(innerPtr); + untrack(header.getAllocationSize(), header.getCategory()); + return header; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void untrack(UnsignedWord size, int category) { + mallocMemorySnapshot.getInfoByCategory(category).untrack(size); + mallocMemorySnapshot.getInfoByCategory(NmtCategory.NMT).untrack(sizeOfNmtHeader()); + mallocMemorySnapshot.getTotalInfo().untrack(size.add(sizeOfNmtHeader())); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static NmtMallocHeader getHeader(PointerBase innerPtr) { + NmtMallocHeader result = (NmtMallocHeader) ((Pointer) innerPtr).subtract(sizeOfNmtHeader()); + assert result.getMagic() == MAGIC : "bad NMT malloc header"; + return result; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static Pointer getInnerPointer(NmtMallocHeader mallocHeader) { + return ((Pointer) mallocHeader).add(sizeOfNmtHeader()); + } + + public long getUsedMemory(NmtCategory category) { + return mallocMemorySnapshot.getInfoByCategory(category).getUsed(); + } + + public static RuntimeSupport.Hook shutdownHook() { + return isFirstIsolate -> NativeMemoryTracking.singleton().printStatistics(); + } + + public void printStatistics() { + if (VMInspectionOptions.PrintNMTStatistics.getValue()) { + System.out.println(); + System.out.println("Native memory tracking"); + System.out.println(" Total used memory: " + mallocMemorySnapshot.getTotalInfo().getUsed() + " bytes"); + System.out.println(" Total alive allocations: " + mallocMemorySnapshot.getTotalInfo().getCount()); + + for (int i = 0; i < NmtCategory.values().length; i++) { + String name = NmtCategory.values()[i].getName(); + NmtMallocMemoryInfo info = mallocMemorySnapshot.getInfoByCategory(i); + + System.out.println(" " + name + " used memory: " + info.getUsed() + " bytes"); + System.out.println(" " + name + " alive allocations: " + info.getCount()); + } + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java new file mode 100644 index 000000000000..ff71cdfabaed --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; + +/** Categories for native memory tracking. */ +public enum NmtCategory { + /** JIT compiler. */ + Compiler("Compiler"), + /** JIT compiled code. */ + Code("Code"), + /** Garbage collector. */ + GC("GC"), + /** Heap dumping infrastructure. */ + HeapDump("Heap Dump"), + /** Java Flight Recorder. */ + JFR("JFR"), + /** Java Native Interface. */ + JNI("JNI"), + /** JVM stat / perf data. */ + JvmStat("jvmstat"), + /** NMT itself. */ + NMT("Native Memory Tracking"), + /** Profile-guided optimizations. */ + PGO("PGO"), + /** Threading. */ + Threading("Threading"), + /** Memory allocated via Unsafe. */ + Unsafe("Unsafe"), + + /** Some other, VM internal reason - avoid if possible, better to add a new category. */ + Internal("Internal"); + + private final String name; + + NmtCategory(String name) { + this.name = name; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public String getName() { + return name; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java new file mode 100644 index 000000000000..9b7786681fe4 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.ImageSingletons; + +import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.jdk.RuntimeSupport; + +@AutomaticallyRegisteredFeature +public class NmtFeature implements InternalFeature { + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return VMInspectionOptions.hasNativeMemoryTrackingSupport(); + } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(NativeMemoryTracking.class, new NativeMemoryTracking()); + } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + RuntimeSupport.getRuntimeSupport().addShutdownHook(NativeMemoryTracking.shutdownHook()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocHeader.java new file mode 100644 index 000000000000..5f2462bb07b7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocHeader.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +/** + * A "malloc header" stores metadata about the native allocation (malloc/calloc/realloc). To do + * this, a small amount of additional space is requested contiguous to the user allocation. This + * metadata is used to update the memory tracking once the block is freed. + */ +@RawStructure +public interface NmtMallocHeader extends PointerBase { + @RawField + UnsignedWord getAllocationSize(); + + @RawField + void setAllocationSize(UnsignedWord value); + + @RawField + int getCategory(); + + @RawField + void setCategory(int value); + + @RawField + int getMagic(); + + @RawField + void setMagic(int value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemoryInfo.java new file mode 100644 index 000000000000..7a54c3b491ca --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemoryInfo.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; + +class NmtMallocMemoryInfo { + private final AtomicLong count = new AtomicLong(0); + private final AtomicLong used = new AtomicLong(0); + + @Platforms(Platform.HOSTED_ONLY.class) + NmtMallocMemoryInfo() { + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void track(UnsignedWord allocationSize) { + count.incrementAndGet(); + used.addAndGet(allocationSize.rawValue()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void untrack(UnsignedWord allocationSize) { + long lastCount = count.decrementAndGet(); + long lastSize = used.addAndGet(-allocationSize.rawValue()); + assert lastSize >= 0 && lastCount >= 0; + } + + long getUsed() { + return used.get(); + } + + long getCount() { + return count.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemorySnapshot.java similarity index 58% rename from substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemorySnapshot.java index 7ee93b64d4e7..d5a8bca94a31 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemorySnapshot.java @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,40 +23,40 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package com.oracle.svm.core.windows; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; -import org.graalvm.word.PointerBase; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.headers.LibC; -@AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) -class WindowsUnmanagedMemorySupportImpl implements UnmanagedMemorySupport { - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T malloc(UnsignedWord size) { - return LibC.malloc(size); +class NmtMallocMemorySnapshot { + private final NmtMallocMemoryInfo[] categories; + private final NmtMallocMemoryInfo total; + + @Platforms(Platform.HOSTED_ONLY.class) + NmtMallocMemorySnapshot() { + total = new NmtMallocMemoryInfo(); + categories = new NmtMallocMemoryInfo[NmtCategory.values().length]; + for (int i = 0; i < categories.length; i++) { + categories[i] = new NmtMallocMemoryInfo(); + } } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T calloc(UnsignedWord size) { - return LibC.calloc(WordFactory.unsigned(1), size); + NmtMallocMemoryInfo getInfoByCategory(NmtCategory category) { + return getInfoByCategory(category.ordinal()); } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T realloc(T ptr, UnsignedWord size) { - return LibC.realloc(ptr, size); + NmtMallocMemoryInfo getInfoByCategory(int category) { + assert category < categories.length; + return categories[category]; } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void free(PointerBase ptr) { - LibC.free(ptr); + NmtMallocMemoryInfo getTotalInfo() { + return total; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java index af9d461fb5c1..753e63da36d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java @@ -26,8 +26,6 @@ import java.io.File; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -41,6 +39,9 @@ import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.snippets.KnownIntrinsics; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; + public abstract class AbstractRawFileOperationSupport implements RawFileOperationSupport { private final boolean useNativeByteOrder; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java index f2638b6de817..c96df4dd61cb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java @@ -26,15 +26,12 @@ import java.nio.ByteOrder; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -47,10 +44,15 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jdk.JavaLangSubstitutions; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.BufferedFileOperationSupport.BufferedFileOperationSupportHolder; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; import com.oracle.svm.core.snippets.KnownIntrinsics; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; + /** * Provides buffered, OS-independent operations on files. Most of the code is implemented in a way * that it can be used from uninterruptible code. @@ -99,7 +101,7 @@ protected BufferedFileOperationSupport(boolean useNativeByteOrder) { * Returns a null pointer otherwise. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public BufferedFile allocate(RawFileDescriptor fd) { + public BufferedFile allocate(RawFileDescriptor fd, NmtCategory nmtCategory) { if (!rawFiles().isValid(fd)) { return WordFactory.nullPointer(); } @@ -110,7 +112,7 @@ public BufferedFile allocate(RawFileDescriptor fd) { /* Use a single allocation for the struct and the corresponding buffer. */ UnsignedWord totalSize = SizeOf.unsigned(BufferedFile.class).add(WordFactory.unsigned(BUFFER_SIZE)); - BufferedFile result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(totalSize); + BufferedFile result = NullableNativeMemory.malloc(totalSize, nmtCategory); if (result.isNull()) { return WordFactory.nullPointer(); } @@ -127,7 +129,7 @@ public BufferedFile allocate(RawFileDescriptor fd) { */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(BufferedFile f) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(f); + NullableNativeMemory.free(f); } /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java index 139af6afa1b6..5ce50b5673fd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java @@ -26,17 +26,18 @@ import java.io.File; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordBase; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.os.AbstractRawFileOperationSupport.RawFileOperationSupportHolder; +import jdk.graal.compiler.api.replacements.Fold; + /** * Provides unbuffered, OS-independent operations on files. Most of the code is implemented in a way * that it can be used from uninterruptible code. @@ -75,7 +76,7 @@ static RawFileOperationSupport nativeByteOrder() { /** * Tries to allocate a platform-dependent C string for the given path. Note that the returned * value needs to be freed manually once it is no longer needed (see - * {@link UnmanagedMemorySupport#free}). + * {@link UntrackedNullableNativeMemory#free}). * * @return If the allocation is successful, a non-null value is returned. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java index a7d2221ff925..b5f9643c1ef0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java @@ -28,7 +28,6 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -37,6 +36,8 @@ import com.oracle.svm.core.jfr.SubstrateJVM; import com.oracle.svm.core.jfr.sampler.JfrExecutionSampler; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; /** * Keeps track of {@link #availableBuffers available} and {@link #fullBuffers full} buffers. If @@ -160,7 +161,7 @@ private SamplerBuffer tryAllocateBuffer0() { UnsignedWord headerSize = SamplerBufferAccess.getHeaderSize(); UnsignedWord dataSize = WordFactory.unsigned(SubstrateJVM.getThreadLocal().getThreadLocalBufferSize()); - SamplerBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize)); + SamplerBuffer result = NullableNativeMemory.malloc(headerSize.add(dataSize), NmtCategory.JFR); if (result.isNonNull()) { bufferCount++; result.setSize(dataSize); @@ -183,7 +184,7 @@ private boolean popAndFree() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void free(SamplerBuffer buffer) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + NullableNativeMemory.free(buffer); bufferCount--; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index ee0355ce92b1..3e33f5c0feed 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -56,7 +56,6 @@ import org.graalvm.nativeimage.ObjectHandles; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CEntryPointLiteral; import org.graalvm.nativeimage.c.function.CFunctionPointer; @@ -95,7 +94,9 @@ import com.oracle.svm.core.jfr.HasJfrSupport; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.monitor.MonitorSupport; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; import com.oracle.svm.core.stack.StackFrameVisitor; @@ -719,14 +720,14 @@ protected T prepareStart(Thread thread, int startDat T startData = WordFactory.nullPointer(); ObjectHandle threadHandle = WordFactory.zero(); try { - startData = UnmanagedMemory.malloc(startDataSize); + startData = NativeMemory.malloc(startDataSize, NmtCategory.Threading); threadHandle = ObjectHandles.getGlobal().create(thread); startData.setIsolate(CurrentIsolate.getIsolate()); startData.setThreadHandle(threadHandle); } catch (Throwable e) { if (startData.isNonNull()) { - UnmanagedMemory.free(startData); + freeStartData(startData); } if (threadHandle.notEqual(WordFactory.zero())) { ObjectHandles.getGlobal().destroy(threadHandle); @@ -784,7 +785,7 @@ private static void decrementNonDaemonThreadsAndNotify() { } protected static void freeStartData(ThreadStartData startData) { - UnmanagedMemory.free(startData); + NativeMemory.free(startData); } void startThread(Thread thread, long stackSize) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index a0f5ae79647c..db22367483b3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -32,7 +32,6 @@ import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.function.CFunction; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.ComparableWord; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -53,6 +52,7 @@ import com.oracle.svm.core.locks.VMLockSupport; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; import com.oracle.svm.core.threadlocal.FastThreadLocal; @@ -254,7 +254,7 @@ public IsolateThread allocateIsolateThread(int isolateThreadSize) { UnsignedWord alignment = WordFactory.unsigned(64); UnsignedWord memorySize = WordFactory.unsigned(isolateThreadSize).add(alignment); - Pointer memory = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(memorySize); + Pointer memory = UntrackedNullableNativeMemory.calloc(memorySize); if (memory.isNull()) { return WordFactory.nullPointer(); } @@ -274,7 +274,7 @@ public void freeCurrentIsolateThread() { @Uninterruptible(reason = "Thread state no longer set up.") protected void freeIsolateThread(IsolateThread thread) { Pointer memory = unalignedIsolateThreadMemoryTL.get(thread); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(memory); + UntrackedNullableNativeMemory.free(memory); } /** diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java index 453fbe54c1cf..664db15b0826 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java @@ -25,7 +25,6 @@ package com.oracle.svm.graal.isolated; import org.graalvm.nativeimage.ObjectHandle; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -38,7 +37,9 @@ import com.oracle.svm.core.code.ReferenceAdjuster; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.handles.ThreadLocalHandles; +import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.meta.DirectSubstrateObjectConstant; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.VMError; import jdk.vm.ci.meta.JavaConstant; @@ -98,9 +99,9 @@ public void setConstantTargetAt(PointerBase address, int length, JavaConstant co @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public NonmovableObjectArray copyOfObjectArray(T[] source) { + public NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory) { @SuppressWarnings("unchecked") - NonmovableObjectArray copy = NonmovableArrays.createObjectArray((Class) source.getClass(), source.length); + NonmovableObjectArray copy = NonmovableArrays.createObjectArray((Class) source.getClass(), source.length, nmtCategory); for (int i = 0; i < source.length; i++) { setObjectInArray(copy, i, source[i]); } @@ -114,14 +115,12 @@ public boolean isFinished() { } public ForeignIsolateReferenceAdjusterData exportData() { - ForeignIsolateReferenceAdjusterData data = UnmanagedMemory.malloc(SizeOf.get(ForeignIsolateReferenceAdjusterData.class)); + ForeignIsolateReferenceAdjusterData data = NativeMemory.malloc(SizeOf.get(ForeignIsolateReferenceAdjusterData.class), NmtCategory.Compiler); data.setCount(count); data.setAddresses(addresses); data.setHandles(handles); - NonmovableArrays.untrackUnmanagedArray(addresses); addresses = WordFactory.nullPointer(); - NonmovableArrays.untrackUnmanagedArray(handles); handles = WordFactory.nullPointer(); count = 0; @@ -129,9 +128,6 @@ public ForeignIsolateReferenceAdjusterData exportData() { } public static void adjustAndDispose(ForeignIsolateReferenceAdjusterData data, ThreadLocalHandles handleSet) { - NonmovableArrays.trackUnmanagedArray(data.getAddresses()); - NonmovableArrays.trackUnmanagedArray(data.getHandles()); - int count = data.getCount(); for (int i = 0; i < count; i++) { @SuppressWarnings("unchecked") @@ -143,7 +139,7 @@ public static void adjustAndDispose(ForeignIsolateRefer NonmovableArrays.releaseUnmanagedArray(data.getAddresses()); NonmovableArrays.releaseUnmanagedArray(data.getHandles()); - UnmanagedMemory.free(data); + NativeMemory.free(data); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -166,8 +162,8 @@ private void growIfFull() { int oldSize = addresses.isNonNull() ? NonmovableArrays.lengthOf(addresses) : 0; if (count == oldSize) { int newSize = (oldSize != 0) ? (2 * oldSize) : 32; - NonmovableArray newAddresses = NonmovableArrays.createWordArray(newSize); - NonmovableArray newHandles = NonmovableArrays.createWordArray(newSize); + NonmovableArray newAddresses = NonmovableArrays.createWordArray(newSize, NmtCategory.Compiler); + NonmovableArray newHandles = NonmovableArrays.createWordArray(newSize, NmtCategory.Compiler); if (addresses.isNonNull()) { NonmovableArrays.arraycopy(addresses, 0, newAddresses, 0, oldSize); NonmovableArrays.releaseUnmanagedArray(addresses); diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java index 9e1bb36ba2f5..2ad0fd3830fd 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java @@ -24,10 +24,7 @@ */ package com.oracle.svm.graal.isolated; -import jdk.graal.compiler.code.CompilationResult; -import jdk.graal.compiler.core.common.CompilationIdentifier; import org.graalvm.nativeimage.IsolateThread; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -40,11 +37,16 @@ import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.deopt.SubstrateInstalledCode; import com.oracle.svm.core.graal.meta.SharedRuntimeMethod; +import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.meta.SharedMethod; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.graal.meta.RuntimeCodeInstaller; import com.oracle.svm.graal.meta.SubstrateInstalledCodeImpl; import com.oracle.svm.graal.meta.SubstrateMethod; +import jdk.graal.compiler.code.CompilationResult; +import jdk.graal.compiler.core.common.CompilationIdentifier; + public final class IsolatedRuntimeCodeInstaller extends RuntimeCodeInstaller { /** @@ -114,23 +116,23 @@ private CodeInstallInfo doPrepareInstall() { id = proxy.getHandle(); } - CodeInstallInfo installInfo = UnmanagedMemory.malloc(SizeOf.get(CodeInstallInfo.class)); + CodeInstallInfo installInfo = NativeMemory.malloc(SizeOf.get(CodeInstallInfo.class), NmtCategory.Compiler); installInfo.setCodeInfo(codeInfo); installInfo.setAdjusterData(adjuster.exportData()); installInfo.setCompilationId(id); - IsolatedRuntimeMethodInfoAccess.untrackInCurrentIsolate(installInfo.getCodeInfo()); + IsolatedRuntimeMethodInfoAccess.untrackInCurrentIsolate(installInfo); return installInfo; } private static void installPrepared(SharedMethod method, CodeInstallInfo installInfo, SubstrateInstalledCode installedCode) { - IsolatedRuntimeMethodInfoAccess.startTrackingInCurrentIsolate(installInfo.getCodeInfo()); + IsolatedRuntimeMethodInfoAccess.startTrackingInCurrentIsolate(installInfo); IsolatedReferenceAdjuster.adjustAndDispose(installInfo.getAdjusterData(), IsolatedCompileClient.get().getHandleSet()); installInfo.setAdjusterData(WordFactory.nullPointer()); doInstallPrepared(method, installInfo.getCodeInfo(), installedCode); - UnmanagedMemory.free(installInfo); + NativeMemory.free(installInfo); } private final IsolateThread targetIsolate; diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java index 54330d27bcbf..a2dbf8c5cf63 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java @@ -25,25 +25,57 @@ package com.oracle.svm.graal.isolated; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.VMInspectionOptions; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.c.NonmovableArrays; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.code.InstalledCodeObserverSupport; import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.code.RuntimeCodeInfoMemory; +import com.oracle.svm.core.nmt.NativeMemoryTracking; final class IsolatedRuntimeMethodInfoAccess { - public static void startTrackingInCurrentIsolate(CodeInfo info) { - RuntimeCodeInfoAccess.forEachArray(info, TRACK_ACTION); + public static void startTrackingInCurrentIsolate(CodeInstallInfo installInfo) { + CodeInfo info = installInfo.getCodeInfo(); InstalledCodeObserverSupport.attachToCurrentIsolate(RuntimeCodeInfoAccess.getCodeObserverHandles(info)); RuntimeCodeInfoMemory.singleton().add(info); + + /* NonmovableArray tracking and native memory tracking. */ + RuntimeCodeInfoAccess.forEachArray(info, TRACK_ACTION); + + ForeignIsolateReferenceAdjusterData adjusterData = installInfo.getAdjusterData(); + NonmovableArrays.trackUnmanagedArray(adjusterData.getAddresses()); + NonmovableArrays.trackUnmanagedArray(adjusterData.getHandles()); + + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().track(installInfo); + NativeMemoryTracking.singleton().track(installInfo.getAdjusterData()); + NativeMemoryTracking.singleton().track(installInfo.getCodeInfo()); + NativeMemoryTracking.singleton().track(adjusterData.getAddresses()); + NativeMemoryTracking.singleton().track(adjusterData.getHandles()); + } } - public static void untrackInCurrentIsolate(CodeInfo info) { + public static void untrackInCurrentIsolate(CodeInstallInfo installInfo) { + CodeInfo info = installInfo.getCodeInfo(); RuntimeCodeInfoMemory.singleton().remove(info); InstalledCodeObserverSupport.detachFromCurrentIsolate(RuntimeCodeInfoAccess.getCodeObserverHandles(info)); + + /* NonmovableArray tracking and native memory tracking. */ RuntimeCodeInfoAccess.forEachArray(info, UNTRACK_ACTION); + + ForeignIsolateReferenceAdjusterData adjusterData = installInfo.getAdjusterData(); + NonmovableArrays.untrackUnmanagedArray(adjusterData.getAddresses()); + NonmovableArrays.untrackUnmanagedArray(adjusterData.getHandles()); + + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().untrack(installInfo); + NativeMemoryTracking.singleton().untrack(installInfo.getAdjusterData()); + NativeMemoryTracking.singleton().untrack(installInfo.getCodeInfo()); + NativeMemoryTracking.singleton().untrack(adjusterData.getAddresses()); + NativeMemoryTracking.singleton().untrack(adjusterData.getHandles()); + } } private static final RuntimeCodeInfoAccess.NonmovableArrayAction TRACK_ACTION = new RuntimeCodeInfoAccess.NonmovableArrayAction() { @@ -51,6 +83,9 @@ public static void untrackInCurrentIsolate(CodeInfo info) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void apply(NonmovableArray array) { NonmovableArrays.trackUnmanagedArray(array); + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().track(array); + } } }; private static final RuntimeCodeInfoAccess.NonmovableArrayAction UNTRACK_ACTION = new RuntimeCodeInfoAccess.NonmovableArrayAction() { @@ -58,6 +93,9 @@ public void apply(NonmovableArray array) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void apply(NonmovableArray array) { NonmovableArrays.untrackUnmanagedArray(array); + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().untrack(array); + } } }; diff --git a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties index 095bac6acbb2..e1470477f070 100644 --- a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties +++ b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties @@ -8,5 +8,5 @@ Args= \ --features=com.oracle.svm.test.jfr.JfrTestFeature \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-exports=org.graalvm.nativeimage.base/com.oracle.svm.util=ALL-UNNAMED \ - --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient \ + --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient,nmt \ -J--enable-preview \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java new file mode 100644 index 000000000000..7c0a5ce10e37 --- /dev/null +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.test.nmt; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; +import org.junit.Test; + +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtCategory; + +public class NativeMemoryTrackingTests { + private static final int ALLOCATION_SIZE = 1024 * 16; + private static final int REALLOC_SIZE = ALLOCATION_SIZE / 2; + + @Test + public void testMalloc() { + assertEquals(0, getUsedMemory()); + + Pointer ptr = NativeMemory.malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtCategory.Code); + assertEquals(ALLOCATION_SIZE, getUsedMemory()); + assertTrue(getUsedMemory() > 0); + + NativeMemory.free(ptr); + + assertEquals(0, getUsedMemory()); + } + + @Test + public void testCalloc() { + assertEquals(0, getUsedMemory()); + Pointer ptr = NativeMemory.calloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtCategory.Code); + + assertEquals(ALLOCATION_SIZE, getUsedMemory()); + assertTrue(getUsedMemory() > 0); + + NativeMemory.free(ptr); + + assertEquals(0, getUsedMemory()); + } + + @Test + public void testRealloc() { + assertEquals(0, getUsedMemory()); + Pointer ptr = NativeMemory.malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtCategory.Code); + + assertEquals(getUsedMemory(), ALLOCATION_SIZE); + assertTrue(getUsedMemory() > 0); + + Pointer reallocPtr = NativeMemory.realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtCategory.Code); + + assertEquals(REALLOC_SIZE, getUsedMemory()); + + NativeMemory.free(reallocPtr); + assertEquals(0, getUsedMemory()); + } + + private static long getUsedMemory() { + return NativeMemoryTracking.singleton().getUsedMemory(NmtCategory.Code); + } +}