diff --git a/common.json b/common.json index 8d457772a47e..bf319e62b6ba 100644 --- a/common.json +++ b/common.json @@ -8,7 +8,7 @@ "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { - "galahad-jdk": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24+24-2891", "platformspecific": true, "extrabundles": ["static-libs"]}, + "galahad-jdk": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24+24-2915", "platformspecific": true, "extrabundles": ["static-libs"]}, "oraclejdk17": {"name": "jpg-jdk", "version": "17.0.7", "build_id": "jdk-17.0.7+8", "platformspecific": true, "extrabundles": ["static-libs"]}, "labsjdk-ce-17": {"name": "labsjdk", "version": "ce-17.0.7+4-jvmci-23.1-b02", "platformspecific": true }, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java index 835b12d972e7..2085be3ee030 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java @@ -302,6 +302,8 @@ public final int arrayOopDescLengthOffset() { public final int threadCarrierThreadObjectOffset = getFieldOffset("JavaThread::_threadObj", Integer.class, "OopHandle"); public final int threadScopedValueCacheOffset = getFieldOffset("JavaThread::_scopedValueCache", Integer.class, "OopHandle"); + public final int javaThreadLockIDOffset = getFieldOffset("JavaThread::_lock_id", Integer.class, "int64_t", -1, JDK > 21); + public final int threadIsInVTMSTransitionOffset = getFieldOffset("JavaThread::_is_in_VTMS_transition", Integer.class, "bool"); public final int threadIsInTmpVTMSTransitionOffset = getFieldOffset("JavaThread::_is_in_tmp_VTMS_transition", Integer.class, "bool", -1, JDK == 21); public final int threadIsDisableSuspendOffset = getFieldOffset("JavaThread::_is_disable_suspend", Integer.class, "bool", -1, JDK >= 22); @@ -387,15 +389,18 @@ public int threadLastJavaFpOffset() { // The following three constants are declared as 64 bits uintptr_t, but known to be 32 bits public final int unlockedValue = getConstant("markWord::unlocked_value", Integer.class); public final int monitorValue = getConstant("markWord::monitor_value", Integer.class); + public final int ageMaskInPlace = getConstant("markWord::age_mask_in_place", Integer.class); + public final int unusedMark = getConstant("markWord::marked_value", Integer.class); // Identity hash code value when uninitialized. public final int uninitializedIdentityHashCodeValue = getConstant("markWord::no_hash", Integer.class); // This field has no type in vmStructs.cpp - public final int objectMonitorOwner = getFieldOffset("ObjectMonitor::_owner", Integer.class, null); + public final int objectMonitorOwner = getFieldOffset("ObjectMonitor::_owner", Integer.class, JDK > 21 ? "int64_t" : null); public final int objectMonitorRecursions = getFieldOffset("ObjectMonitor::_recursions", Integer.class, "intptr_t"); public final int objectMonitorCxq = getFieldOffset("ObjectMonitor::_cxq", Integer.class, "ObjectWaiter*"); public final int objectMonitorEntryList = getFieldOffset("ObjectMonitor::_EntryList", Integer.class, "ObjectWaiter*"); - public final int objectMonitorSucc = getFieldOffset("ObjectMonitor::_succ", Integer.class, "JavaThread*"); + public final int objectMonitorSucc = getFieldOffset("ObjectMonitor::_succ", Integer.class, JDK > 21 ? "int64_t" : "JavaThread*"); + public final int objectMonitorStackLocker = getFieldOffset("ObjectMonitor::_stack_locker", Integer.class, "BasicLock*"); public final int contEntry = getFieldOffset("JavaThread::_cont_entry", Integer.class, "ContinuationEntry*", -1, JDK >= 24); public final int pinCount = getFieldOffset("ContinuationEntry::_pin_count", Integer.class, "uint32_t", -1, JDK >= 24); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java index aa7054fa8975..206f00aff5aa 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java @@ -40,8 +40,12 @@ import static jdk.graal.compiler.hotspot.HotSpotBackend.SHAREDRUNTIME_NOTIFY_JVMTI_VTHREAD_UNMOUNT; import static jdk.graal.compiler.hotspot.HotSpotBackend.UPDATE_BYTES_CRC32; import static jdk.graal.compiler.hotspot.HotSpotBackend.UPDATE_BYTES_CRC32C; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_CARRIER_THREAD_OOP_HANDLE_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_CONTINUATION_ENTRY_PIN_COUNT; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_JAVA_THREAD_CONT_ENTRY; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_ID_LOCATION; import static jdk.graal.compiler.java.BytecodeParserOptions.InlineDuringParsing; import static jdk.graal.compiler.nodes.ConstantNode.forBoolean; import static jdk.graal.compiler.nodes.ProfileData.BranchProbabilityData.injected; @@ -69,7 +73,9 @@ import jdk.graal.compiler.core.common.memory.BarrierType; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; import jdk.graal.compiler.core.common.spi.ForeignCallDescriptor; +import jdk.graal.compiler.core.common.type.ObjectStamp; import jdk.graal.compiler.core.common.type.StampFactory; +import jdk.graal.compiler.core.common.type.TypeReference; import jdk.graal.compiler.debug.GraalError; import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; import jdk.graal.compiler.hotspot.HotSpotBackend; @@ -78,6 +84,7 @@ import jdk.graal.compiler.hotspot.nodes.HotSpotLoadReservedReferenceNode; import jdk.graal.compiler.hotspot.nodes.HotSpotStoreReservedReferenceNode; import jdk.graal.compiler.hotspot.nodes.KlassFullyInitializedCheckNode; +import jdk.graal.compiler.hotspot.nodes.VirtualThreadUpdateJFRNode; import jdk.graal.compiler.hotspot.replacements.CallSiteTargetNode; import jdk.graal.compiler.hotspot.replacements.DigestBaseSnippets; import jdk.graal.compiler.hotspot.replacements.FastNotifyNode; @@ -142,6 +149,8 @@ import jdk.graal.compiler.nodes.java.DynamicNewInstanceWithExceptionNode; import jdk.graal.compiler.nodes.java.NewArrayNode; import jdk.graal.compiler.nodes.java.ValidateNewInstanceClassNode; +import jdk.graal.compiler.nodes.memory.ReadNode; +import jdk.graal.compiler.nodes.memory.WriteNode; import jdk.graal.compiler.nodes.memory.address.AddressNode; import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; import jdk.graal.compiler.nodes.spi.Replacements; @@ -663,14 +672,28 @@ private static boolean isAnnotatedByChangesCurrentThread(ResolvedJavaMethod meth return false; } + private static AddressNode getScopedValueCacheAddress(GraphBuilderContext b, HotSpotInvocationPluginHelper helper) { + CurrentJavaThreadNode javaThread = b.add(new CurrentJavaThreadNode(helper.getWordKind())); + ValueNode scopedValueCacheHandle = helper.readJavaThreadScopedValueCache(javaThread); + return b.add(OffsetAddressNode.create(scopedValueCacheHandle)); + } + private static void registerThreadPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, Replacements replacements) { + BarrierSet barrierSet = replacements.getProviders().getPlatformConfigurationProvider().getBarrierSet(); Registration r = new Registration(plugins, Thread.class, replacements); r.register(new InvocationPlugin("currentThread") { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { - ValueNode value = helper.readCurrentThreadObject(true); - b.push(JavaKind.Object, value); + CurrentJavaThreadNode thread = b.add(new CurrentJavaThreadNode(helper.getWordKind())); + ValueNode vthreadHandle = helper.readJavaThreadVthread(thread); + // Read the Object from the OopHandle + AddressNode handleAddress = b.add(OffsetAddressNode.create(vthreadHandle)); + // JavaThread::_vthread is never compressed + ObjectStamp threadStamp = StampFactory.objectNonNull(TypeReference.create(b.getAssumptions(), b.getMetaAccess().lookupJavaType(Thread.class))); + ValueNode read = new ReadNode(handleAddress, HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION, threadStamp, + barrierSet.readBarrierType(HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION, handleAddress, threadStamp), MemoryOrderMode.PLAIN); + b.addPush(JavaKind.Object, read); } return true; } @@ -680,8 +703,15 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { - ValueNode value = helper.readCurrentThreadObject(false); - b.push(JavaKind.Object, value); + CurrentJavaThreadNode thread = b.add(new CurrentJavaThreadNode(helper.getWordKind())); + ValueNode cthreadHandle = helper.readJavaThreadThreadObj(thread); + // Read the Object from the OopHandle + AddressNode handleAddress = b.add(OffsetAddressNode.create(cthreadHandle)); + // JavaThread::_threadObj is never compressed + ObjectStamp threadStamp = StampFactory.objectNonNull(TypeReference.create(b.getAssumptions(), b.getMetaAccess().lookupJavaType(Thread.class))); + ValueNode read = new ReadNode(handleAddress, HOTSPOT_CARRIER_THREAD_OOP_HANDLE_LOCATION, threadStamp, + barrierSet.readBarrierType(HOTSPOT_CARRIER_THREAD_OOP_HANDLE_LOCATION, handleAddress, threadStamp), MemoryOrderMode.PLAIN); + b.addPush(JavaKind.Object, read); } return true; } @@ -693,7 +723,22 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec GraalError.guarantee(ImageInfo.inImageRuntimeCode() || isAnnotatedByChangesCurrentThread(b.getMethod()), "method changes current Thread but is not annotated ChangesCurrentThread"); try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { receiver.get(true); - helper.setCurrentThread(thread); + CurrentJavaThreadNode javaThread = b.add(new CurrentJavaThreadNode(helper.getWordKind())); + ValueNode threadObjectHandle = helper.readJavaThreadVthread(javaThread); + AddressNode handleAddress = b.add(OffsetAddressNode.create(threadObjectHandle)); + b.add(new WriteNode(handleAddress, HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION, thread, + barrierSet.writeBarrierType(HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION), MemoryOrderMode.PLAIN)); + + if (JavaVersionUtil.JAVA_SPEC > 21) { + GraalError.guarantee(config.javaThreadLockIDOffset != -1, "JavaThread::_lock_id should have been exported"); + // Change the lock_id of the JavaThread + ValueNode tid = helper.loadField(thread, helper.getField(b.getMetaAccess().lookupJavaType(Thread.class), "tid")); + OffsetAddressNode address = b.add(new OffsetAddressNode(javaThread, helper.asWord(config.javaThreadLockIDOffset))); + b.add(new JavaWriteNode(JavaKind.Long, address, JAVA_THREAD_LOCK_ID_LOCATION, tid, BarrierType.NONE, false)); + } + if (HotSpotReplacementsUtil.supportsVirtualThreadUpdateJFR(config)) { + b.add(new VirtualThreadUpdateJFRNode(thread)); + } } return true; } @@ -703,7 +748,11 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { - b.push(JavaKind.Object, helper.readThreadScopedValueCache()); + AddressNode handleAddress = getScopedValueCacheAddress(b, helper); + ObjectStamp stamp = StampFactory.object(TypeReference.create(b.getAssumptions(), b.getMetaAccess().lookupJavaType(Object[].class))); + b.push(JavaKind.Object, b.add(new ReadNode(handleAddress, HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION, stamp, + barrierSet.readBarrierType(HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION, handleAddress, stamp), + MemoryOrderMode.PLAIN))); } return true; } @@ -713,7 +762,10 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode cache) { try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { - helper.setThreadScopedValueCache(cache); + AddressNode handleAddress = getScopedValueCacheAddress(b, helper); + b.add(new WriteNode(handleAddress, HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION, cache, + barrierSet.writeBarrierType(HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION), + MemoryOrderMode.PLAIN)); } return true; } @@ -911,6 +963,21 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec } }); } + + if (JavaVersionUtil.JAVA_SPEC > 21) { + r.registerConditional(config.javaThreadLockIDOffset != -1, new InvocationPlugin("setLockId", Receiver.class, long.class) { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode tid) { + try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { + receiver.get(true); + CurrentJavaThreadNode javaThread = b.add(new CurrentJavaThreadNode(helper.getWordKind())); + OffsetAddressNode address = b.add(new OffsetAddressNode(javaThread, helper.asWord(config.javaThreadLockIDOffset))); + b.add(new JavaWriteNode(JavaKind.Long, address, JAVA_THREAD_LOCK_ID_LOCATION, tid, BarrierType.NONE, false)); + } + return true; + } + }); + } } private static ResolvedJavaType resolveTypeAESCrypt(ResolvedJavaType context) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotInvocationPluginHelper.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotInvocationPluginHelper.java index 557f380fc071..d3c964bc6e88 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotInvocationPluginHelper.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotInvocationPluginHelper.java @@ -26,9 +26,6 @@ import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.ARRAY_KLASS_COMPONENT_MIRROR; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION; -import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_CARRIER_THREAD_OOP_HANDLE_LOCATION; -import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION; -import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_OOP_HANDLE_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_CARRIER_THREAD_OBJECT_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_CURRENT_THREAD_OBJECT_LOCATION; @@ -41,17 +38,15 @@ import java.util.function.Function; -import jdk.graal.compiler.core.common.type.AbstractPointerStamp; import org.graalvm.word.LocationIdentity; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; -import jdk.graal.compiler.core.common.type.ObjectStamp; +import jdk.graal.compiler.core.common.type.AbstractPointerStamp; import jdk.graal.compiler.core.common.type.Stamp; import jdk.graal.compiler.core.common.type.StampFactory; import jdk.graal.compiler.core.common.type.TypeReference; import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; import jdk.graal.compiler.hotspot.nodes.CurrentJavaThreadNode; -import jdk.graal.compiler.hotspot.nodes.VirtualThreadUpdateJFRNode; import jdk.graal.compiler.hotspot.nodes.type.KlassPointerStamp; import jdk.graal.compiler.nodes.NodeView; import jdk.graal.compiler.nodes.PiNode; @@ -61,9 +56,7 @@ import jdk.graal.compiler.nodes.gc.BarrierSet; import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; import jdk.graal.compiler.nodes.memory.ReadNode; -import jdk.graal.compiler.nodes.memory.WriteNode; import jdk.graal.compiler.nodes.memory.address.AddressNode; -import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; import jdk.graal.compiler.nodes.type.StampTool; import jdk.graal.compiler.replacements.InvocationPluginHelper; import jdk.vm.ci.meta.JavaKind; @@ -225,60 +218,24 @@ public ValueNode loadArrayKlass(ValueNode componentType) { } /** - * Reads thread object from current thread. {@code readVirtualThread} indicates whether the - * caller wants the current virtual thread or its carrier -- the current platform thread. + * Read {@code JavaThread::_threadObj}. */ - public ValueNode readCurrentThreadObject(boolean readVirtualThread) { - CurrentJavaThreadNode thread = b.add(new CurrentJavaThreadNode(getWordKind())); - // JavaThread::_vthread and JavaThread::_threadObj are never compressed - ObjectStamp threadStamp = StampFactory.objectNonNull(TypeReference.create(b.getAssumptions(), b.getMetaAccess().lookupJavaType(Thread.class))); - Stamp fieldStamp = StampFactory.forKind(getWordKind()); - ValueNode value = readLocation(thread, readVirtualThread ? HotSpotVMConfigField.JAVA_THREAD_THREAD_OBJECT : HotSpotVMConfigField.JAVA_THREAD_CARRIER_THREAD_OBJECT, fieldStamp); - - // Read the Object from the OopHandle - AddressNode handleAddress = b.add(OffsetAddressNode.create(value)); - LocationIdentity location = readVirtualThread ? HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION : HOTSPOT_CARRIER_THREAD_OOP_HANDLE_LOCATION; - value = b.add(new ReadNode(handleAddress, location, threadStamp, barrierSet.readBarrierType(location, handleAddress, threadStamp), MemoryOrderMode.PLAIN)); - return value; - } - - /** - * Sets {@code thread} into {@code JavaThread::_vthread}. - */ - public void setCurrentThread(ValueNode thread) { - CurrentJavaThreadNode javaThread = b.add(new CurrentJavaThreadNode(getWordKind())); - ValueNode threadObjectHandle = readLocation(javaThread, HotSpotVMConfigField.JAVA_THREAD_THREAD_OBJECT, StampFactory.forKind(getWordKind())); - AddressNode handleAddress = b.add(OffsetAddressNode.create(threadObjectHandle)); - b.add(new WriteNode(handleAddress, HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION, thread, barrierSet.writeBarrierType(HOTSPOT_CURRENT_THREAD_OOP_HANDLE_LOCATION), MemoryOrderMode.PLAIN)); - if (HotSpotReplacementsUtil.supportsVirtualThreadUpdateJFR(config)) { - b.add(new VirtualThreadUpdateJFRNode(thread)); - } - } - - private AddressNode scopedValueCacheHelper() { - CurrentJavaThreadNode javaThread = b.add(new CurrentJavaThreadNode(getWordKind())); - ValueNode scopedValueCacheHandle = readLocation(javaThread, HotSpotVMConfigField.JAVA_THREAD_SCOPED_VALUE_CACHE_OFFSET, StampFactory.forKind(getWordKind())); - return b.add(OffsetAddressNode.create(scopedValueCacheHandle)); + public ValueNode readJavaThreadThreadObj(CurrentJavaThreadNode javaThread) { + return readLocation(javaThread, HotSpotVMConfigField.JAVA_THREAD_CARRIER_THREAD_OBJECT, StampFactory.forKind(getWordKind())); } /** - * Reads {@code JavaThread::_scopedValueCache}. + * Read {@code JavaThread::_vthread}. */ - public ValueNode readThreadScopedValueCache() { - AddressNode handleAddress = scopedValueCacheHelper(); - ObjectStamp stamp = StampFactory.object(TypeReference.create(b.getAssumptions(), b.getMetaAccess().lookupJavaType(Object[].class))); - return b.add(new ReadNode(handleAddress, HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION, stamp, - barrierSet.readBarrierType(HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION, handleAddress, stamp), MemoryOrderMode.PLAIN)); + public ValueNode readJavaThreadVthread(CurrentJavaThreadNode javaThread) { + return readLocation(javaThread, HotSpotVMConfigField.JAVA_THREAD_THREAD_OBJECT, StampFactory.forKind(getWordKind())); } /** - * Sets {@code cache} into {@code JavaThread::_scopedValueCache}. + * Read {@code JavaThread::_scopedValueCache}. */ - public void setThreadScopedValueCache(ValueNode cache) { - AddressNode handleAddress = scopedValueCacheHelper(); - b.add(new WriteNode(handleAddress, HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION, cache, - barrierSet.writeBarrierType(HOTSPOT_JAVA_THREAD_SCOPED_VALUE_CACHE_HANDLE_LOCATION), - MemoryOrderMode.PLAIN)); + public ValueNode readJavaThreadScopedValueCache(CurrentJavaThreadNode javaThread) { + return readLocation(javaThread, HotSpotVMConfigField.JAVA_THREAD_SCOPED_VALUE_CACHE_OFFSET, StampFactory.forKind(getWordKind())); } /** diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index 451289e033c1..e7420699868f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -327,6 +327,8 @@ public static Object getPendingException(Word thread) { public static final LocationIdentity JAVA_THREAD_CARRIER_THREAD_OBJECT_LOCATION = NamedLocationIdentity.mutable("JavaThread::_threadObj"); + public static final LocationIdentity JAVA_THREAD_LOCK_ID_LOCATION = NamedLocationIdentity.mutable("JavaThread::_lock_id"); + public static final LocationIdentity JAVA_THREAD_OSTHREAD_LOCATION = NamedLocationIdentity.mutable("JavaThread::_osthread"); public static final LocationIdentity JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION = NamedLocationIdentity.mutable("JavaThread::_held_monitor_count"); @@ -558,6 +560,11 @@ public static int monitorValue(@InjectedParameter GraalHotSpotVMConfig config) { return config.monitorValue; } + @Fold + public static int unusedMark(@InjectedParameter GraalHotSpotVMConfig config) { + return config.unusedMark; + } + @Fold public static int objectMonitorOwnerOffset(@InjectedParameter GraalHotSpotVMConfig config) { return config.objectMonitorOwner; @@ -583,6 +590,11 @@ public static int objectMonitorSuccOffset(@InjectedParameter GraalHotSpotVMConfi return config.objectMonitorSucc; } + @Fold + public static int objectMonitorStackLockerOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.objectMonitorStackLocker; + } + /** * Mask for a biasable, locked or unlocked mark word. It is the least significant 3 bits prior * to Java 18 (1 bit for biased locking and 2 bits for stack locking or heavy locking), and 2 @@ -736,6 +748,8 @@ public static int secondarySupersOffset(@InjectedParameter GraalHotSpotVMConfig public static final LocationIdentity OBJECT_MONITOR_SUCC_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_succ"); + public static final LocationIdentity OBJECT_MONITOR_STACK_LOCKER_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_stack_locker"); + @Fold public static int lockMetadataOffset(@InjectedParameter GraalHotSpotVMConfig config) { return config.basicLockMetadataOffset; @@ -746,6 +760,11 @@ static int heldMonitorCountOffset(@InjectedParameter GraalHotSpotVMConfig config return config.threadHeldMonitorCountOffset; } + @Fold + static int javaThreadLockIDOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.javaThreadLockIDOffset; + } + @Fold static int javaThreadLockStackTopOffset(@InjectedParameter GraalHotSpotVMConfig config) { return config.threadLockStackOffset + config.lockStackTopOffset; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java index aec8dff3db3e..2fce9ee82686 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java @@ -34,6 +34,7 @@ import static jdk.graal.compiler.hotspot.nodes.VMErrorNode.vmError; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.BASICLOCK_METADATA_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_ID_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_STACK_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_STACK_TOP_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_OM_CACHE_LOCATION; @@ -45,9 +46,12 @@ import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_ENTRY_LIST_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_STACK_LOCKER_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_SUCC_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.diagnoseSyncOnValueBasedClasses; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.heldMonitorCountOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.isCAssertEnabled; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.javaThreadLockIDOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.javaThreadLockStackEndOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.javaThreadLockStackTopOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.javaThreadOomCacheOffset; @@ -71,6 +75,7 @@ import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.shouldUseKlassMiscFlags; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.unlockedValue; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.unusedMark; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.useLightweightLocking; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.useObjectMonitorTable; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.useStackLocking; @@ -89,6 +94,7 @@ import static jdk.graal.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; import static jdk.graal.compiler.replacements.nodes.CStringConstant.cstring; import static org.graalvm.word.LocationIdentity.any; +import static org.graalvm.word.WordFactory.nullPointer; import static org.graalvm.word.WordFactory.unsigned; import static org.graalvm.word.WordFactory.zero; @@ -234,7 +240,9 @@ public static void monitorenter(Object object, KlassPointer hub, @ConstantParame } if (tryFastPathLocking(object, stackPointerRegister, trace, counters, mark, lock, thread)) { - incrementHeldMonitorCount(thread); + if (JavaVersionUtil.JAVA_SPEC == 21 || useStackLocking(INJECTED_VMCONFIG)) { + incrementHeldMonitorCount(thread); + } } else { // slow-path runtime-call monitorenterStubC(MONITORENTER, object, lock); @@ -294,22 +302,27 @@ private static boolean tryEnterInflated(Object object, Word lock, Word mark, Wor cacheIter = cacheIter.add(stepSize); } } + + // Cache the monitor for unlock before trashing box. On failure to acquire + // the lock, the slow path will reset the entry accordingly (see CacheSetter). + writeMonitorCache(lock, monitor); } else { // mark is a pointer to the ObjectMonitor + monitorMask - monitor = mark.subtract(HotSpotReplacementsUtil.monitorValue(INJECTED_VMCONFIG)); + monitor = mark.subtract(monitorValue(INJECTED_VMCONFIG)); } - int ownerOffset = HotSpotReplacementsUtil.objectMonitorOwnerOffset(INJECTED_VMCONFIG); - Word owner = monitor.readWord(ownerOffset, HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION); + int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG); + Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION); + Word newOwner = JavaVersionUtil.JAVA_SPEC == 21 ? thread : thread.readWord(javaThreadLockIDOffset(INJECTED_VMCONFIG), JAVA_THREAD_LOCK_ID_LOCATION); + // The following owner null check is essential. In the case where the null check fails, it // avoids the subsequent bound-to-fail CAS operation, which would have caused the // invalidation of the L1 cache of the core that runs the lock owner thread, and thus causes // the lock to be held slightly longer. - if (probability(FREQUENT_PROBABILITY, owner.equal(0))) { - // it appears unlocked (owner == 0) - if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, thread, HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION))) { + if (probability(FREQUENT_PROBABILITY, owner.equal(nullPointer()))) { + // it appears unlocked (owner == null) + if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, newOwner, OBJECT_MONITOR_OWNER_LOCATION))) { // success - writeMonitorCache(lock, monitor); traceObject(trace, "+lock{heavyweight:cas}", object, true); counters.lockHeavyCas.inc(); return true; @@ -317,11 +330,10 @@ private static boolean tryEnterInflated(Object object, Word lock, Word mark, Wor traceObject(trace, "+lock{heavyweight:failed-cas}", object, true); counters.lockHeavyFailedCas.inc(); } - } else if (probability(NOT_LIKELY_PROBABILITY, owner.equal(thread))) { - int recursionsOffset = HotSpotReplacementsUtil.objectMonitorRecursionsOffset(INJECTED_VMCONFIG); - Word recursions = monitor.readWord(recursionsOffset, HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION); - monitor.writeWord(recursionsOffset, recursions.add(1), HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION); - writeMonitorCache(lock, monitor); + } else if (probability(NOT_LIKELY_PROBABILITY, owner.equal(newOwner))) { + int recursionsOffset = objectMonitorRecursionsOffset(INJECTED_VMCONFIG); + Word recursions = monitor.readWord(recursionsOffset, OBJECT_MONITOR_RECURSION_LOCATION); + monitor.writeWord(recursionsOffset, recursions.add(1), OBJECT_MONITOR_RECURSION_LOCATION); traceObject(trace, "+lock{heavyweight:recursive}", object, true); counters.lockHeavyRecursive.inc(); return true; @@ -336,7 +348,7 @@ private static boolean tryStackLocking(Object object, Word lock, Word mark, Word if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorValue(INJECTED_VMCONFIG)).notEqual(0))) { // Inflated case // Set the lock slot's displaced mark to unused. Any non-0 value suffices. - lock.writeWord(lockMetadataOffset(INJECTED_VMCONFIG), WordFactory.unsigned(3), BASICLOCK_METADATA_LOCATION); + lock.writeWord(lockMetadataOffset(INJECTED_VMCONFIG), WordFactory.unsigned(unusedMark(INJECTED_VMCONFIG)), BASICLOCK_METADATA_LOCATION); return tryEnterInflated(object, lock, mark, thread, trace, counters); } @@ -468,7 +480,9 @@ public static void monitorexit(Object object, @ConstantParameter int lockDepth, trace(trace, " lock: 0x%016lx\n", lock); if (tryFastPathUnlocking(object, trace, counters, thread, lock)) { - decrementHeldMonitorCount(thread); + if (JavaVersionUtil.JAVA_SPEC == 21 || useStackLocking(INJECTED_VMCONFIG)) { + decrementHeldMonitorCount(thread); + } } else { monitorexitStubC(MONITOREXIT, object, lock); } @@ -503,7 +517,7 @@ private static boolean tryStackUnlocking(Object object, Word thread, Word lock, Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG)); if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorValue(INJECTED_VMCONFIG)).notEqual(0))) { - return tryExitInflated(object, thread, lock, trace, counters); + return tryExitInflated(object, mark, thread, lock, trace, counters); } if (probability(VERY_FAST_PATH_PROBABILITY, Word.objectToTrackedPointer(object).logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), @@ -543,7 +557,7 @@ private static boolean tryLightweightUnlocking(Object object, Word thread, Word AssertionSnippets.vmMessageC(VM_MESSAGE_C, true, cstring("Fast Unlock not monitor"), 0L, 0L, 0L); } } - return tryExitInflated(object, thread, lock, trace, counters); + return tryExitInflated(object, mark, thread, lock, trace, counters); } // Pop lock-stack. @@ -580,10 +594,8 @@ private static boolean tryLightweightUnlocking(Object object, Word thread, Word return false; } - private static boolean tryExitInflated(Object object, Word thread, Word lock, boolean trace, Counters counters) { + private static boolean tryExitInflated(Object object, Word mark, Word thread, Word lock, boolean trace, Counters counters) { // Inflated case - // mark is a pointer to the ObjectMonitor + monitorMask - Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG)); Word monitor; if (useObjectMonitorTable(INJECTED_VMCONFIG)) { monitor = lock.readWord(lockMetadataOffset(INJECTED_VMCONFIG), BASICLOCK_METADATA_LOCATION); @@ -593,10 +605,12 @@ private static boolean tryExitInflated(Object object, Word thread, Word lock, bo return false; } } else { + // mark is a pointer to the ObjectMonitor + monitorMask monitor = mark.subtract(monitorValue(INJECTED_VMCONFIG)); } int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG); + int recursionsOffset = objectMonitorRecursionsOffset(INJECTED_VMCONFIG); Word recursions = monitor.readWord(recursionsOffset, OBJECT_MONITOR_RECURSION_LOCATION); if (probability(FAST_PATH_PROBABILITY, recursions.equal(0))) { @@ -609,7 +623,7 @@ private static boolean tryExitInflated(Object object, Word thread, Word lock, bo // Nobody is waiting, success // release_store memoryBarrier(MembarNode.FenceKind.STORE_RELEASE); - monitor.writeWord(ownerOffset, zero()); + monitor.writeWord(ownerOffset, nullPointer(), OBJECT_MONITOR_OWNER_LOCATION); traceObject(trace, "-lock{heavyweight:simple}", object, false); counters.unlockHeavySimple.inc(); return true; @@ -619,7 +633,7 @@ private static boolean tryExitInflated(Object object, Word thread, Word lock, bo if (probability(FREQUENT_PROBABILITY, succ.isNonNull())) { // There may be a thread spinning on this monitor. Temporarily setting // the monitor owner to null, and hope that the other thread will grab it. - monitor.writeWordVolatile(ownerOffset, zero()); + monitor.writeWordVolatile(ownerOffset, nullPointer()); succ = monitor.readWordVolatile(succOffset, OBJECT_MONITOR_SUCC_LOCATION); if (probability(NOT_FREQUENT_PROBABILITY, succ.isNonNull())) { // We manage to release the monitor before the other running thread even @@ -692,8 +706,8 @@ private static void decrementHeldMonitorCount(Word thread) { } private static void updateHeldMonitorCount(Word thread, int increment) { - Word heldMonitorCount = thread.readWord(HotSpotReplacementsUtil.heldMonitorCountOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); - thread.writeWord(HotSpotReplacementsUtil.heldMonitorCountOffset(INJECTED_VMCONFIG), heldMonitorCount.add(increment), HotSpotReplacementsUtil.JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); + Word heldMonitorCount = thread.readWord(heldMonitorCountOffset(INJECTED_VMCONFIG), JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); + thread.writeWord(heldMonitorCountOffset(INJECTED_VMCONFIG), heldMonitorCount.add(increment), JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); } @Fold @@ -821,7 +835,8 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot JAVA_THREAD_LOCK_STACK_LOCATION, JAVA_THREAD_LOCK_STACK_TOP_LOCATION, JAVA_THREAD_OM_CACHE_LOCATION, - JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; + JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION, + JAVA_THREAD_LOCK_ID_LOCATION}; exitLocations = new LocationIdentity[]{ JAVA_THREAD_LOCK_STACK_LOCATION, JAVA_THREAD_LOCK_STACK_TOP_LOCATION, @@ -831,10 +846,13 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot OBJECT_MONITOR_ENTRY_LIST_LOCATION, OBJECT_MONITOR_RECURSION_LOCATION, OBJECT_MONITOR_SUCC_LOCATION, + OBJECT_MONITOR_STACK_LOCKER_LOCATION, MARK_WORD_LOCATION, JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; } else { - enterLocations = new LocationIdentity[]{JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; + enterLocations = new LocationIdentity[]{ + JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION, + JAVA_THREAD_LOCK_ID_LOCATION}; exitLocations = new LocationIdentity[]{ BASICLOCK_METADATA_LOCATION, OBJECT_MONITOR_OWNER_LOCATION, @@ -842,6 +860,7 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot OBJECT_MONITOR_ENTRY_LIST_LOCATION, OBJECT_MONITOR_RECURSION_LOCATION, OBJECT_MONITOR_SUCC_LOCATION, + OBJECT_MONITOR_STACK_LOCKER_LOCATION, MARK_WORD_LOCATION, JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/methodhandles/MethodHandleFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/methodhandles/MethodHandleFeature.java index 485fc871aeb9..01699fc87374 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/methodhandles/MethodHandleFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/methodhandles/MethodHandleFeature.java @@ -121,10 +121,16 @@ public void duringSetup(DuringSetupAccess access) { if (JavaVersionUtil.JAVA_SPEC >= 22) { try { Class referencedKeySetClass = ReflectionUtil.lookupClass("jdk.internal.util.ReferencedKeySet"); - Method create = ReflectionUtil.lookupMethod(referencedKeySetClass, "create", boolean.class, boolean.class, Supplier.class); // The following call must match the static initializer of MethodType#internTable. - runtimeMethodTypeInternTable = create.invoke(null, - /* isSoft */ false, /* useNativeQueue */ true, (Supplier) () -> new ConcurrentHashMap<>(512)); + if (JavaVersionUtil.JAVA_SPEC >= 24) { + Method create = ReflectionUtil.lookupMethod(referencedKeySetClass, "create", boolean.class, Supplier.class); + runtimeMethodTypeInternTable = create.invoke(null, + /* isSoft */ false, (Supplier) () -> new ConcurrentHashMap<>(512)); + } else { + Method create = ReflectionUtil.lookupMethod(referencedKeySetClass, "create", boolean.class, boolean.class, Supplier.class); + runtimeMethodTypeInternTable = create.invoke(null, + /* isSoft */ false, /* useNativeQueue */ true, (Supplier) () -> new ConcurrentHashMap<>(512)); + } referencedKeySetAdd = ReflectionUtil.lookupMethod(referencedKeySetClass, "add", Object.class); } catch (ReflectiveOperationException e) { throw VMError.shouldNotReachHere(e);