diff --git a/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSnippets.java b/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSnippets.java index 8aba90fc02b9..dff15e0bb506 100644 --- a/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSnippets.java +++ b/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSnippets.java @@ -34,6 +34,7 @@ import org.graalvm.compiler.hotspot.HotSpotBackend; import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.PiNode; +import org.graalvm.compiler.nodes.SnippetAnchorNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; @@ -69,31 +70,31 @@ static int implCompressMultiBlock0(Object receiver, byte[] buf, int ofs, int lim @Snippet.ConstantParameter ResolvedJavaType sha256type, @Snippet.ConstantParameter ResolvedJavaType sha512type, @Snippet.ConstantParameter ResolvedJavaType sha3type) { - Object realReceiver = PiNode.piCastNonNull(receiver, receiverType); + Object realReceiver = PiNode.piCast(receiver, receiverType, false, true, SnippetAnchorNode.anchor()); Word bufAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(buf, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + ofs)); if (useMD5Intrinsics(INJECTED_VMCONFIG) && doInstanceof(md5type, realReceiver)) { - Object md5obj = PiNode.piCastNonNull(realReceiver, md5type); + Object md5obj = PiNode.piCast(realReceiver, md5type, false, true, SnippetAnchorNode.anchor()); Object state = RawLoadNode.load(md5obj, HotSpotReplacementsUtil.getFieldOffset(md5type, "state"), JavaKind.Object, LocationIdentity.any()); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); return HotSpotBackend.md5ImplCompressMBStub(bufAddr, stateAddr, ofs, limit); } else if (useSHA1Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha1type, realReceiver)) { - Object sha1obj = PiNode.piCastNonNull(realReceiver, sha1type); + Object sha1obj = PiNode.piCast(realReceiver, sha1type, false, true, SnippetAnchorNode.anchor()); Object state = RawLoadNode.load(sha1obj, HotSpotReplacementsUtil.getFieldOffset(sha1type, "state"), JavaKind.Object, LocationIdentity.any()); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); return HotSpotBackend.shaImplCompressMBStub(bufAddr, stateAddr, ofs, limit); } else if (useSHA256Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha256type, realReceiver)) { - Object sha256obj = PiNode.piCastNonNull(realReceiver, sha256type); + Object sha256obj = PiNode.piCast(realReceiver, sha256type, false, true, SnippetAnchorNode.anchor()); Object state = RawLoadNode.load(sha256obj, HotSpotReplacementsUtil.getFieldOffset(sha256type, "state"), JavaKind.Object, LocationIdentity.any()); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); return HotSpotBackend.sha2ImplCompressMBStub(bufAddr, stateAddr, ofs, limit); } else if (useSHA512Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha512type, realReceiver)) { - Object sha512obj = PiNode.piCastNonNull(realReceiver, sha512type); + Object sha512obj = PiNode.piCast(realReceiver, sha512type, false, true, SnippetAnchorNode.anchor()); Object state = RawLoadNode.load(sha512obj, HotSpotReplacementsUtil.getFieldOffset(sha512type, "state"), JavaKind.Object, LocationIdentity.any()); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Long))); return HotSpotBackend.sha5ImplCompressMBStub(bufAddr, stateAddr, ofs, limit); } else if (useSHA3Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha3type, realReceiver)) { - Object sha3obj = PiNode.piCastNonNull(realReceiver, sha3type); + Object sha3obj = PiNode.piCast(realReceiver, sha3type, false, true, SnippetAnchorNode.anchor()); Object state = RawLoadNode.load(sha3obj, HotSpotReplacementsUtil.getFieldOffset(sha3type, "state"), JavaKind.Object, LocationIdentity.any()); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); return HotSpotBackend.shaImplCompressMBStub(bufAddr, stateAddr, ofs, limit); diff --git a/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/HotSpotArraycopySnippets.java b/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/HotSpotArraycopySnippets.java index a0c0275ea810..c0c97d31f414 100644 --- a/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/HotSpotArraycopySnippets.java +++ b/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/HotSpotArraycopySnippets.java @@ -34,6 +34,7 @@ import org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil; import org.graalvm.compiler.hotspot.word.KlassPointer; import org.graalvm.compiler.nodes.PiNode; +import org.graalvm.compiler.nodes.SnippetAnchorNode; import org.graalvm.compiler.replacements.arraycopy.ArrayCopyCallNode; import org.graalvm.compiler.replacements.arraycopy.ArrayCopySnippets; import org.graalvm.compiler.word.Word; @@ -86,8 +87,8 @@ protected int heapWordSize() { @SuppressWarnings("unused") protected void doCheckcastArraycopySnippet(Object src, int srcPos, Object dest, int destPos, int length, JavaKind elementKind, LocationIdentity arrayLocation, Counters counters) { if (probability(FREQUENT_PROBABILITY, length > 0)) { - Object nonNullSrc = PiNode.asNonNullObject(src); - Object nonNullDest = PiNode.asNonNullObject(dest); + Object nonNullSrc = PiNode.piCastNonNull(src, SnippetAnchorNode.anchor()); + Object nonNullDest = PiNode.piCastNonNull(dest, SnippetAnchorNode.anchor()); Pointer srcKlass = loadHub(nonNullSrc); Pointer destKlass = loadHub(nonNullDest); if (probability(LIKELY_PROBABILITY, srcKlass == destKlass) || probability(LIKELY_PROBABILITY, nonNullDest.getClass() == Object[].class)) { diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java index f2ff8dd2ac8b..f7a080a54098 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java @@ -164,10 +164,10 @@ public static boolean intrinsify(GraphBuilderContext b, ValueNode input, ValueNo return true; } - public static boolean intrinsify(GraphBuilderContext b, ValueNode object, ResolvedJavaType toType, boolean exactType, boolean nonNull) { + public static boolean intrinsify(GraphBuilderContext b, ValueNode object, ResolvedJavaType toType, boolean exactType, boolean nonNull, ValueNode guard) { Stamp stamp = StampFactory.object(exactType ? TypeReference.createExactTrusted(toType) : TypeReference.createWithoutAssumptions(toType), nonNull || StampTool.isPointerNonNull(object.stamp(NodeView.DEFAULT))); - ValueNode value = canonical(object, stamp, null, null); + ValueNode value = canonical(object, stamp, (GuardingNode) guard, null); if (value == null) { value = new PiNode(object, stamp); } @@ -301,23 +301,6 @@ public void setOriginalNode(ValueNode newNode) { assert piStamp.isCompatible(object.stamp(NodeView.DEFAULT)) : "New object stamp not compatible to piStamp"; } - /** - * Casts an object to have an exact, non-null stamp representing {@link Class}. - */ - public static Class asNonNullClass(Object object) { - return asNonNullClassIntrinsic(object, Class.class, true, true); - } - - /** - * Casts an object to have an exact, non-null stamp representing {@link Class}. - */ - public static Class asNonNullObject(Object object) { - return asNonNullClassIntrinsic(object, Object.class, false, true); - } - - @NodeIntrinsic(PiNode.class) - private static native Class asNonNullClassIntrinsic(Object object, @ConstantNodeParameter Class toType, @ConstantNodeParameter boolean exactType, @ConstantNodeParameter boolean nonNull); - /** * Changes the stamp of an object inside a snippet to be the stamp of the node replaced by the * snippet. @@ -375,16 +358,13 @@ public static Class piCastNonNullClass(Class type, GuardingNode guard) { @NodeIntrinsic private static native Class intrinsified(Class object, GuardingNode guard, @ConstantNodeParameter IntrinsifyOp intrinsifyOp); - /** - * Changes the stamp of an object to represent a given type and to indicate that the object is - * not null. - */ - public static Object piCastNonNull(Object object, @ConstantNodeParameter ResolvedJavaType toType) { - return piCast(object, toType, false, true); - } + @NodeIntrinsic + public static native Object piCast(Object object, @ConstantNodeParameter ResolvedJavaType toType, + @ConstantNodeParameter boolean exactType, @ConstantNodeParameter boolean nonNull, GuardingNode guard); @NodeIntrinsic - public static native Object piCast(Object object, @ConstantNodeParameter ResolvedJavaType toType, @ConstantNodeParameter boolean exactType, @ConstantNodeParameter boolean nonNull); + public static native Object piCast(Object object, @ConstantNodeParameter Class toType, + @ConstantNodeParameter boolean exactType, @ConstantNodeParameter boolean nonNull, GuardingNode guard); /** * A placeholder node in a snippet that will be replaced with a {@link PiNode} when the snippet diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitor.java index 6a51765ddbcf..5b0d159b074e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitor.java @@ -43,8 +43,8 @@ * Only the relevant methods from the JDK sources have been kept. Some additional Native * Image-specific functionality has been added. */ -public final class JavaMonitor extends JavaMonitorQueuedSynchronizer { - private long latestJfrTid; +public class JavaMonitor extends JavaMonitorQueuedSynchronizer { + protected long latestJfrTid; public JavaMonitor() { latestJfrTid = 0; @@ -53,7 +53,7 @@ public JavaMonitor() { public void monitorEnter(Object obj) { if (!tryLock()) { long startTicks = JfrTicks.elapsedTicks(); - lock(); + acquire(1); JavaMonitorEnterEvent.emit(obj, latestJfrTid, startTicks); } @@ -80,23 +80,6 @@ protected JavaMonitorConditionObject getOrCreateCondition(boolean createIfNotExi return newCondition; } - /** - * Creates a new {@link JavaMonitor} that is locked by the provided thread. This requires - * patching of internal state. - */ - public static JavaMonitor newLockedMonitorForThread(Thread thread, int recursionDepth) { - JavaMonitor result = new JavaMonitor(); - for (int i = 0; i < recursionDepth; i++) { - result.lock(); - } - - result.latestJfrTid = SubstrateJVM.getThreadId(thread); - assert result.getExclusiveOwnerThread() == Thread.currentThread() : "Must be locked by current thread"; - result.setExclusiveOwnerThread(thread); - - return result; - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void relockObject() { /* @@ -134,24 +117,6 @@ public void relockObject() { private JavaMonitorConditionObject condition; - // see ReentrantLock.NonFairSync.initialTryLock() - boolean initialTryLock() { - Thread current = Thread.currentThread(); - if (compareAndSetState(0, 1)) { // first attempt is unguarded - setExclusiveOwnerThread(current); - return true; - } else if (getExclusiveOwnerThread() == current) { - int c = getState() + 1; - if (c < 0) { // overflow - throw new Error("Maximum lock count exceeded"); - } - setState(c); - return true; - } else { - return false; - } - } - // see ReentrantLock.NonFairSync.tryAcquire(int) @Override protected boolean tryAcquire(int acquires) { @@ -181,13 +146,6 @@ boolean tryLock() { return false; } - // see ReentrantLock.Sync.lock() - void lock() { - if (!initialTryLock()) { - acquire(1); - } - } - // see ReentrantLock.Sync.tryRelease() @Override protected boolean tryRelease(int releases) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitorQueuedSynchronizer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitorQueuedSynchronizer.java index 0ebf3953d108..6b498becaf8c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitorQueuedSynchronizer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitorQueuedSynchronizer.java @@ -217,6 +217,19 @@ private static void signalNext(Node h) { } } + protected boolean hasReleaseSuccessor() { + Node h = head; + if (h == null) { + return false; + } + Node s = h.next; + return s != null && s.status != 0; + } + + protected void signalReleaseSuccessor() { + signalNext(head); + } + // see AbstractQueuedSynchronizer.acquire(Node, int, boolean, boolean, boolean, long) @SuppressWarnings("all") final int acquire(Node node, int arg) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MultiThreadedMonitorSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MultiThreadedMonitorSupport.java index f5524846fe0c..5cb7a5a8c28f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MultiThreadedMonitorSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MultiThreadedMonitorSupport.java @@ -375,19 +375,28 @@ protected static Object replaceObject(Object unreplacedObject) { return unreplacedObject; } - protected final JavaMonitor getOrCreateMonitor(Object unreplacedObject, boolean createIfNotExisting) { - Object obj = replaceObject(unreplacedObject); - assert obj != null; + protected final JavaMonitor getOrCreateMonitor(Object obj, boolean createIfNotExisting) { int monitorOffset = getMonitorOffset(obj); if (monitorOffset != 0) { /* The common case: pointer to the monitor reserved in the object. */ return getOrCreateMonitorFromObject(obj, createIfNotExisting, monitorOffset); } else { - /* No memory reserved for a lock in the object, fall back to secondary storage. */ - return getOrCreateMonitorFromMap(obj, createIfNotExisting); + return getOrCreateMonitorSlow(obj, createIfNotExisting); } } + private JavaMonitor getOrCreateMonitorSlow(Object unreplacedObject, boolean createIfNotExisting) { + Object replacedObject = replaceObject(unreplacedObject); + if (replacedObject != unreplacedObject) { + int monitorOffset = getMonitorOffset(replacedObject); + if (monitorOffset != 0) { + return getOrCreateMonitorFromObject(replacedObject, createIfNotExisting, monitorOffset); + } + } + /* No memory reserved for a lock in the object, fall back to secondary storage. */ + return getOrCreateMonitorFromMap(replacedObject, createIfNotExisting); + } + protected JavaMonitor getOrCreateMonitorFromObject(Object obj, boolean createIfNotExisting, int monitorOffset) { JavaMonitor existingMonitor = (JavaMonitor) BarrieredAccess.readObject(obj, monitorOffset); if (existingMonitor != null || !createIfNotExisting) { @@ -427,7 +436,7 @@ protected JavaMonitor getOrCreateMonitorFromMap(Object obj, boolean createIfNotE } } - protected static JavaMonitor newMonitorLock() { + protected JavaMonitor newMonitorLock() { return new JavaMonitor(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java index b319efc1358c..4fdf2d469260 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java @@ -31,6 +31,8 @@ import java.util.Objects; import java.util.concurrent.ThreadFactory; +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platforms; @@ -283,7 +285,11 @@ public long getId() { @TargetElement(onlyWith = ContinuationsNotSupported.class) static Thread currentThread() { Thread thread = PlatformThreads.currentThread.get(); - assert thread != null : "Thread has not been set yet"; + if (GraalDirectives.inIntrinsic()) { + ReplacementsUtil.dynamicAssert(thread != null, "Thread has not been set yet"); + } else { + assert thread != null : "Thread has not been set yet"; + } return thread; }