[GR-45962] Backport: Use markWord::lock_mask_in_place in identity hash code fast path after JDK 18.

PullRequest: graal/14672

Author: mur47x111

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -223,7 +223,7 @@ public final int logMinObjAlignment() {
     public final int markOffset = getFieldOffset("oopDesc::_mark", Integer.class, markWord);
     public final int hubOffset = getFieldOffset("oopDesc::_metadata._klass", Integer.class, "Klass*");
 
-    public final int prototypeMarkWordOffset = getFieldOffset("Klass::_prototype_header", Integer.class, markWord, -1, JDK < 18);
+    public final Integer prototypeMarkWordOffset = getFieldOffset("Klass::_prototype_header", Integer.class, markWord, null, JDK < 18);
     public final int superCheckOffsetOffset = getFieldOffset("Klass::_super_check_offset", Integer.class, "juint");
     public final int secondarySuperCacheOffset = getFieldOffset("Klass::_secondary_super_cache", Integer.class, "Klass*");
     public final int secondarySupersOffset = getFieldOffset("Klass::_secondary_supers", Integer.class, "Array<Klass*>*");
@@ -404,13 +404,28 @@ public int threadLastJavaFpOffset() {
     public final int frameInterpreterFrameSenderSpOffset = getConstant("frame::interpreter_frame_sender_sp_offset", Integer.class, 0, osArch.equals("amd64"));
     public final int frameInterpreterFrameLastSpOffset = getConstant("frame::interpreter_frame_last_sp_offset", Integer.class, 0, osArch.equals("amd64"));
 
-    public final int biasedLockMaskInPlace = getConstant(markWordField("biased_lock_mask_in_place"), Integer.class, -1, JDK < 18);
+    private final Integer biasedLockMaskInPlace = getConstant(markWordField("biased_lock_mask_in_place"), Integer.class, null, JDK < 18);
+    private final Integer lockMaskInPlace = getConstant(markWordField("lock_mask_in_place"), Integer.class, null, JDK == 20 && jvmciGE(JVMCI_23_0_b12));
+
+    public int getLockMaskInPlace() {
+        if (JDK >= 18) {
+            if (lockMaskInPlace != null) {
+                return lockMaskInPlace;
+            } else {
+                // This presumes markWord::unlocked_value is non-zero.
+                return 0;
+            }
+        } else {
+            return biasedLockMaskInPlace;
+        }
+    }
+
     public final int ageMaskInPlace = getConstant(markWordField("age_mask_in_place"), Integer.class);
-    public final int epochMaskInPlace = getConstant(markWordField("epoch_mask_in_place"), Integer.class, -1, JDK < 18);
+    public final Integer epochMaskInPlace = getConstant(markWordField("epoch_mask_in_place"), Integer.class, null, JDK < 18);
 
     public final int unlockedMask = getConstant(markWordField("unlocked_value"), Integer.class);
     public final int monitorMask = getConstant(markWordField("monitor_value"), Integer.class);
-    public final int biasedLockPattern = getConstant(markWordField("biased_lock_pattern"), Integer.class, -1, JDK < 18);
+    public final Integer biasedLockPattern = getConstant(markWordField("biased_lock_pattern"), Integer.class, null, JDK < 18);
 
     // This field has no type in vmStructs.cpp
     public final int objectMonitorOwner = getFieldOffset("ObjectMonitor::_owner", Integer.class, null);

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfigAccess.java

@@ -126,13 +126,15 @@ protected static String getProperty(String name) {
     protected static final Version JVMCI_23_0_b06 = new Version(23, 0, 6);
     protected static final Version JVMCI_23_0_b07 = new Version(23, 0, 7);
     protected static final Version JVMCI_23_0_b10 = new Version(23, 0, 10);
+    protected static final Version JVMCI_23_0_b12 = new Version(23, 0, 12);
 
     public static boolean jvmciGE(Version v) {
         return JVMCI && !JVMCI_VERSION.isLessThan(v);
     }
 
-    static final int JDK = Runtime.version().feature();
+    public static final int JDK = Runtime.version().feature();
     static final int JDK_UPDATE = Runtime.version().update();
+    static final int JDK_BUILD = Runtime.version().build().orElse(0);
     public static final boolean IS_OPENJDK = getProperty("java.vm.name", "").startsWith("OpenJDK");
     public static final Version JVMCI_VERSION;
     public static final boolean JVMCI;

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotHashCodeSnippets.java

@@ -26,29 +26,47 @@
 
 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.IDENTITY_HASHCODE;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockMaskInPlace;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.identityHashCode;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.identityHashCodeShift;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadWordFromObject;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockMaskInPlace;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.markOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.uninitializedIdentityHashCodeValue;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.unlockedMask;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
 
+import org.graalvm.compiler.lir.StubPort;
 import org.graalvm.compiler.replacements.IdentityHashCodeSnippets;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.word.WordFactory;
 
+// @formatter:off
+@StubPort(path      = "src/hotspot/share/opto/library_call.cpp",
+          lineStart = 4311,
+          lineEnd   = 4435,
+          commit    = "540c706bbcbb809ae1304aac4f2a16a5e83cb458",
+          sha1      = "bbf28398bfdef37afbb856160110d010c60e87e7")
+// @formatter:on
 public class HotSpotHashCodeSnippets extends IdentityHashCodeSnippets {
 
     @Override
     protected int computeIdentityHashCode(final Object x) {
         Word mark = loadWordFromObject(x, markOffset(INJECTED_VMCONFIG));
 
-        // this code is independent from biased locking (although it does not look that way)
-        final Word biasedLock = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG));
-        if (probability(FAST_PATH_PROBABILITY, biasedLock.equal(WordFactory.unsigned(unlockedMask(INJECTED_VMCONFIG))))) {
+        // When the object is unlocked, HotSpot reuses upper bits (e.g., [38:8] in 64-bits VM) of
+        // the mark word in object header for caching identity hash code. The same bits are part of
+        // the thread pointer in the case of biased locking, pointer to the displaced mark in a
+        // thread's stack in the case of stack locking, or pointer to the monitor object in the case
+        // of inflated lock.
+        //
+        // To test if an object is unlocked, we simply need to compare the lock bits against the
+        // constant unlocked value. The lock bits are the least significant 3 bits prior to Java 18
+        // (1 bit for biased locking and 2 bits for stack locking or heavy locking), and 2 bits
+        // afterwards due to elimination of the biased locking. The unlocked values are 001 and 01
+        // respectively. See src/hotspot/share/oops/markWord.hpp for more details.
+        final Word lockBits = mark.and(lockMaskInPlace(INJECTED_VMCONFIG));
+        if (probability(FAST_PATH_PROBABILITY, lockBits.equal(WordFactory.unsigned(unlockedMask(INJECTED_VMCONFIG))))) {
             int hash = (int) mark.unsignedShiftRight(identityHashCodeShift(INJECTED_VMCONFIG)).rawValue();
             if (probability(FAST_PATH_PROBABILITY, hash != uninitializedIdentityHashCodeValue(INJECTED_VMCONFIG))) {
                 return hash;

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java

@@ -511,18 +511,13 @@ public static int objectMonitorSuccOffset(@InjectedParameter GraalHotSpotVMConfi
     }
 
     /**
-     * Mask for a biasable, locked or unlocked mark word.
-     *
-     * <pre>
-     * +----------------------------------+-+-+
-     * |                                 1|1|1|
-     * +----------------------------------+-+-+
-     * </pre>
-     *
+     * Mask for a biasable, locked or unlocked mark word. It is the least significant 3 bits prior
+     * to Java 18 (1 bit for biased locking and 2 bits for stack locking or heavy locking), and 2
+     * bits afterwards due to elimination of the biased locking.
      */
     @Fold
-    public static int biasedLockMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
-        return config.biasedLockMaskInPlace;
+    public static int lockMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
+        return config.getLockMaskInPlace();
     }
 
     @Fold

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java

@@ -26,6 +26,7 @@
 
 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_OPTIONVALUES;
 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
+import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigAccess.JDK;
 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallDescriptor.Reexecutability.NOT_REEXECUTABLE;
 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallDescriptor.Transition.SAFEPOINT;
 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallDescriptor.Transition.STACK_INSPECTABLE_LEAF;
@@ -44,7 +45,6 @@
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_SUCC_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.ageMaskInPlace;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockMaskInPlace;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockPattern;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.diagnoseSyncOnValueBasedClasses;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.epochMaskInPlace;
@@ -53,6 +53,7 @@
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.klassAccessFlagsOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadWordFromObject;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockDisplacedMarkOffset;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockMaskInPlace;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.markOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.monitorMask;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorCxqOffset;
@@ -274,7 +275,7 @@ public static void monitorenter(Object object, KlassPointer hub, @ConstantParame
             }
         }
 
-        if (useBiasedLocking(INJECTED_VMCONFIG)) {
+        if (probability(NOT_LIKELY_PROBABILITY, JDK < 18) && useBiasedLocking(INJECTED_VMCONFIG)) {
             if (tryEnterBiased(object, hub, lock, mark, thread, trace, counters)) {
                 return;
             }
@@ -347,7 +348,7 @@ private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock
         // whether the epoch is still valid.
         // Note that the runtime guarantees sufficient alignment of JavaThread
         // pointers to allow age to be placed into low bits.
-        final Word biasableLockBits = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG));
+        final Word biasableLockBits = mark.and(lockMaskInPlace(INJECTED_VMCONFIG));
 
         // Check whether the bias pattern is present in the object's mark word
         // and the bias owner and the epoch are both still current.
@@ -376,7 +377,7 @@ private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock
             // If the low three bits in the xor result aren't clear, that means
             // the prototype header is no longer biasable and we have to revoke
             // the bias on this object.
-            if (probability(FREQUENT_PROBABILITY, tmp.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(0))) {
+            if (probability(FREQUENT_PROBABILITY, tmp.and(lockMaskInPlace(INJECTED_VMCONFIG)).equal(0))) {
                 // Biasing is still enabled for object's type. See whether the
                 // epoch of the current bias is still valid, meaning that the epoch
                 // bits of the mark word are equal to the epoch bits of the
@@ -393,7 +394,7 @@ private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock
                     // fails we will go in to the runtime to revoke the object's bias.
                     // Note that we first construct the presumed unbiased header so we
                     // don't accidentally blow away another thread's valid bias.
-                    Word unbiasedMark = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG) | ageMaskInPlace(INJECTED_VMCONFIG) | epochMaskInPlace(INJECTED_VMCONFIG));
+                    Word unbiasedMark = mark.and(lockMaskInPlace(INJECTED_VMCONFIG) | ageMaskInPlace(INJECTED_VMCONFIG) | epochMaskInPlace(INJECTED_VMCONFIG));
                     Word biasedMark = unbiasedMark.or(thread);
                     trace(trace, "     unbiasedMark: 0x%016lx\n", unbiasedMark);
                     trace(trace, "       biasedMark: 0x%016lx\n", biasedMark);
@@ -533,7 +534,7 @@ public static void monitorexit(Object object, @ConstantParameter int lockDepth,
                     @ConstantParameter Counters counters) {
         Word thread = registerAsWord(threadRegister);
         trace(trace, "           object: 0x%016lx\n", Word.objectToTrackedPointer(object));
-        if (useBiasedLocking(INJECTED_VMCONFIG)) {
+        if (probability(NOT_LIKELY_PROBABILITY, JDK < 18) && useBiasedLocking(INJECTED_VMCONFIG)) {
             // Check for biased locking unlock case, which is a no-op
             // Note: we do not have to check the thread ID for two reasons.
             // First, the interpreter checks for IllegalMonitorStateException at
@@ -542,7 +543,7 @@ public static void monitorexit(Object object, @ConstantParameter int lockDepth,
             // the bias bit would be clear.
             final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
             trace(trace, "             mark: 0x%016lx\n", mark);
-            if (probability(FREQUENT_PROBABILITY, mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
+            if (probability(FREQUENT_PROBABILITY, mark.and(lockMaskInPlace(INJECTED_VMCONFIG)).equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
                 endLockScope();
                 decCounter();
                 traceObject(trace, "-lock{bias}", object, false);

compiler/src/org.graalvm.micro.benchmarks/src/micro/benchmarks/HashBenchmark.java

@@ -47,4 +47,9 @@ public int hash(ThreadState state) {
         }
         return value;
     }
+
+    @Benchmark
+    public int identityHashCode(ThreadState state) {
+        return System.identityHashCode(state.characters);
+    }
 }