Allow virtual threads to unmount while holding monitors

Author: pchilano

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -119,8 +119,8 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     cbnz(hdr, slow_case);
     // done
     bind(done);
+    inc_held_monitor_count();
   }
-  increment(Address(rthread, JavaThread::held_monitor_count_offset()));
   return null_check_offset;
 }
 
@@ -159,8 +159,8 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
     }
     // done
     bind(done);
+    dec_held_monitor_count();
   }
-  decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 }
 
 

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -57,7 +57,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
   Label count, no_count;
 
   assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
-  assert_different_registers(oop, box, tmp, disp_hdr);
+  assert_different_registers(oop, box, tmp, disp_hdr, rscratch2);
 
   // Load markWord from object into displaced_header.
   ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
@@ -111,11 +111,13 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
   bind(object_has_monitor);
 
   // The object's monitor m is unlocked iff m->owner == nullptr,
-  // otherwise m->owner may contain a thread or a stack address.
+  // otherwise m->owner may contain a thread id, a stack address for LM_LEGACY,
+  // or the ANONYMOUS_OWNER constant for LM_LIGHTWEIGHT.
   //
   // Try to CAS m->owner from null to current thread.
+  ldr(rscratch2, Address(rthread, JavaThread::lock_id_offset()));
   add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
-  cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
+  cmpxchg(tmp, zr, rscratch2, Assembler::xword, /*acquire*/ true,
           /*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
 
   // Store a non-null value into the box to avoid looking like a re-entrant
@@ -127,7 +129,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
 
   br(Assembler::EQ, cont); // CAS success means locking succeeded
 
-  cmp(tmp3Reg, rthread);
+  cmp(tmp3Reg, rscratch2);
   br(Assembler::NE, cont); // Check for recursive locking
 
   // Recursive lock case
@@ -140,7 +142,9 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
   br(Assembler::NE, no_count);
 
   bind(count);
-  increment(Address(rthread, JavaThread::held_monitor_count_offset()));
+  if (LockingMode == LM_LEGACY) {
+    inc_held_monitor_count();
+  }
 
   bind(no_count);
 }
@@ -247,15 +251,17 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
   br(Assembler::NE, no_count);
 
   bind(count);
-  decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
+  if (LockingMode == LM_LEGACY) {
+    dec_held_monitor_count();
+  }
 
   bind(no_count);
 }
 
 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register t1,
                                               Register t2, Register t3) {
   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
-  assert_different_registers(obj, box, t1, t2, t3);
+  assert_different_registers(obj, box, t1, t2, t3, rscratch2);
 
   // Handle inflated monitor.
   Label inflated;
@@ -371,13 +377,14 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist
     // Compute owner address.
     lea(t2_owner_addr, owner_address);
 
-    // CAS owner (null => current thread).
-    cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
+    // CAS owner (null => current thread id).
+    ldr(rscratch2, Address(rthread, JavaThread::lock_id_offset()));
+    cmpxchg(t2_owner_addr, zr, rscratch2, Assembler::xword, /*acquire*/ true,
             /*release*/ false, /*weak*/ false, t3_owner);
     br(Assembler::EQ, monitor_locked);
 
     // Check if recursive.
-    cmp(t3_owner, rthread);
+    cmp(t3_owner, rscratch2);
     br(Assembler::NE, slow_path);
 
     // Recursive.
@@ -390,7 +397,6 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist
   }
 
   bind(locked);
-  increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 
 #ifdef ASSERT
   // Check that locked label is reached with Flags == EQ.
@@ -559,7 +565,6 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi
   }
 
   bind(unlocked);
-  decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
   cmp(zr, zr); // Set Flags to EQ => fast path
 
 #ifdef ASSERT

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

@@ -697,7 +697,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
 
     if (LockingMode == LM_LIGHTWEIGHT) {
       lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
-      b(count);
+      b(done);
     } else if (LockingMode == LM_LEGACY) {
       // Load (object->mark() | 1) into swap_reg
       ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
@@ -747,18 +747,18 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
 
       // Save the test result, for recursive case, the result is zero
       str(swap_reg, Address(lock_reg, mark_offset));
-      br(Assembler::EQ, count);
+      br(Assembler::NE, slow_case);
+
+      bind(count);
+      inc_held_monitor_count();
+      b(done);
     }
     bind(slow_case);
 
     // Call the runtime routine for slow case
     call_VM(noreg,
             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
             lock_reg);
-    b(done);
-
-    bind(count);
-    increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 
     bind(done);
   }
@@ -804,11 +804,10 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
     // Free entry
     str(zr, Address(lock_reg, BasicObjectLock::obj_offset()));
 
+    Label slow_case;
     if (LockingMode == LM_LIGHTWEIGHT) {
-      Label slow_case;
       lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
-      b(count);
-      bind(slow_case);
+      b(done);
     } else if (LockingMode == LM_LEGACY) {
       // Load the old header from BasicLock structure
       ldr(header_reg, Address(swap_reg,
@@ -818,16 +817,17 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
       cbz(header_reg, count);
 
       // Atomic swap back the old header
-      cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
+      cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, &slow_case);
+
+      bind(count);
+      dec_held_monitor_count();
+      b(done);
     }
+
+    bind(slow_case);
     // Call the runtime routine for slow case.
     str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj
     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
-    b(done);
-
-    bind(count);
-    decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
-
     bind(done);
     restore_bcp();
   }

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -39,6 +39,7 @@
 #include "gc/shared/tlab_globals.hpp"
 #include "interpreter/bytecodeHistogram.hpp"
 #include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
 #include "jvm.h"
 #include "memory/resourceArea.hpp"
 #include "memory/universe.hpp"
@@ -5327,6 +5328,38 @@ void MacroAssembler::tlab_allocate(Register obj,
   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 }
 
+void MacroAssembler::inc_held_monitor_count() {
+  Address dst = Address(rthread, JavaThread::held_monitor_count_offset());
+#ifdef ASSERT
+  ldr(rscratch2, dst);
+  increment(rscratch2);
+  str(rscratch2, dst);
+  Label ok;
+  tbz(rscratch2, 63, ok);
+  STOP("assert(held monitor count underflow)");
+  should_not_reach_here();
+  bind(ok);
+#else
+  increment(dst);
+#endif
+}
+
+void MacroAssembler::dec_held_monitor_count() {
+  Address dst = Address(rthread, JavaThread::held_monitor_count_offset());
+#ifdef ASSERT
+  ldr(rscratch2, dst);
+  decrement(rscratch2);
+  str(rscratch2, dst);
+  Label ok;
+  tbz(rscratch2, 63, ok);
+  STOP("assert(held monitor count underflow)");
+  should_not_reach_here();
+  bind(ok);
+#else
+  decrement(dst);
+#endif
+}
+
 void MacroAssembler::verify_tlab() {
 #ifdef ASSERT
   if (UseTLAB && VerifyOops) {

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -940,6 +940,8 @@ class MacroAssembler: public Assembler {
 
   void push_cont_fastpath(Register java_thread);
   void pop_cont_fastpath(Register java_thread);
+  void inc_held_monitor_count();
+  void dec_held_monitor_count();
 
   // Round up to a power of two
   void round_to(Register reg, int modulus);

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -1816,12 +1816,13 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       // Save the test result, for recursive case, the result is zero
       __ str(swap_reg, Address(lock_reg, mark_word_offset));
       __ br(Assembler::NE, slow_path_lock);
+
+      __ bind(count);
+      __ inc_held_monitor_count();
     } else {
       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
       __ lightweight_lock(lock_reg, obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
     }
-    __ bind(count);
-    __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 
     // Slow path will re-enter here
     __ bind(lock_done);
@@ -1926,7 +1927,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       // Simple recursive lock?
       __ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
       __ cbnz(rscratch1, not_recursive);
-      __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
+      __ dec_held_monitor_count();
       __ b(done);
     }
 
@@ -1949,11 +1950,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       Label count;
       __ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, count, &slow_path_unlock);
       __ bind(count);
-      __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
+      __ dec_held_monitor_count();
     } else {
       assert(LockingMode == LM_LIGHTWEIGHT, "");
       __ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock);
-      __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
     }
 
     // slow path re-enters here

src/hotspot/cpu/ppc/macroAssembler_ppc.cpp

@@ -2625,10 +2625,12 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
 
   // Try to CAS m->owner from null to current thread.
   addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
+  Register thread_id = displaced_header;
+  ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
   cmpxchgd(/*flag=*/flag,
            /*current_value=*/current_header,
            /*compare_value=*/(intptr_t)0,
-           /*exchange_value=*/R16_thread,
+           /*exchange_value=*/thread_id,
            /*where=*/temp,
            MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
            MacroAssembler::cmpxchgx_hint_acquire_lock());
@@ -2638,7 +2640,7 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
   beq(flag, success);
 
   // Check for recursive locking.
-  cmpd(flag, current_header, R16_thread);
+  cmpd(flag, current_header, thread_id);
   bne(flag, failure);
 
   // Current thread already owns the lock. Just increment recursions.
@@ -2895,18 +2897,20 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla
       addi(owner_addr, monitor, in_bytes(ObjectMonitor::owner_offset()));
     }
 
-    // CAS owner (null => current thread).
+    // CAS owner (null => current thread id).
+    Register thread_id = tmp1;
+    ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
     cmpxchgd(/*flag=*/CCR0,
             /*current_value=*/t,
             /*compare_value=*/(intptr_t)0,
-            /*exchange_value=*/R16_thread,
+            /*exchange_value=*/thread_id,
             /*where=*/owner_addr,
             MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
             MacroAssembler::cmpxchgx_hint_acquire_lock());
     beq(CCR0, monitor_locked);
 
     // Check if recursive.
-    cmpd(CCR0, t, R16_thread);
+    cmpd(CCR0, t, thread_id);
     bne(CCR0, slow_path);
 
     // Recursive.

src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp

@@ -83,8 +83,8 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     // displaced header address in the object header - if it is not the same, get the
     // object header instead
     la(temp, Address(obj, hdr_offset));
-    cmpxchgptr(hdr, disp_hdr, temp, t1, done, /*fallthough*/nullptr);
     // if the object header was the same, we're done
+    cmpxchgptr(hdr, disp_hdr, temp, t1, done, /*fallthough*/nullptr);
     // if the object header was not the same, it is now in the hdr register
     // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
     //
@@ -106,11 +106,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     sd(hdr, Address(disp_hdr, 0));
     // otherwise we don't care about the result and handle locking via runtime call
     bnez(hdr, slow_case, /* is_far */ true);
+
     // done
     bind(done);
+    inc_held_monitor_count();
   }
 
-  increment(Address(xthread, JavaThread::held_monitor_count_offset()));
   return null_check_offset;
 }
 
@@ -146,11 +147,11 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
     } else {
       cmpxchgptr(disp_hdr, hdr, obj, t1, done, &slow_case);
     }
+
     // done
     bind(done);
+    dec_held_monitor_count();
   }
-
-  decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
 }
 
 // Defines obj, preserves var_size_in_bytes

src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp

@@ -70,5 +70,3 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
   __ relocate(entry_guard_Relocation::spec());
   __ emit_int32(0);  // nmethod guard value
 }
-
-#undef __