WIP: AOSCOS: LOONGARCH: Port of JEP 491: Synchronize Virtual Threads without Pinning

Fixes: 78b8015 ("8338383: Implement JEP 491: Synchronize Virtual Threads without Pinning")
Fixes: c113f82 ("8343957: Rename ObjectMonitor::owner_from() and JavaThread::_lock_id")
Follow-up: 78b8015 ("8338383: Implement JEP 491: Synchronize Virtual Threads without Pinning")
Follow-up: c113f82 ("8343957: Rename ObjectMonitor::owner_from() and JavaThread::_lock_id")
Link: openjdk#21565
Link: openjdk#22524
Signed-off-by: Bingwu Zhang <[email protected]>

Author: xtexx

src/hotspot/cpu/loongarch/assembler_loongarch.hpp

@@ -94,6 +94,7 @@ constexpr Register LVP            = S7;
 // temporary callee saved register
 constexpr Register TSR            = S2;
 
+// java thread pointer
 constexpr Register TREG           = S6;
 
 constexpr Register S5_heapbase    = S5;

src/hotspot/cpu/loongarch/c1_CodeStubs_loongarch_64.cpp

@@ -320,7 +320,7 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
                          relocInfo::static_call_type);
   address call = __ trampoline_call(resolve);
   if (call == nullptr) {
-    ce->bailout("trampoline stub overflow");
+    ce->bailout("reloc call address stub overflow");
     return;
   }
   ce->add_call_info_here(info());

src/hotspot/cpu/loongarch/c1_LIRAssembler_loongarch_64.cpp

@@ -2267,7 +2267,7 @@ void LIR_Assembler::align_call(LIR_Code code) {}
 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
   address call = __ trampoline_call(AddressLiteral(op->addr(), rtype));
   if (call == nullptr) {
-    bailout("trampoline stub overflow");
+    bailout("reloc call address stub overflow");
     return;
   }
   add_call_info(code_offset(), op->info());
@@ -2277,7 +2277,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
   address call = __ ic_call(op->addr());
   if (call == nullptr) {
-    bailout("trampoline stub overflow");
+    bailout("reloc call address stub overflow");
     return;
   }
   add_call_info(code_offset(), op->info());

src/hotspot/cpu/loongarch/c1_MacroAssembler_loongarch_64.cpp

@@ -40,7 +40,7 @@
 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   const int aligned_mask = BytesPerWord -1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
-  assert_different_registers(hdr, obj, disp_hdr);
+  assert_different_registers(hdr, obj, disp_hdr, T0);
   int null_check_offset = -1;
 
   verify_oop(obj);
@@ -96,15 +96,15 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     bnez(hdr, slow_case);
     // done
     bind(done);
+    inc_held_monitor_count(T0);
   }
-  increment(Address(TREG, JavaThread::held_monitor_count_offset()), 1);
   return null_check_offset;
 }
 
 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   const int aligned_mask = BytesPerWord -1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
-  assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
+  assert_different_registers(hdr, obj, disp_hdr, T0);
   Label done;
 
   if (LockingMode != LM_LIGHTWEIGHT) {
@@ -134,8 +134,8 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
     }
     // done
     bind(done);
+    dec_held_monitor_count(T0);
   }
-  decrement(Address(TREG, JavaThread::held_monitor_count_offset()), 1);
 }
 
 // Defines obj, preserves var_size_in_bytes

src/hotspot/cpu/loongarch/c1_Runtime1_loongarch_64.cpp

@@ -160,15 +160,15 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result,
 }
 
 enum return_state_t {
-  does_not_return, requires_return
+  does_not_return, requires_return, requires_pop_epilogue_return
 };
 
 // Implementation of StubFrame
 
 class StubFrame: public StackObj {
  private:
   StubAssembler* _sasm;
-  bool _return_state;
+  return_state_t _return_state;
 
  public:
   StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments,
@@ -183,8 +183,16 @@ void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
   enter();
 }
 
-void StubAssembler::epilogue() {
-  leave();
+void StubAssembler::epilogue(bool use_pop) {
+  // Avoid using a leave instruction when this frame may
+  // have been frozen, since the current value of fp
+  // restored from the stub would be invalid. We still
+  // must restore the fp value saved on enter though.
+  if (use_pop) {
+    pop2(RA, FP);
+  } else {
+    leave();
+  }
   jr(RA);
 }
 
@@ -204,11 +212,12 @@ void StubFrame::load_argument(int offset_in_words, Register reg) {
 }
 
 StubFrame::~StubFrame() {
-  if (_return_state == requires_return) {
-    __ epilogue();
-  } else {
+  if (_return_state == does_not_return) {
     __ should_not_reach_here();
+  } else {
+    __ epilogue(_return_state == requires_pop_epilogue_return);
   }
+  _sasm = nullptr;
 }
 
 #undef __
@@ -256,6 +265,9 @@ static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
     }
   }
 
+  int sp_offset = cpu_reg_save_offsets[TREG->encoding()];
+  oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), TREG->as_VMReg());
+
   if (save_fpu_registers) {
     for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
       FloatRegister r = as_FloatRegister(i);
@@ -333,6 +345,16 @@ void Runtime1::initialize_pd() {
   }
 }
 
+// return: offset in 64-bit words.
+uint Runtime1::runtime_blob_current_thread_offset(frame f) {
+  CodeBlob* cb = f.cb();
+  assert(cb == Runtime1::blob_for(C1StubId::monitorenter_id) ||
+         cb == Runtime1::blob_for(C1StubId::monitorenter_nofpu_id), "must be");
+  assert(cb != nullptr && cb->is_runtime_stub(), "invalid frame");
+  int offset = cpu_reg_save_offsets[TREG->encoding()];
+  return offset / 2;   // SP offsets are in halfwords
+}
+
 // target: the entry point of the method that creates and posts the exception oop
 // has_argument: true if the exception needs arguments (passed in SCR1 and SCR2)
 
@@ -860,7 +882,7 @@ OopMapSet* Runtime1::generate_code_for(C1StubId id, StubAssembler* sasm) {
       // fall through
     case C1StubId::monitorenter_id:
       {
-        StubFrame f(sasm, "monitorenter", dont_gc_arguments);
+        StubFrame f(sasm, "monitorenter", dont_gc_arguments, requires_pop_epilogue_return);
         OopMap* map = save_live_registers(sasm, save_fpu_registers);
 
         // Called with store_parameter and not C abi

src/hotspot/cpu/loongarch/c2_MacroAssembler_loongarch.cpp

@@ -43,13 +43,13 @@
 
 // using the cr register as the bool result: 0 for failed; others success.
 void C2_MacroAssembler::fast_lock_c2(Register oop, Register box, Register flag,
-                                  Register disp_hdr, Register tmp) {
+                                  Register disp_hdr, Register tmp, Register tmp1) {
   Label cont;
   Label object_has_monitor;
   Label count, no_count;
 
   assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
-  assert_different_registers(oop, box, tmp, disp_hdr, flag);
+  assert_different_registers(oop, box, tmp, tmp1, disp_hdr, flag);
 
   // Load markWord from object into displaced_header.
   assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
@@ -91,7 +91,7 @@ void C2_MacroAssembler::fast_lock_c2(Register oop, Register box, Register flag,
     sub_d(disp_hdr, tmp, SP);
     li(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place));
     // If (mark & lock_mask) == 0 and mark - sp < page_size,
-    // we are stack-locking and goto cont,
+    // we are stack-locking and goto label locked,
     // hence we can store 0 as the displaced header in the box,
     // which indicates that it is a recursive lock.
     andr(tmp, disp_hdr, tmp);
@@ -103,12 +103,11 @@ void C2_MacroAssembler::fast_lock_c2(Register oop, Register box, Register flag,
   // Handle existing monitor.
   bind(object_has_monitor);
 
-  // The object's monitor m is unlocked if m->owner is null,
-  // otherwise m->owner may contain a thread or a stack address.
-  //
-  // Try to CAS m->owner from null to current thread.
+  // Try to CAS owner (no owner => current thread's _lock_id).
   move(AT, R0);
   addi_d(tmp, disp_hdr, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
+  Register tid = tmp1;
+  ld_d(tid, Address(TREG, JavaThread::monitor_owner_id_offset()));
   cmpxchg(Address(tmp, 0), AT, TREG, flag, true, true /* acquire */);
 
   // Store a non-null value into the box to avoid looking like a re-entrant
@@ -120,7 +119,7 @@ void C2_MacroAssembler::fast_lock_c2(Register oop, Register box, Register flag,
 
   bnez(flag, cont); // CAS success means locking succeeded
 
-  bne(AT, TREG, cont); // Check for recursive locking
+  bne(AT, tid, cont); // Check for recursive locking
 
   // Recursive lock case
   li(flag, 1);
@@ -132,7 +131,9 @@ void C2_MacroAssembler::fast_lock_c2(Register oop, Register box, Register flag,
   beqz(flag, no_count);
 
   bind(count);
-  increment(Address(TREG, JavaThread::held_monitor_count_offset()), 1);
+  if (LockingMode == LM_LEGACY) {
+    inc_held_monitor_count(T0);
+  }
 
   bind(no_count);
 }
@@ -231,14 +232,16 @@ void C2_MacroAssembler::fast_unlock_c2(Register oop, Register box, Register flag
   beqz(flag, no_count);
 
   bind(count);
-  decrement(Address(TREG, JavaThread::held_monitor_count_offset()), 1);
+  if (LockingMode == LM_LEGACY) {
+    dec_held_monitor_count(T0);
+  }
 
   bind(no_count);
 }
 
-void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register flag, Register tmp1, Register tmp2, Register tmp3) {
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register flag, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
-  assert_different_registers(obj, box, tmp1, tmp2, tmp3, flag);
+  assert_different_registers(obj, box, tmp1, tmp2, tmp3, tmp4, flag);
 
   // Handle inflated monitor.
   Label inflated;
@@ -306,6 +309,7 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist
     bind(inflated);
 
     const Register tmp1_monitor = tmp1;
+
     if (!UseObjectMonitorTable) {
       assert(tmp1_monitor == tmp1_mark, "should be the same here");
     } else {
@@ -353,12 +357,14 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist
     lea(tmp2_owner_addr, owner_address);
 
     move(tmp3_owner, R0);
-    // CAS owner (null => current thread).
-    cmpxchg(Address(tmp2_owner_addr, 0), tmp3_owner, TREG, flag, true, true /* acquire */);
+    // Try to CAS owner (no owner => current thread's _monitor_owner_id).
+    Register tid = tmp4;
+    ld_d(tid, Address(TREG, JavaThread::monitor_owner_id_offset()));
+    cmpxchg(Address(tmp2_owner_addr, 0), tmp3_owner, tid, flag, true, true /* acquire */);
     bnez(flag, locked);
 
     // Check if recursive.
-    bne(tmp3_owner, TREG, slow_path);
+    bne(tmp3_owner, tid, slow_path);
 
     // Recursive.
     increment(recursions_address, 1);
@@ -372,7 +378,6 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist
   bind(locked);
   // Set flag != 0
   li(flag, 1);
-  increment(Address(TREG, JavaThread::held_monitor_count_offset()), 1);
 
 #ifdef ASSERT
   // Check that locked label is reached with flag != 0.
@@ -535,7 +540,6 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi
   bind(unlocked);
   // Set flag != 0
   li(flag, 1);
-  decrement(Address(TREG, JavaThread::held_monitor_count_offset()), 1);
 
 #ifdef ASSERT
   // Check that unlocked label is reached with flag != 0.

src/hotspot/cpu/loongarch/c2_MacroAssembler_loongarch.hpp

@@ -34,12 +34,12 @@
   void cmp_branchEqNe_off21(int flag, Register op1, Label& L);
 
   void fast_lock_c2(Register oop, Register box, Register flag,
-                 Register disp_hdr, Register tmp);
+                 Register disp_hdr, Register tmp, Register tmp1);
   void fast_unlock_c2(Register oop, Register box, Register flag,
                    Register disp_hdr, Register tmp);
   // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file.
   void fast_lock_lightweight(Register object, Register box, Register flag,
-                             Register tmp1, Register tmp2, Register tmp3);
+                             Register tmp1, Register tmp2, Register tmp3, Register tmp4);
   void fast_unlock_lightweight(Register object, Register box, Register flag,
                                Register tmp1, Register tmp2, Register tmp3);
 

src/hotspot/cpu/loongarch/continuationFreezeThaw_loongarch.inline.hpp

@@ -128,6 +128,11 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) {
   }
 }
 
+inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) {
+  assert(f.interpreter_frame_last_sp() == nullptr, "should be null for top frame");
+  f.interpreter_frame_set_last_sp(f.unextended_sp());
+}
+
 inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) {
   assert(hf.fp() == hf.unextended_sp() + (f.fp() - f.unextended_sp()), "");
   assert((f.at(frame::interpreter_frame_last_sp_offset) != 0)
@@ -191,7 +196,8 @@ inline void Thaw<ConfigT>::patch_caller_links(intptr_t* sp, intptr_t* bottom) {
 
 inline frame ThawBase::new_entry_frame() {
   intptr_t* sp = _cont.entrySP();
-  return frame(sp, sp, _cont.entryFP(), _cont.entryPC()); // TODO PERF: This finds code blob and computes deopt state
+  // TODO PERF: This finds code blob and computes deopt state
+  return frame(sp, sp, _cont.entryFP(), _cont.entryPC());
 }
 
 template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) {
@@ -203,7 +209,6 @@ template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame&
     // If caller is interpreted it already made room for the callee arguments
     int overlap = caller.is_interpreted_frame() ? ContinuationHelper::InterpretedFrame::stack_argsize(hf) : 0;
     const int fsize = (int)(ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp() - overlap);
-    const int locals = hf.interpreter_frame_method()->max_locals();
     intptr_t* frame_sp = caller.unextended_sp() - fsize;
     intptr_t* fp = frame_sp + (hf.fp() - heap_sp);
     DEBUG_ONLY(intptr_t* unextended_sp = fp + *hf.addr_at(frame::interpreter_frame_last_sp_offset);)
@@ -219,7 +224,7 @@ template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame&
     int fsize = FKind::size(hf);
     intptr_t* frame_sp = caller.unextended_sp() - fsize;
     if (bottom || caller.is_interpreted_frame()) {
-      int argsize = hf.compiled_frame_stack_argsize();
+      int argsize = FKind::stack_argsize(hf);
 
       fsize += argsize;
       frame_sp -= argsize;
@@ -234,11 +239,13 @@ template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame&
     intptr_t* fp;
     if (PreserveFramePointer) {
       // we need to recreate a "real" frame pointer, pointing into the stack
-      fp = frame_sp + FKind::size(hf) - 2;
+      fp = frame_sp + FKind::size(hf) - frame::sender_sp_offset;
     } else {
-      fp = FKind::stub
-        ? frame_sp + fsize - 2 // this value is used for the safepoint stub
-        : *(intptr_t**)(hf.sp() - 2); // we need to re-read fp because it may be an oop and we might have fixed the frame.
+      fp = FKind::stub || FKind::native
+        // fp always points to the address above the pushed return pc. We need correct address.
+        ? frame_sp + fsize - frame::sender_sp_offset
+        // we need to re-read fp because it may be an oop and we might have fixed the frame.
+        : *(intptr_t**)(hf.sp() - 2);
     }
     return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); // TODO PERF : this computes deopt state; is it necessary?
   }
@@ -261,6 +268,22 @@ inline void ThawBase::patch_pd(frame& f, const frame& caller) {
   patch_callee_link(caller, caller.fp());
 }
 
+inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {
+  intptr_t* fp = caller_sp - frame::sender_sp_offset;
+  patch_callee_link(f, fp);
+}
+
+inline intptr_t* ThawBase::push_cleanup_continuation() {
+  frame enterSpecial = new_entry_frame();
+  intptr_t* sp = enterSpecial.sp();
+
+  sp[-1] = (intptr_t)ContinuationEntry::cleanup_pc();
+  sp[-2] = (intptr_t)enterSpecial.fp();
+
+  log_develop_trace(continuations, preempt)("push_cleanup_continuation initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT, p2i(sp + 2 * frame::metadata_words), p2i(sp));
+  return sp;
+}
+
 inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) {
   // Make sure that last_sp is kept relativized.
   assert((intptr_t*)f.at_relative(frame::interpreter_frame_last_sp_offset) == f.unextended_sp(), "");