WIP: AOSCOS: LOONGARCH: Port of JEP 450: Compact Object Headers (Experimental)

Fixes: 44ec501 ("8305895: Implement JEP 450: Compact Object Headers (Experimental)")
Fixes: ff12ff5 ("8340453: C2: Improve encoding of LoadNKlass for compact headers")
Follow-up: 44ec501 ("8305895: Implement JEP 450: Compact Object Headers (Experimental)")
Follow-up: ff12ff5 ("8340453: C2: Improve encoding of LoadNKlass for compact headers")
Link: openjdk#20677
Link: openjdk#22203
Signed-off-by: Bingwu Zhang <[email protected]>

Author: xtexx

src/hotspot/cpu/loongarch/c1_LIRAssembler_loongarch_64.cpp

@@ -2516,8 +2516,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
 
   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
-  Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
-  Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
 
   // test for null
   if (flags & LIR_OpArrayCopy::src_null_check) {
@@ -2573,14 +2571,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     // We don't know the array types are compatible
     if (basic_type != T_OBJECT) {
       // Simple test for basic type arrays
-      if (UseCompressedClassPointers) {
-        __ ld_wu(tmp, src_klass_addr);
-        __ ld_wu(SCR1, dst_klass_addr);
-      } else {
-        __ ld_d(tmp, src_klass_addr);
-        __ ld_d(SCR1, dst_klass_addr);
-      }
-      __ bne_far(tmp, SCR1, *stub->entry());
+      __ cmp_klass(dst, src, tmp, SCR1, *stub->entry(), false, true);
     } else {
       // For object arrays, if src is a sub class of dst then we can
       // safely do the copy.
@@ -2716,26 +2707,10 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     }
 
     if (basic_type != T_OBJECT) {
-
-      if (UseCompressedClassPointers) {
-        __ ld_wu(SCR1, dst_klass_addr);
-      } else {
-        __ ld_d(SCR1, dst_klass_addr);
-      }
-      __ bne(tmp, SCR1, halt);
-      if (UseCompressedClassPointers) {
-        __ ld_wu(SCR1, src_klass_addr);
-      } else {
-        __ ld_d(SCR1, src_klass_addr);
-      }
-      __ beq(tmp, SCR1, known_ok);
+      __ cmp_klass_compressed(dst, tmp, SCR1, halt, false);
+      __ cmp_klass_compressed(src, tmp, SCR1, known_ok, true);
     } else {
-      if (UseCompressedClassPointers) {
-        __ ld_wu(SCR1, dst_klass_addr);
-      } else {
-        __ ld_d(SCR1, dst_klass_addr);
-      }
-      __ beq(tmp, SCR1, known_ok);
+      __ cmp_klass_compressed(dst, tmp, SCR1, known_ok, true);
       __ beq(src, dst, known_ok);
     }
     __ bind(halt);
@@ -2814,12 +2789,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
     add_debug_info_for_null_check_here(info);
   }
 
-  if (UseCompressedClassPointers) {
-    __ ld_wu(result, obj, oopDesc::klass_offset_in_bytes());
-    __ decode_klass_not_null(result);
-  } else {
-    __ ld_d(result, obj, oopDesc::klass_offset_in_bytes());
-  }
+  __ load_klass(result, obj);
 }
 
 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {

src/hotspot/cpu/loongarch/c1_MacroAssembler_loongarch.hpp

@@ -46,7 +46,7 @@ using MacroAssembler::null_check;
     Label&   slow_case          // continuation point if fast allocation fails
   );
 
-  void initialize_header(Register obj, Register klass, Register len, Register t1, Register t2);
+  void initialize_header(Register obj, Register klass, Register len, Register t1);
   void initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1, Register t2);
 
   // locking

src/hotspot/cpu/loongarch/c1_MacroAssembler_loongarch_64.cpp

@@ -33,6 +33,7 @@
 #include "oops/arrayOop.hpp"
 #include "oops/markWord.hpp"
 #include "runtime/basicLock.hpp"
+#include "runtime/globals.hpp"
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
@@ -150,17 +151,21 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes,
 }
 
 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len,
-                                          Register t1, Register t2) {
+                                          Register t1) {
   assert_different_registers(obj, klass, len);
-  // This assumes that all prototype bits fit in an int32_t
-  li(t1, (int32_t)(intptr_t)markWord::prototype().value());
-  st_d(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
-
-  if (UseCompressedClassPointers) { // Take care not to kill klass
-    encode_klass_not_null(t1, klass);
-    st_w(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
+  if (UseCompactObjectHeaders) {
+    ld_d(t1, Address(klass, Klass::prototype_header_offset()));
+    st_d(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
   } else {
-    st_d(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+    // This assumes that all prototype bits fit in an int32_t
+    li(t1, (int32_t)(intptr_t)markWord::prototype().value());
+    st_d(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
+    if (UseCompressedClassPointers) { // Take care not to kill klass
+      encode_klass_not_null(t1, klass);
+      st_w(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
+    } else {
+      st_d(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+    }
   }
 
   if (len->is_valid()) {
@@ -171,7 +176,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
       // Clear gap/first 4 bytes following the length field.
       st_w(R0, Address(obj, base_offset));
     }
-  } else if (UseCompressedClassPointers) {
+  } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
     store_klass_gap(obj, R0);
   }
 }
@@ -221,7 +226,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
          "con_size_in_bytes is not multiple of alignment");
   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
 
-  initialize_header(obj, klass, noreg, t1, t2);
+  initialize_header(obj, klass, noreg, t1);
 
   if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
      // clear rest of allocated space
@@ -272,7 +277,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
 
   try_allocate(obj, arr_size, 0, t1, t2, slow_case);
 
-  initialize_header(obj, klass, len, t1, t2);
+  initialize_header(obj, klass, len, t1);
 
   // Align-up to word boundary, because we clear the 4 bytes potentially
   // following the length field in initialize_header().

src/hotspot/cpu/loongarch/compressedKlass_loongarch.cpp

@@ -57,9 +57,9 @@ char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size
     result = reserve_address_space_for_zerobased_encoding(size, aslr);
   }
 
-  // Failing that, optimize for case (3) - a base with only bits set between [33-52)
+  // Failing that, optimize for case (3) - a base with only bits set between [32-52)
   if (result == nullptr) {
-    const uintptr_t from = nth_bit(32 + (optimize_for_zero_base ? LogKlassAlignmentInBytes : 0));
+    const uintptr_t from = nth_bit(32);
     constexpr uintptr_t to = nth_bit(52);
     constexpr size_t alignment = nth_bit(32);
     result = reserve_address_space_X(from, to, size, alignment, aslr);

src/hotspot/cpu/loongarch/loongarch_64.ad

@@ -4276,6 +4276,7 @@ instruct loadKlass(mRegP dst, memory mem) %{
 // Load narrow Klass Pointer
 instruct loadNKlass(mRegN dst, memory mem)
 %{
+  predicate(!UseCompactObjectHeaders);
   match(Set dst (LoadNKlass mem));
 
   ins_cost(125);
@@ -4288,6 +4289,25 @@ instruct loadNKlass(mRegN dst, memory mem)
   ins_pipe( ialu_load );
 %}
 
+instruct loadNKlassCompactHeaders(mRegN dst, memory mem)
+%{
+  predicate(UseCompactObjectHeaders);
+  match(Set dst (LoadNKlass mem));
+
+  ins_cost(125);
+  format %{
+    "ld_wu    $dst, $mem\t# compressed klass ptr, shifted\n\t"
+    "srli_d   $dst, $dst, markWord::klass_shift_at_offset"
+  %}
+
+  ins_encode %{
+    __ ld_wu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp));
+    __ srli_d(as_Register($dst$$reg), as_Register($dst$$reg), (unsigned) markWord::klass_shift_at_offset);
+  %}
+
+  ins_pipe( ialu_load );
+%}
+
 instruct loadN2PKlass(mRegP dst, memory mem)
 %{
   match(Set dst (DecodeNKlass (LoadNKlass mem)));

src/hotspot/cpu/loongarch/macroAssembler_loongarch.cpp

@@ -492,6 +492,49 @@ void MacroAssembler::wrap_label(Register r, Label &L, reg_addr_insn insn) {
   }
 }
 
+void MacroAssembler::cmp_klass(Register obj1, Register obj2, Register tmp1,
+                               Register tmp2, Label &L, bool equal, bool far) {
+  if (UseCompactObjectHeaders) {
+    load_narrow_klass_compact(tmp1, obj1);
+    load_narrow_klass_compact(tmp2, obj2);
+  } else if (UseCompressedClassPointers) {
+    ld_wu(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
+    ld_wu(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
+  } else {
+    ld_d(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
+    ld_d(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
+  }
+  if (equal) {
+    if (far) {
+      beq_far(tmp1, tmp2, L);
+    } else {
+      beq(tmp1, tmp2, L);
+    }
+  } else {
+    if (far) {
+      bne_far(tmp1, tmp2, L);
+    } else {
+      bne(tmp1, tmp2, L);
+    }
+  }
+}
+
+void MacroAssembler::cmp_klass_compressed(Register obj, Register klass,
+                                          Register tmp1, Label &L, bool equal) {
+  if (UseCompactObjectHeaders) {
+    load_narrow_klass_compact(tmp1, obj);
+  } else if (UseCompressedClassPointers) {
+    ld_wu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
+  } else {
+    ld_d(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
+  }
+  if (equal) {
+    beq(tmp1, klass, L);
+  } else {
+    bne(tmp1, klass, L);
+  }
+}
+
 // Move an oop into a register.
 void MacroAssembler::movoop(Register dst, jobject obj) {
   int oop_index;
@@ -610,7 +653,10 @@ address MacroAssembler::ic_call(address entry, jint method_index) {
 }
 
 int MacroAssembler::ic_check_size() {
-  return 4 * 5;
+  return 4 * (/* beq */ 1 + /* patchable jump(force patchable) */ 2 +
+              /* load klass */
+              (UseCompactObjectHeaders ? 3 /* load_narrow_klass_compact + 1 */
+                                       : 2 /* ld + ld */));
 }
 
 int MacroAssembler::ic_check(int end_alignment) {
@@ -626,7 +672,10 @@ int MacroAssembler::ic_check(int end_alignment) {
   align(end_alignment, offset() + ic_check_size());
   int uep_offset = offset();
 
-  if (UseCompressedClassPointers) {
+  if (UseCompactObjectHeaders) {
+    load_narrow_klass_compact(tmp1, receiver);
+    ld_wu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+  } else if (UseCompressedClassPointers) {
     ld_wu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
     ld_wu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
   } else {
@@ -2049,15 +2098,25 @@ void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod)
 
 // for UseCompressedOops Option
 void MacroAssembler::load_klass(Register dst, Register src) {
-  if(UseCompressedClassPointers){
+  if (UseCompactObjectHeaders) {
+    load_narrow_klass_compact(dst, src);
+    decode_klass_not_null(dst);
+  } else if (UseCompressedClassPointers) {
     ld_wu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
     decode_klass_not_null(dst);
   } else {
     ld_d(dst, src, oopDesc::klass_offset_in_bytes());
   }
 }
 
+void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
+  assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
+  ld_d(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+  srli_d(dst, dst, markWord::klass_shift);
+}
+
 void MacroAssembler::store_klass(Register dst, Register src) {
+  assert(!UseCompactObjectHeaders, "not with compact headers");
   if(UseCompressedClassPointers){
     encode_klass_not_null(src);
     st_w(src, dst, oopDesc::klass_offset_in_bytes());
@@ -2067,6 +2126,7 @@ void MacroAssembler::store_klass(Register dst, Register src) {
 }
 
 void MacroAssembler::store_klass_gap(Register dst, Register src) {
+  assert(!UseCompactObjectHeaders, "not with compact headers");
   if (UseCompressedClassPointers) {
     st_w(src, dst, oopDesc::klass_gap_offset_in_bytes());
   }
@@ -2350,8 +2410,7 @@ void MacroAssembler::encode_klass_not_null(Register r) {
     sub_d(r, r, AT);
   }
   if (CompressedKlassPointers::shift() != 0) {
-    assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-    srli_d(r, r, LogKlassAlignmentInBytes);
+    srli_d(r, r, CompressedKlassPointers::shift());
   }
 }
 
@@ -2368,13 +2427,11 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
       li(dst, (int64_t)CompressedKlassPointers::base());
       sub_d(dst, src, dst);
       if (CompressedKlassPointers::shift() != 0) {
-        assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-        srli_d(dst, dst, LogKlassAlignmentInBytes);
+        srli_d(dst, dst, CompressedKlassPointers::shift());
       }
     } else {
       if (CompressedKlassPointers::shift() != 0) {
-        assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-        srli_d(dst, src, LogKlassAlignmentInBytes);
+        srli_d(dst, src, CompressedKlassPointers::shift());
       } else {
         move(dst, src);
       }
@@ -2397,15 +2454,16 @@ void MacroAssembler::decode_klass_not_null(Register r) {
         add_d(r, r, AT);
       }
     } else {
-      assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-      assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
+      assert(CompressedKlassPointers::shift() == Address::times_8,
+             "klass not aligned on 64bits?");
       li(AT, (int64_t)CompressedKlassPointers::base());
       alsl_d(r, r, AT, Address::times_8 - 1);
     }
   } else {
     if (CompressedKlassPointers::shift() != 0) {
-      assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-      slli_d(r, r, LogKlassAlignmentInBytes);
+      assert(CompressedKlassPointers::shift() == Address::times_8,
+             "klass not aligned on 64bits?");
+      slli_d(r, r, CompressedKlassPointers::shift());
     }
   }
 }
@@ -2428,15 +2486,16 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
           add_d(dst, dst, src);
         }
       } else {
-        assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-        assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
+        assert(CompressedKlassPointers::shift() == Address::times_8,
+               "klass not aligned on 64bits?");
         li(dst, (int64_t)CompressedKlassPointers::base());
         alsl_d(dst, src, dst, Address::times_8 - 1);
       }
     } else {
       if (CompressedKlassPointers::shift() != 0) {
-        assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-        slli_d(dst, src, LogKlassAlignmentInBytes);
+        assert(CompressedKlassPointers::shift() == Address::times_8,
+               "klass not aligned on 64bits?");
+        slli_d(dst, src, CompressedKlassPointers::shift());
       } else {
         move(dst, src);
       }

src/hotspot/cpu/loongarch/macroAssembler_loongarch.hpp

@@ -277,7 +277,15 @@ class MacroAssembler: public Assembler {
 
   // oop manipulations
   void load_klass(Register dst, Register src);
+  // load narrow klass ptr from obj ptr (src) to dst, 2 insns
+  void load_narrow_klass_compact(Register dst, Register src);
   void store_klass(Register dst, Register src);
+  // compare if obj1 and obj2 has the same (or diff) klass ptr
+  // and jump to L.
+  void cmp_klass(Register obj1, Register obj2, Register tmp1, Register tmp2,
+                 Label &L, bool equal, bool far);
+  void cmp_klass_compressed(Register obj, Register klass, Register tmp1,
+                 Label &L, bool equal);
 
   void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
                       Register tmp1, Register tmp2);
@@ -562,6 +570,7 @@ class MacroAssembler: public Assembler {
   static bool reachable_from_branch_short(jlong offs);
 
   void patchable_jump_far(Register ra, jlong offs);
+  // 2 insns if force_patchable, 1 insn if not
   void patchable_jump(address target, bool force_patchable = false);
   void patchable_call(address target, address call_size = 0);