diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp index a53d83296459d..7869bbd472eb2 100644 --- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp @@ -343,7 +343,7 @@ class Address { enum mode { no_mode, base_plus_offset, pre, post, post_reg, base_plus_offset_reg, literal }; - + Register _obj_start = noreg; // Shift and extend for base reg + reg offset addressing class extend { int _option, _shift; @@ -464,7 +464,7 @@ class Address { } } - Address(const Address& a) : _mode(a._mode) { copy_data(a); } + Address(const Address& a) : _obj_start(a._obj_start), _mode(a._mode) { copy_data(a); } // Verify the value is trivially destructible regardless of mode, so our // destructor can also be trivial, and so our assignment operator doesn't @@ -474,6 +474,7 @@ class Address { Address& operator=(const Address& a) { _mode = a._mode; + _obj_start = a._obj_start; copy_data(a); return *this; } diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp index a847289e3ab2f..ccf0d10d34501 100644 --- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp @@ -174,7 +174,11 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i if (UseTLAB) { tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); } else { - b(slow_case); + if (UseThirdPartyHeap) { + eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); + } else { + b(slow_case); + } } } diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp index bfe9df8685001..75c6c667b011c 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp @@ -47,7 +47,7 @@ class BarrierSetAssembler: public CHeapObj { virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs) {} virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, - Register start, Register count, Register tmp, RegSet saved_regs) {} + Register src, Register dst, Register count, Register tmp, RegSet saved_regs) {} virtual void copy_load_at(MacroAssembler* masm, DecoratorSet decorators, @@ -111,6 +111,17 @@ class BarrierSetAssembler: public CHeapObj { Label& slow_case // continuation point if fast allocation fails ); + virtual void eden_allocate(MacroAssembler* masm, + Register obj, // result: pointer to object after successful allocation + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register t1, // temp register + Register t2, // temp register + Label& slow_case // continuation point if fast allocation fails + ) { + assert(false, "To be overriden by MMTk"); + } + virtual void barrier_stubs_init() {} virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::stw_instruction_and_data_patch; } diff --git a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp index a50152d244fba..96ffa3a2ab18b 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp @@ -37,10 +37,10 @@ void ModRefBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decorat } void ModRefBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, - Register start, Register count, Register tmp, + Register src, Register dst, Register count, Register tmp, RegSet saved_regs) { if (is_oop) { - gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs); + gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp, saved_regs); } } diff --git a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp index 22f98441f4ea6..d8b5383061b51 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp @@ -46,7 +46,7 @@ class ModRefBarrierSetAssembler: public BarrierSetAssembler { virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs); virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, - Register start, Register count, Register tmp, RegSet saved_regs); + Register src, Register dst, Register count, Register tmp, RegSet saved_regs); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3); }; diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 3be34786f19c3..be051fec11795 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -4782,6 +4782,16 @@ void MacroAssembler::tlab_allocate(Register obj, bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); } +void MacroAssembler::eden_allocate(Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register t1, + Register t2, + Label& slow_case) { + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); +} + void MacroAssembler::verify_tlab() { #ifdef ASSERT if (UseTLAB && VerifyOops) { diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index 6b45be8ce43ad..f61755d748fd0 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -933,6 +933,15 @@ class MacroAssembler: public Assembler { ); void verify_tlab(); + void eden_allocate( + Register obj, // result: pointer to object after successful allocation + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register t1, // temp register + Register t2, // temp register + Label& slow_case // continuation point if fast allocation fails + ); + // interface method calling void lookup_interface_method(Register recv_klass, Register intf_klass, diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 24930367319c9..e33df91118a20 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -1514,7 +1514,7 @@ class StubGenerator: public StubCodeGenerator { verify_oop_array(size, d, count, r16); } - bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet()); + bs->arraycopy_epilogue(_masm, decorators, is_oop, s, d, count, rscratch1, RegSet()); __ leave(); __ mov(r0, zr); // return 0 @@ -1583,7 +1583,7 @@ class StubGenerator: public StubCodeGenerator { if (VerifyOops) verify_oop_array(size, d, count, r16); } - bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet()); + bs->arraycopy_epilogue(_masm, decorators, is_oop, s, d, count, rscratch1, RegSet()); __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); @@ -1865,6 +1865,7 @@ class StubGenerator: public StubCodeGenerator { const Register copied_oop = r22; // actual oop copied const Register count_save = r21; // orig elementscount const Register start_to = r20; // destination array start address + const Register start_from = r23; // source array start address const Register r19_klass = r19; // oop._klass // Registers used as gc temps (r5, r6, r7 are save-on-call) @@ -1904,7 +1905,7 @@ class StubGenerator: public StubCodeGenerator { // Empty array: Nothing to do. __ cbz(count, L_done); - __ push(RegSet::of(r19, r20, r21, r22), sp); + __ push(RegSet::of(r19, r20, r21, r22, r23), sp); #ifdef ASSERT BLOCK_COMMENT("assert consistent ckoff/ckval"); @@ -1935,6 +1936,7 @@ class StubGenerator: public StubCodeGenerator { // Copy from low to high addresses __ mov(start_to, to); // Save destination array start address + __ mov(start_from, from); __ b(L_load_element); // ======== begin loop ======== @@ -1975,10 +1977,10 @@ class StubGenerator: public StubCodeGenerator { __ br(Assembler::EQ, L_done_pop); __ BIND(L_do_card_marks); - bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, rscratch1, wb_post_saved_regs); + bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, start_from, count_save, rscratch1, wb_post_saved_regs); __ bind(L_done_pop); - __ pop(RegSet::of(r19, r20, r21, r22), sp); + __ pop(RegSet::of(r19, r20, r21, r22, r23), sp); inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); __ bind(L_done); diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index 46c649d77b910..2238c1fb634c7 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -1105,6 +1105,7 @@ void TemplateTable::aastore() { __ ldr(r3, at_tos_p2()); // array Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop)); + element_address._obj_start = r3; index_check(r3, r2); // kills r1 __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); @@ -2724,6 +2725,7 @@ void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteContr // field address const Address field(obj, off); + field._obj_start = obj; Label notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj, notDouble; @@ -2992,7 +2994,8 @@ void TemplateTable::fast_storefield(TosState state) pop_and_check_object(r2); // field address - const Address field(r2, r1); + Address field(r2, r1); + field._obj_start = r2; // access field switch (bytecode()) { @@ -3558,8 +3561,12 @@ void TemplateTable::_new() { // // Go to slow path. - if (UseTLAB) { - __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case); + if (UseTLAB || UseThirdPartyHeap) { + if (UseTLAB) { + __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case); + } else { + __ eden_allocate(r0, r3, 0, noreg, r1, slow_case); + } if (ZeroTLAB) { // the fields have been already cleared