Skip to content

Commit f64a15d

Browse files
rkennkerkennke
committed
8254315: Shenandoah: Concurrent weak reference processing
Reviewed-by: zgu, shade
1 parent 83f3cf4 commit f64a15d

File tree

55 files changed

+1864
-802
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+1864
-802
lines changed

‎src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
109109
__ xchg(access.resolved_addr(), value_opr, result, tmp);
110110

111111
if (access.is_oop()) {
112-
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), false);
112+
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), ShenandoahBarrierSet::AccessKind::NORMAL);
113113
LIR_Opr tmp = gen->new_register(type);
114114
__ move(result, tmp);
115115
result = tmp;

‎src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp‎

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssemb
225225
}
226226
}
227227

228-
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, bool weak) {
228+
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind) {
229229
assert(ShenandoahLoadRefBarrier, "Should be enabled");
230230
assert(dst != rscratch2, "need rscratch2");
231231
assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
@@ -252,22 +252,34 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
252252
__ mov(r0, dst);
253253

254254
// Test for in-cset
255-
if (!weak) {
255+
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
256256
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
257257
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
258258
__ ldrb(rscratch2, Address(rscratch2, rscratch1));
259259
__ tbz(rscratch2, 0, not_cset);
260260
}
261261

262262
__ push_call_clobbered_registers();
263-
if (weak) {
264-
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
265-
} else {
266-
if (UseCompressedOops) {
267-
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
268-
} else {
269-
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
270-
}
263+
switch (kind) {
264+
case ShenandoahBarrierSet::AccessKind::NORMAL:
265+
if (UseCompressedOops) {
266+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
267+
} else {
268+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
269+
}
270+
break;
271+
case ShenandoahBarrierSet::AccessKind::WEAK:
272+
if (UseCompressedOops) {
273+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
274+
} else {
275+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
276+
}
277+
break;
278+
case ShenandoahBarrierSet::AccessKind::NATIVE:
279+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
280+
break;
281+
default:
282+
ShouldNotReachHere();
271283
}
272284
__ blr(lr);
273285
__ mov(rscratch1, r0);
@@ -326,8 +338,8 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
326338

327339
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
328340

329-
bool weak = ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type);
330-
load_reference_barrier(masm, dst, src, weak);
341+
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
342+
load_reference_barrier(masm, dst, src, kind);
331343

332344
if (dst != result_dst) {
333345
__ mov(result_dst, dst);
@@ -641,10 +653,18 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
641653
__ bind(slow_path);
642654
ce->store_parameter(res, 0);
643655
ce->store_parameter(addr, 1);
644-
if (stub->is_weak()) {
645-
__ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
646-
} else {
647-
__ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
656+
switch (stub->kind()) {
657+
case ShenandoahBarrierSet::AccessKind::NORMAL:
658+
__ far_call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin()));
659+
break;
660+
case ShenandoahBarrierSet::AccessKind::WEAK:
661+
__ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
662+
break;
663+
case ShenandoahBarrierSet::AccessKind::NATIVE:
664+
__ far_call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin()));
665+
break;
666+
default:
667+
ShouldNotReachHere();
648668
}
649669

650670
__ b(*stub->continuation());
@@ -700,19 +720,33 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss
700720
__ epilogue();
701721
}
702722

703-
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak) {
723+
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) {
704724
__ prologue("shenandoah_load_reference_barrier", false);
705725
// arg0 : object to be resolved
706726

707727
__ push_call_clobbered_registers();
708728
__ load_parameter(0, r0);
709729
__ load_parameter(1, r1);
710-
if (is_weak) {
711-
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
712-
} else if (UseCompressedOops) {
713-
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
714-
} else {
715-
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
730+
switch (kind) {
731+
case ShenandoahBarrierSet::AccessKind::NORMAL:
732+
if (UseCompressedOops) {
733+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
734+
} else {
735+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
736+
}
737+
break;
738+
case ShenandoahBarrierSet::AccessKind::WEAK:
739+
if (UseCompressedOops) {
740+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
741+
} else {
742+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
743+
}
744+
break;
745+
case ShenandoahBarrierSet::AccessKind::NATIVE:
746+
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
747+
break;
748+
default:
749+
ShouldNotReachHere();
716750
}
717751
__ blr(lr);
718752
__ mov(rscratch1, r0);

‎src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
#include "asm/macroAssembler.hpp"
2929
#include "gc/shared/barrierSetAssembler.hpp"
30+
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
3031
#ifdef COMPILER1
3132
class LIR_Assembler;
3233
class ShenandoahPreBarrierStub;
@@ -55,7 +56,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
5556

5657
void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
5758
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
58-
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, bool weak);
59+
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind);
5960

6061
public:
6162

@@ -65,7 +66,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
6566
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
6667
void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
6768
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
68-
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak);
69+
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind);
6970
#endif
7071

7172
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,

‎src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
111111
__ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
112112

113113
if (access.is_oop()) {
114-
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), false);
114+
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), ShenandoahBarrierSet::AccessKind::NORMAL);
115115
LIR_Opr tmp = gen->new_register(type);
116116
__ move(result, tmp);
117117
result = tmp;

‎src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp‎

Lines changed: 68 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
268268
__ bind(done);
269269
}
270270

271-
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, bool weak) {
271+
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind) {
272272
assert(ShenandoahLoadRefBarrier, "Should be enabled");
273273

274274
Label heap_stable, not_cset;
@@ -292,7 +292,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
292292
__ jcc(Assembler::zero, heap_stable);
293293

294294
Register tmp1 = noreg;
295-
if (!weak) {
295+
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
296296
// Test for object in cset
297297
// Allocate tmp-reg.
298298
for (int i = 0; i < 8; i++) {
@@ -338,14 +338,26 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
338338
__ lea(tmp2, src);
339339

340340
save_xmm_registers(masm);
341-
if (weak) {
342-
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), dst, tmp2);
343-
} else {
344-
if (UseCompressedOops) {
345-
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), dst, tmp2);
346-
} else {
347-
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), dst, tmp2);
348-
}
341+
switch (kind) {
342+
case ShenandoahBarrierSet::AccessKind::NORMAL:
343+
if (UseCompressedOops) {
344+
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), dst, tmp2);
345+
} else {
346+
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), dst, tmp2);
347+
}
348+
break;
349+
case ShenandoahBarrierSet::AccessKind::WEAK:
350+
if (UseCompressedOops) {
351+
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), dst, tmp2);
352+
} else {
353+
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), dst, tmp2);
354+
}
355+
break;
356+
case ShenandoahBarrierSet::AccessKind::NATIVE:
357+
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), dst, tmp2);
358+
break;
359+
default:
360+
ShouldNotReachHere();
349361
}
350362
restore_xmm_registers(masm);
351363

@@ -370,7 +382,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
370382

371383
__ bind(not_cset);
372384

373-
if (!weak) {
385+
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
374386
__ pop(tmp1);
375387
}
376388

@@ -467,8 +479,8 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
467479

468480
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
469481

470-
bool weak = ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type);
471-
load_reference_barrier(masm, dst, src, weak);
482+
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
483+
load_reference_barrier(masm, dst, src, kind);
472484

473485
// Move loaded oop to final destination
474486
if (dst != result_dst) {
@@ -818,10 +830,18 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
818830
__ bind(slow_path);
819831
ce->store_parameter(res, 0);
820832
ce->store_parameter(addr, 1);
821-
if (stub->is_weak()) {
822-
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
823-
} else {
824-
__ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
833+
switch (stub->kind()) {
834+
case ShenandoahBarrierSet::AccessKind::NORMAL:
835+
__ call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin()));
836+
break;
837+
case ShenandoahBarrierSet::AccessKind::WEAK:
838+
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
839+
break;
840+
case ShenandoahBarrierSet::AccessKind::NATIVE:
841+
__ call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin()));
842+
break;
843+
default:
844+
ShouldNotReachHere();
825845
}
826846
__ jmp(*stub->continuation());
827847
}
@@ -886,7 +906,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss
886906
__ epilogue();
887907
}
888908

889-
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak) {
909+
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) {
890910
__ prologue("shenandoah_load_reference_barrier", false);
891911
// arg0 : object to be resolved
892912

@@ -895,20 +915,40 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s
895915
#ifdef _LP64
896916
__ load_parameter(0, c_rarg0);
897917
__ load_parameter(1, c_rarg1);
898-
if (is_weak) {
899-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
900-
} else if (UseCompressedOops) {
901-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
902-
} else {
903-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
918+
switch (kind) {
919+
case ShenandoahBarrierSet::AccessKind::NORMAL:
920+
if (UseCompressedOops) {
921+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
922+
} else {
923+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
924+
}
925+
break;
926+
case ShenandoahBarrierSet::AccessKind::WEAK:
927+
if (UseCompressedOops) {
928+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
929+
} else {
930+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
931+
}
932+
break;
933+
case ShenandoahBarrierSet::AccessKind::NATIVE:
934+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
935+
break;
936+
default:
937+
ShouldNotReachHere();
904938
}
905939
#else
906940
__ load_parameter(0, rax);
907941
__ load_parameter(1, rbx);
908-
if (is_weak) {
909-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx);
910-
} else {
911-
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
942+
switch (kind) {
943+
case ShenandoahBarrierSet::AccessKind::NORMAL:
944+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
945+
break;
946+
case ShenandoahBarrierSet::AccessKind::WEAK:
947+
case ShenandoahBarrierSet::AccessKind::NATIVE:
948+
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx);
949+
break;
950+
default:
951+
ShouldNotReachHere();
912952
}
913953
#endif
914954

‎src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727

2828
#include "asm/macroAssembler.hpp"
2929
#include "gc/shared/barrierSetAssembler.hpp"
30+
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
31+
3032
#ifdef COMPILER1
3133
class LIR_Assembler;
3234
class ShenandoahPreBarrierStub;
@@ -62,10 +64,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
6264
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
6365
void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
6466
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
65-
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak);
67+
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind);
6668
#endif
6769

68-
void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, bool weak);
70+
void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind);
6971

7072
void cmpxchg_oop(MacroAssembler* masm,
7173
Register res, Address addr, Register oldval, Register newval,

0 commit comments

Comments
 (0)