Skip to content

Commit 3036866

Browse files
xmas92xmas92
committed
8367972: ZGC: Reduce ZBarrierSet includes
Reviewed-by: stefank, eosterlund
1 parent 3183a13 commit 3036866

File tree

10 files changed

+137
-73
lines changed

10 files changed

+137
-73
lines changed

‎src/hotspot/share/gc/z/zBarrierSet.cpp‎

Lines changed: 93 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
* questions.
2222
*/
2323

24+
#include "gc/z/zAddress.inline.hpp"
25+
#include "gc/z/zBarrier.inline.hpp"
2426
#include "gc/z/zBarrierSet.hpp"
2527
#include "gc/z/zBarrierSetAssembler.hpp"
2628
#include "gc/z/zBarrierSetNMethod.hpp"
@@ -30,6 +32,7 @@
3032
#include "gc/z/zHeap.inline.hpp"
3133
#include "gc/z/zStackWatermark.hpp"
3234
#include "gc/z/zThreadLocalData.hpp"
35+
#include "runtime/atomicAccess.hpp"
3336
#include "runtime/deoptimization.hpp"
3437
#include "runtime/frame.inline.hpp"
3538
#include "runtime/javaThread.hpp"
@@ -46,6 +49,96 @@
4649
class ZBarrierSetC1;
4750
class ZBarrierSetC2;
4851

52+
class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
53+
public:
54+
virtual void do_oop(oop* p_) {
55+
volatile zpointer* const p = (volatile zpointer*)p_;
56+
const zpointer ptr = ZBarrier::load_atomic(p);
57+
const zaddress addr = ZPointer::uncolor(ptr);
58+
AtomicAccess::store(p, ZAddress::store_good(addr));
59+
}
60+
61+
virtual void do_oop(narrowOop* p) {
62+
ShouldNotReachHere();
63+
}
64+
};
65+
66+
class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
67+
public:
68+
virtual void do_oop(oop* p) {
69+
ZBarrier::load_barrier_on_oop_field((zpointer*)p);
70+
}
71+
72+
virtual void do_oop(narrowOop* p) {
73+
ShouldNotReachHere();
74+
}
75+
};
76+
77+
void ZBarrierSet::load_barrier_all(oop src, size_t size) {
78+
check_is_valid_zaddress(src);
79+
80+
ZLoadBarrierOopClosure cl;
81+
ZIterator::oop_iterate(src, &cl);
82+
}
83+
84+
void ZBarrierSet::color_store_good_all(oop dst, size_t size) {
85+
check_is_valid_zaddress(dst);
86+
assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
87+
88+
ZColorStoreGoodOopClosure cl_sg;
89+
ZIterator::oop_iterate(dst, &cl_sg);
90+
}
91+
92+
zaddress ZBarrierSet::load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o) {
93+
return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
94+
}
95+
96+
zaddress ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
97+
return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
98+
}
99+
100+
zaddress ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
101+
return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
102+
}
103+
104+
zaddress ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
105+
return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
106+
}
107+
108+
zaddress ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
109+
return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
110+
}
111+
112+
void ZBarrierSet::store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal) {
113+
ZBarrier::store_barrier_on_heap_oop_field(p, heal);
114+
}
115+
116+
void ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p) {
117+
ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
118+
}
119+
120+
void ZBarrierSet::store_barrier_on_native_oop_field(volatile zpointer* p, bool heal) {
121+
ZBarrier::store_barrier_on_native_oop_field(p, heal);
122+
}
123+
124+
zaddress ZBarrierSet::load_barrier_on_oop_field(volatile zpointer* p) {
125+
return ZBarrier::load_barrier_on_oop_field(p);
126+
}
127+
128+
void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj) {
129+
volatile zpointer* src = (volatile zpointer*)src_obj->base();
130+
volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
131+
const int length = src_obj->length();
132+
133+
for (const volatile zpointer* const end = src + length; src < end; src++, dst++) {
134+
zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
135+
// We avoid healing here because the store below colors the pointer store good,
136+
// hence avoiding the cost of a CAS.
137+
ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */);
138+
AtomicAccess::store(dst, ZAddress::store_good(elem));
139+
}
140+
}
141+
49142
ZBarrierSet::ZBarrierSet()
50143
: BarrierSet(make_barrier_set_assembler<ZBarrierSetAssembler>(),
51144
make_barrier_set_c1<ZBarrierSetC1>(),
@@ -153,20 +246,6 @@ void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
153246
deoptimize_allocation(thread);
154247
}
155248

156-
void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj) {
157-
volatile zpointer* src = (volatile zpointer*)src_obj->base();
158-
volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
159-
const int length = src_obj->length();
160-
161-
for (const volatile zpointer* const end = src + length; src < end; src++, dst++) {
162-
zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
163-
// We avoid healing here because the store below colors the pointer store good,
164-
// hence avoiding the cost of a CAS.
165-
ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */);
166-
AtomicAccess::store(dst, ZAddress::store_good(elem));
167-
}
168-
}
169-
170249
void ZBarrierSet::print_on(outputStream* st) const {
171250
st->print_cr("ZBarrierSet");
172251
}

‎src/hotspot/share/gc/z/zBarrierSet.hpp‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,29 @@ class ZBarrierSet : public BarrierSet {
3333
private:
3434
static zpointer store_good(oop obj);
3535

36+
static void load_barrier_all(oop src, size_t size);
37+
static void color_store_good_all(oop dst, size_t size);
38+
39+
static zaddress load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o);
40+
static zaddress no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
41+
static zaddress no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
42+
static zaddress load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
43+
static zaddress load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
44+
45+
static void store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal);
46+
static void no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p);
47+
static void store_barrier_on_native_oop_field(volatile zpointer* p, bool heal);
48+
49+
static zaddress load_barrier_on_oop_field(volatile zpointer* p);
50+
51+
static void clone_obj_array(objArrayOop src, objArrayOop dst);
52+
3653
public:
3754
ZBarrierSet();
3855

3956
static ZBarrierSetAssembler* assembler();
4057
static bool barrier_needed(DecoratorSet decorators, BasicType type);
4158

42-
static void clone_obj_array(objArrayOop src, objArrayOop dst);
43-
4459
virtual void on_thread_create(Thread* thread);
4560
virtual void on_thread_destroy(Thread* thread);
4661
virtual void on_thread_attach(Thread* thread);

‎src/hotspot/share/gc/z/zBarrierSet.inline.hpp‎

Lines changed: 22 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,9 @@
2828

2929
#include "gc/shared/accessBarrierSupport.inline.hpp"
3030
#include "gc/z/zAddress.inline.hpp"
31-
#include "gc/z/zBarrier.inline.hpp"
32-
#include "gc/z/zIterator.inline.hpp"
31+
#include "gc/z/zHeap.hpp"
3332
#include "gc/z/zNMethod.hpp"
34-
#include "memory/iterator.inline.hpp"
33+
#include "oops/objArrayOop.hpp"
3534
#include "utilities/debug.hpp"
3635

3736
template <DecoratorSet decorators, typename BarrierSetT>
@@ -68,21 +67,21 @@ inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrie
6867
if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
6968
if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
7069
// Load barriers on strong oop refs don't keep objects alive
71-
return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
70+
return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
7271
} else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
73-
return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
72+
return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
7473
} else {
7574
assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
76-
return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
75+
return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
7776
}
7877
} else {
7978
if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
80-
return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
79+
return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
8180
} else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
82-
return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
81+
return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
8382
} else {
8483
assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
85-
return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
84+
return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
8685
}
8786
}
8887
}
@@ -97,21 +96,21 @@ inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrie
9796
if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
9897
if (decorators_known_strength & ON_STRONG_OOP_REF) {
9998
// Load barriers on strong oop refs don't keep objects alive
100-
return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
99+
return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
101100
} else if (decorators_known_strength & ON_WEAK_OOP_REF) {
102-
return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
101+
return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
103102
} else {
104103
assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
105-
return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
104+
return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
106105
}
107106
} else {
108107
if (decorators_known_strength & ON_STRONG_OOP_REF) {
109-
return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
108+
return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
110109
} else if (decorators_known_strength & ON_WEAK_OOP_REF) {
111-
return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
110+
return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
112111
} else {
113112
assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
114-
return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
113+
return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
115114
}
116115
}
117116
}
@@ -126,7 +125,7 @@ inline zpointer ZBarrierSet::store_good(oop obj) {
126125
template <DecoratorSet decorators, typename BarrierSetT>
127126
inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
128127
if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
129-
ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */);
128+
ZBarrierSet::store_barrier_on_heap_oop_field(p, true /* heal */);
130129
} else {
131130
assert(false, "Should not be used on uninitialized memory");
132131
}
@@ -135,21 +134,21 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_h
135134
template <DecoratorSet decorators, typename BarrierSetT>
136135
inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
137136
if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
138-
ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
137+
ZBarrierSet::store_barrier_on_heap_oop_field(p, false /* heal */);
139138
}
140139
}
141140

142141
template <DecoratorSet decorators, typename BarrierSetT>
143142
inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
144143
if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
145-
ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
144+
ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(p);
146145
}
147146
}
148147

149148
template <DecoratorSet decorators, typename BarrierSetT>
150149
inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
151150
if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
152-
ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */);
151+
ZBarrierSet::store_barrier_on_native_oop_field(p, true /* heal */);
153152
} else {
154153
assert(false, "Should not be used on uninitialized memory");
155154
}
@@ -158,7 +157,7 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_n
158157
template <DecoratorSet decorators, typename BarrierSetT>
159158
inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
160159
if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
161-
ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */);
160+
ZBarrierSet::store_barrier_on_native_oop_field(p, false /* heal */);
162161
}
163162
}
164163

@@ -325,7 +324,7 @@ template <DecoratorSet decorators, typename BarrierSetT>
325324
inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
326325
store_barrier_heap_without_healing(dst);
327326

328-
return ZBarrier::load_barrier_on_oop_field(src);
327+
return ZBarrierSet::load_barrier_on_oop_field(src);
329328
}
330329

331330
template <DecoratorSet decorators, typename BarrierSetT>
@@ -402,31 +401,6 @@ inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_i
402401
return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
403402
}
404403

405-
class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
406-
public:
407-
virtual void do_oop(oop* p_) {
408-
volatile zpointer* const p = (volatile zpointer*)p_;
409-
const zpointer ptr = ZBarrier::load_atomic(p);
410-
const zaddress addr = ZPointer::uncolor(ptr);
411-
AtomicAccess::store(p, ZAddress::store_good(addr));
412-
}
413-
414-
virtual void do_oop(narrowOop* p) {
415-
ShouldNotReachHere();
416-
}
417-
};
418-
419-
class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
420-
public:
421-
virtual void do_oop(oop* p) {
422-
ZBarrier::load_barrier_on_oop_field((zpointer*)p);
423-
}
424-
425-
virtual void do_oop(narrowOop* p) {
426-
ShouldNotReachHere();
427-
}
428-
};
429-
430404
template <DecoratorSet decorators, typename BarrierSetT>
431405
inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
432406
check_is_valid_zaddress(src);
@@ -443,17 +417,13 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(o
443417
}
444418

445419
// Fix the oops
446-
ZLoadBarrierOopClosure cl;
447-
ZIterator::oop_iterate(src, &cl);
420+
ZBarrierSet::load_barrier_all(src, size);
448421

449422
// Clone the object
450423
Raw::clone_in_heap(src, dst, size);
451424

452-
assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
453-
454425
// Color store good before handing out
455-
ZColorStoreGoodOopClosure cl_sg;
456-
ZIterator::oop_iterate(dst, &cl_sg);
426+
ZBarrierSet::color_store_good_all(dst, size);
457427
}
458428

459429
//

‎src/hotspot/share/gc/z/zObjArrayAllocator.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
* questions.
2222
*/
2323

24+
#include "gc/z/zGeneration.inline.hpp"
2425
#include "gc/z/zObjArrayAllocator.hpp"
2526
#include "gc/z/zThreadLocalData.hpp"
2627
#include "gc/z/zUtils.inline.hpp"

‎src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "gc/z/zAddress.inline.hpp"
2626
#include "gc/z/zArray.inline.hpp"
2727
#include "gc/z/zGlobals.hpp"
28+
#include "gc/z/zGranuleMap.inline.hpp"
2829
#include "gc/z/zLargePages.inline.hpp"
2930
#include "gc/z/zList.inline.hpp"
3031
#include "gc/z/zNMT.hpp"

‎src/hotspot/share/gc/z/zRangeRegistry.inline.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "gc/z/zRangeRegistry.hpp"
2828

2929
#include "gc/z/zAddress.inline.hpp"
30+
#include "gc/z/zArray.inline.hpp"
3031
#include "gc/z/zList.inline.hpp"
3132
#include "gc/z/zLock.inline.hpp"
3233

‎src/hotspot/share/precompiled/precompiled.hpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
4646
#endif
4747
#if INCLUDE_ZGC
48-
#include "gc/z/zBarrier.inline.hpp"
4948
#include "gc/z/zGeneration.inline.hpp"
5049
#include "gc/z/zHeap.inline.hpp"
5150
#endif

‎src/hotspot/share/prims/whitebox.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@
120120
#endif // INCLUDE_SERIALGC
121121
#if INCLUDE_ZGC
122122
#include "gc/z/zAddress.inline.hpp"
123+
#include "gc/z/zHeap.inline.hpp"
123124
#endif // INCLUDE_ZGC
124125
#if INCLUDE_JVMCI
125126
#include "jvmci/jvmciEnv.hpp"

‎src/hotspot/share/runtime/stackValue.cpp‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,6 @@
3030
#include "runtime/globals.hpp"
3131
#include "runtime/handles.inline.hpp"
3232
#include "runtime/stackValue.hpp"
33-
#if INCLUDE_ZGC
34-
#include "gc/z/zBarrier.inline.hpp"
35-
#endif
3633
#if INCLUDE_SHENANDOAHGC
3734
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
3835
#endif

‎test/hotspot/gtest/runtime/test_os_windows.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#include "logging/log.hpp"
2828
#include "runtime/flags/flagSetting.hpp"
2929
#include "runtime/globals_extension.hpp"
30-
#include "runtime/os.hpp"
30+
#include "runtime/os.inline.hpp"
3131
#include "concurrentTestRunner.inline.hpp"
3232
#include "unittest.hpp"
3333

0 commit comments

Comments
 (0)