8305896: Alternative full GC forwarding

src/hotspot/share/gc/g1/g1Arguments.cpp

@@ -35,6 +35,7 @@
 #include "gc/g1/g1HeapVerifier.hpp"
 #include "gc/shared/cardTable.hpp"
 #include "gc/shared/gcArguments.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/workerPolicy.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/globals_extension.hpp"
@@ -247,6 +248,7 @@ void G1Arguments::initialize() {
 
 void G1Arguments::initialize_heap_flags_and_sizes() {
   GCArguments::initialize_heap_flags_and_sizes();
+  GCForwarding::initialize_flags(heap_reserved_size_bytes());
 }
 
 CollectedHeap* G1Arguments::create_heap() {

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -78,6 +78,7 @@
 #include "gc/shared/classUnloadingContext.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
 #include "gc/shared/gcBehaviours.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcTimer.hpp"
@@ -1434,6 +1435,8 @@ jint G1CollectedHeap::initialize() {
 
   G1InitLogger::print();
 
+  GCForwarding::initialize(heap_rs.region());
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -29,6 +29,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCCompactTask.hpp"
 #include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "logging/log.hpp"
 #include "oops/oop.inline.hpp"
@@ -41,7 +42,7 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
 
 size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
   size_t size = obj->size();
-  if (obj->is_forwarded()) {
+  if (GCForwarding::is_forwarded(obj)) {
     G1FullGCCompactTask::copy_object_to_new_location(obj);
   }
 
@@ -52,13 +53,13 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
 }
 
 void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
-  assert(obj->is_forwarded(), "Sanity!");
-  assert(obj->forwardee() != obj, "Object must have a new location");
+  assert(GCForwarding::is_forwarded(obj), "Sanity!");
+  assert(GCForwarding::forwardee(obj) != obj, "Object must have a new location");
 
   size_t size = obj->size();
   // Copy object and reinit its mark.
   HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
-  HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+  HeapWord* destination = cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj));
   Copy::aligned_conjoint_words(obj_addr, destination, size);
 
   // There is no need to transform stack chunks - marking already did that.
@@ -121,7 +122,7 @@ void G1FullGCCompactTask::compact_humongous_obj(G1HeapRegion* src_hr) {
   size_t word_size = obj->size();
 
   uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
-  HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+  HeapWord* destination = cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj));
 
   assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
   collector()->mark_bitmap()->clear(obj);

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -26,6 +26,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1HeapRegion.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/debug.hpp"
@@ -106,10 +107,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
     if (!object->is_forwarded()) {
       preserved_stack()->push_if_necessary(object, object->mark());
     }
-    object->forward_to(cast_to_oop(_compaction_top));
-    assert(object->is_forwarded(), "must be forwarded");
+    GCForwarding::forward_to(object, cast_to_oop(_compaction_top));
+    assert(GCForwarding::is_forwarded(object), "must be forwarded");
   } else {
-    assert(!object->is_forwarded(), "must not be forwarded");
+    assert(!GCForwarding::is_forwarded(object), "must not be forwarded");
   }
 
   // Update compaction values.
@@ -172,8 +173,8 @@ void G1FullGCCompactionPoint::forward_humongous(G1HeapRegion* hr) {
   preserved_stack()->push_if_necessary(obj, obj->mark());
 
   G1HeapRegion* dest_hr = _compaction_regions->at(range_begin);
-  obj->forward_to(cast_to_oop(dest_hr->bottom()));
-  assert(obj->is_forwarded(), "Object must be forwarded!");
+  GCForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom()));
+  assert(GCForwarding::is_forwarded(obj), "Object must be forwarded!");
 
   // Add the humongous object regions to the compaction point.
   add_humongous(hr);

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -32,6 +32,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
 #include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"
@@ -65,8 +66,8 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
     return;
   }
 
-  if (obj->is_forwarded()) {
-    oop forwardee = obj->forwardee();
+  if (GCForwarding::is_forwarded(obj)) {
+    oop forwardee = GCForwarding::forwardee(obj);
     // Forwarded, just update.
     assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
     RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);

src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp

@@ -32,6 +32,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCScope.hpp"
 #include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 
 void G1DetermineCompactionQueueClosure::free_empty_humongous_region(G1HeapRegion* hr) {
   _g1h->free_humongous_region(hr, nullptr);
@@ -114,10 +115,10 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(G1HeapRegion* hr)
 }
 
 inline size_t G1SerialRePrepareClosure::apply(oop obj) {
-  if (obj->is_forwarded()) {
+  if (GCForwarding::is_forwarded(obj)) {
     // We skip objects compiled into the first region or
     // into regions not part of the serial compaction point.
-    if (cast_from_oop<HeapWord*>(obj->forwardee()) < _dense_prefix_top) {
+    if (cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj)) < _dense_prefix_top) {
       return obj->size();
     }
   }

src/hotspot/share/gc/parallel/parallelArguments.cpp

@@ -28,6 +28,7 @@
 #include "gc/parallel/parallelScavengeHeap.hpp"
 #include "gc/shared/adaptiveSizePolicy.hpp"
 #include "gc/shared/gcArguments.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/genArguments.hpp"
 #include "gc/shared/workerPolicy.hpp"
 #include "logging/log.hpp"
@@ -127,6 +128,7 @@ void ParallelArguments::initialize_heap_flags_and_sizes() {
     // Redo everything from the start
     initialize_heap_flags_and_sizes_one_pass();
   }
+  GCForwarding::initialize_flags(heap_reserved_size_bytes());
 }
 
 size_t ParallelArguments::heap_reserved_size_bytes() {

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -33,6 +33,7 @@
 #include "gc/parallel/psPromotionManager.hpp"
 #include "gc/parallel/psScavenge.hpp"
 #include "gc/parallel/psVMOperations.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcLocker.inline.hpp"
 #include "gc/shared/gcWhen.hpp"
@@ -129,6 +130,8 @@ jint ParallelScavengeHeap::initialize() {
 
   ParallelInitLogger::print();
 
+  GCForwarding::initialize(heap_rs.region());
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/parallel/psParallelCompact.cpp

@@ -45,6 +45,7 @@
 #include "gc/parallel/psYoungGen.hpp"
 #include "gc/shared/classUnloadingContext.hpp"
 #include "gc/shared/gcCause.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcLocker.hpp"
@@ -1592,7 +1593,7 @@ void PSParallelCompact::forward_to_new_addr() {
             oop obj = cast_to_oop(cur_addr);
             if (new_addr != cur_addr) {
               cm->preserved_marks()->push_if_necessary(obj, obj->mark());
-              obj->forward_to(cast_to_oop(new_addr));
+              GCForwarding::forward_to(obj, cast_to_oop(new_addr));
             }
             size_t obj_size = obj->size();
             live_words += obj_size;
@@ -1635,7 +1636,7 @@ void PSParallelCompact::verify_forward() {
       }
       oop obj = cast_to_oop(cur_addr);
       if (cur_addr != bump_ptr) {
-        assert(obj->forwardee() == cast_to_oop(bump_ptr), "inv");
+        assert(GCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv");
       }
       bump_ptr += obj->size();
       cur_addr += obj->size();
@@ -2398,8 +2399,8 @@ void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) {
   if (copy_destination() != source()) {
     DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
     assert(source() != destination(), "inv");
-    assert(cast_to_oop(source())->is_forwarded(), "inv");
-    assert(cast_to_oop(source())->forwardee() == cast_to_oop(destination()), "inv");
+    assert(GCForwarding::is_forwarded(cast_to_oop(source())), "inv");
+    assert(GCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
     Copy::aligned_conjoint_words(source(), copy_destination(), words);
     cast_to_oop(copy_destination())->init_mark();
   }

src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp

@@ -31,6 +31,7 @@
 #include "gc/parallel/parMarkBitMap.inline.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/klass.hpp"
@@ -79,7 +80,7 @@ inline void PSParallelCompact::adjust_pointer(T* p) {
     if (!obj->is_forwarded()) {
       return;
     }
-    oop new_obj = obj->forwardee();
+    oop new_obj = GCForwarding::forwardee(obj);
     assert(new_obj != nullptr, "non-null address for live objects");
     assert(new_obj != obj, "inv");
     assert(ParallelScavengeHeap::heap()->is_in_reserved(new_obj),

src/hotspot/share/gc/serial/serialArguments.cpp

@@ -23,10 +23,16 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/genArguments.hpp"
 #include "gc/serial/serialArguments.hpp"
 #include "gc/serial/serialHeap.hpp"
 
+void SerialArguments::initialize_heap_flags_and_sizes() {
+  GenArguments::initialize_heap_flags_and_sizes();
+  GCForwarding::initialize_flags(MaxNewSize + MaxOldSize);
+}
+
 CollectedHeap* SerialArguments::create_heap() {
   return new SerialHeap();
 }

src/hotspot/share/gc/serial/serialArguments.hpp

@@ -32,6 +32,7 @@ class CollectedHeap;
 class SerialArguments : public GenArguments {
 private:
   virtual CollectedHeap* create_heap();
+  void initialize_heap_flags_and_sizes();
 };
 
 #endif // SHARE_GC_SERIAL_SERIALARGUMENTS_HPP

src/hotspot/share/gc/serial/serialFullGC.cpp

@@ -43,6 +43,7 @@
 #include "gc/shared/classUnloadingContext.hpp"
 #include "gc/shared/collectedHeap.inline.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcTimer.hpp"
 #include "gc/shared/gcTrace.hpp"
@@ -230,7 +231,7 @@ class Compacter {
   static void forward_obj(oop obj, HeapWord* new_addr) {
     prefetch_write_scan(obj);
     if (cast_from_oop<HeapWord*>(obj) != new_addr) {
-      obj->forward_to(cast_to_oop(new_addr));
+      GCForwarding::forward_to(obj, cast_to_oop(new_addr));
     } else {
       assert(obj->is_gc_marked(), "inv");
       // This obj will stay in-place. Fix the markword.
@@ -255,7 +256,7 @@ class Compacter {
     prefetch_read_scan(addr);
 
     oop obj = cast_to_oop(addr);
-    oop new_obj = obj->forwardee();
+    oop new_obj = GCForwarding::forwardee(obj);
     HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
     assert(addr != new_addr, "inv");
     prefetch_write_copy(new_addr);
@@ -352,13 +353,13 @@ class Compacter {
       HeapWord* top = space->top();
 
       // Check if the first obj inside this space is forwarded.
-      if (!cast_to_oop(cur_addr)->is_forwarded()) {
+      if (!GCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
         // Jump over consecutive (in-place) live-objs-chunk
         cur_addr = get_first_dead(i);
       }
 
       while (cur_addr < top) {
-        if (!cast_to_oop(cur_addr)->is_forwarded()) {
+        if (!GCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
           cur_addr = *(HeapWord**) cur_addr;
           continue;
         }
@@ -624,8 +625,8 @@ template <class T> void SerialFullGC::adjust_pointer(T* p) {
     oop obj = CompressedOops::decode_not_null(heap_oop);
     assert(Universe::heap()->is_in(obj), "should be in heap");
 
-    if (obj->is_forwarded()) {
-      oop new_obj = obj->forwardee();
+    if (GCForwarding::is_forwarded(obj)) {
+      oop new_obj = GCForwarding::forwardee(obj);
       assert(is_object_aligned(new_obj), "oop must be aligned");
       RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
     }

src/hotspot/share/gc/serial/serialHeap.cpp

@@ -40,6 +40,7 @@
 #include "gc/shared/collectedHeap.inline.hpp"
 #include "gc/shared/collectorCounters.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcInitLogger.hpp"
 #include "gc/shared/gcLocker.inline.hpp"
@@ -200,6 +201,8 @@ jint SerialHeap::initialize() {
 
   GCInitLogger::print();
 
+  GCForwarding::initialize(_reserved);
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/shared/gcForwarding.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcForwarding.hpp"
+#include "memory/memRegion.hpp"
+#include "runtime/globals_extension.hpp"
+
+HeapWord* GCForwarding::_heap_base = nullptr;
+int GCForwarding::_num_low_bits = 0;
+
+void GCForwarding::initialize_flags(size_t max_heap_size) {
+#ifdef _LP64
+  // Nothing to do here, yet. As soon as we have compact
+  // object headers, we will disable the flag when the
+  // heap size exceeds the narrow-encodable address space.
+
+  // size_t max_narrow_heap_size = right_n_bits(NumLowBitsNarrow - Shift);
+  // if (UseCompactObjectHeaders && max_heap_size > max_narrow_heap_size * HeapWordSize) {
+  //   FLAG_SET_DEFAULT(UseCompactObjectHeaders, false);
+  // }
+#endif
+}
+
+void GCForwarding::initialize(MemRegion heap) {
+#ifdef _LP64
+  _heap_base = heap.start();
+  if (heap.word_size() <= right_n_bits(NumLowBitsNarrow - Shift)) {
+    _num_low_bits = NumLowBitsNarrow;
+  } else {
+    // assert(!UseCompactObjectHeaders, "Compact object headers should be turned off for large heaps");
+    _num_low_bits = NumLowBitsWide;
+  }
+#endif
+}