8305896: Alternative full GC forwarding

Author: rkennke

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -76,6 +76,7 @@
 #include "gc/g1/heapRegionSet.inline.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
 #include "gc/shared/gcBehaviours.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcLocker.inline.hpp"
@@ -1675,6 +1676,8 @@ jint G1CollectedHeap::initialize() {
 
   G1InitLogger::print();
 
+  GCForwarding::initialize(heap_rs.region(), HeapRegion::LogOfHRGrainBytes - LogHeapWordSize);
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -39,6 +39,7 @@
 #include "gc/g1/g1OopClosures.hpp"
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/referenceProcessor.hpp"
@@ -207,6 +208,8 @@ void G1FullCollector::collect() {
   // Don't add any more derived pointers during later phases
   deactivate_derived_pointers();
 
+  GCForwarding::begin();
+
   phase2_prepare_compaction();
 
   if (has_compaction_targets()) {
@@ -219,6 +222,8 @@ void G1FullCollector::collect() {
     log_info(gc, phases) ("No Regions selected for compaction. Skipping Phase 3: Adjust pointers and Phase 4: Compact heap");
   }
 
+  GCForwarding::end();
+
   phase5_reset_metadata();
 
   G1CollectedHeap::finish_codecache_marking_cycle();
@@ -341,7 +346,7 @@ void G1FullCollector::phase2_prepare_compaction() {
   // Try to avoid OOM immediately after Full GC in case there are no free regions
   // left after determining the result locations (i.e. this phase). Prepare to
   // maximally compact the tail regions of the compaction queues serially.
-  if (scope()->do_maximal_compaction() || !has_free_compaction_targets) {
+  if (!UseAltGCForwarding && (scope()->do_maximal_compaction() || !has_free_compaction_targets)) {
     phase2c_prepare_serial_compaction();
   }
 }

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -29,6 +29,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCCompactTask.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "logging/log.hpp"
 #include "oops/oop.inline.hpp"
@@ -41,8 +42,8 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
 
 size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
   size_t size = obj->size();
-  if (obj->is_forwarded()) {
-    HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+  if (GCForwarding::is_forwarded(obj)) {
+    HeapWord* destination = cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj));
 
     // copy object and reinit its mark
     HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -26,6 +26,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/heapRegion.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/debug.hpp"
 
@@ -101,10 +102,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
 
   // Store a forwarding pointer if the object should be moved.
   if (cast_from_oop<HeapWord*>(object) != _compaction_top) {
-    object->forward_to(cast_to_oop(_compaction_top));
-    assert(object->is_forwarded(), "must be forwarded");
+    GCForwarding::forward_to(object, cast_to_oop(_compaction_top));
+    assert(GCForwarding::is_forwarded(object), "must be forwarded");
   } else {
-    assert(!object->is_forwarded(), "must not be forwarded");
+    assert(GCForwarding::is_not_forwarded(object), "must not be forwarded");
   }
 
   // Update compaction values.

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -32,6 +32,7 @@
 #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"
@@ -65,8 +66,8 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
     return;
   }
 
-  if (obj->is_forwarded()) {
-    oop forwardee = obj->forwardee();
+  if (GCForwarding::is_forwarded(obj)) {
+    oop forwardee = GCForwarding::forwardee(obj);
     // Forwarded, just update.
     assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
     RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);

src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp

@@ -32,6 +32,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCScope.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
+#include "gc/shared/gcForwarding.hpp"
 
 template<bool is_humongous>
 void G1DetermineCompactionQueueClosure::free_pinned_region(HeapRegion* hr) {
@@ -112,10 +113,10 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(HeapRegion* hr) {
 }
 
 inline size_t G1SerialRePrepareClosure::apply(oop obj) {
-  if (obj->is_forwarded()) {
+  if (GCForwarding::is_forwarded(obj)) {
     // We skip objects compiled into the first region or
     // into regions not part of the serial compaction point.
-    if (cast_from_oop<HeapWord*>(obj->forwardee()) < _dense_prefix_top) {
+    if (cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj)) < _dense_prefix_top) {
       return obj->size();
     }
   }

src/hotspot/share/gc/serial/genMarkSweep.cpp

@@ -36,6 +36,7 @@
 #include "gc/serial/genMarkSweep.hpp"
 #include "gc/serial/serialGcRefProcProxyTask.hpp"
 #include "gc/shared/collectedHeap.inline.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcTimer.hpp"
 #include "gc/shared/gcTrace.hpp"
@@ -87,6 +88,8 @@ void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
 
   mark_sweep_phase1(clear_all_softrefs);
 
+  GCForwarding::begin();
+
   mark_sweep_phase2();
 
   // Don't add any more derived pointers during phase3
@@ -105,6 +108,8 @@ void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
   // (Should this be in general part?)
   gch->save_marks();
 
+  GCForwarding::end();
+
   deallocate_stacks();
 
   MarkSweep::_string_dedup_requests->flush();

src/hotspot/share/gc/serial/markSweep.inline.hpp

@@ -30,6 +30,7 @@
 #include "classfile/classLoaderData.inline.hpp"
 #include "classfile/javaClasses.inline.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/serial/serialStringDedup.hpp"
 #include "memory/universe.hpp"
 #include "oops/markWord.hpp"
@@ -45,8 +46,8 @@ template <class T> inline void MarkSweep::adjust_pointer(T* p) {
     oop obj = CompressedOops::decode_not_null(heap_oop);
     assert(Universe::heap()->is_in(obj), "should be in heap");
 
-    if (obj->is_forwarded()) {
-      oop new_obj = obj->forwardee();
+    if (GCForwarding::is_forwarded(obj)) {
+      oop new_obj = GCForwarding::forwardee(obj);
       assert(is_object_aligned(new_obj), "oop must be aligned");
       RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
     }

src/hotspot/share/gc/shared/gcForwarding.cpp

@@ -0,0 +1,57 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcForwarding.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/slidingForwarding.hpp"
+
+SlidingForwarding* GCForwarding::_sliding_forwarding = nullptr;
+
+void GCForwarding::initialize(MemRegion heap) {
+  if (UseAltGCForwarding) {
+    assert(_sliding_forwarding == nullptr, "only call this once");
+    _sliding_forwarding = new SlidingForwarding(heap);
+  }
+}
+
+void GCForwarding::initialize(MemRegion heap, size_t region_size_words_shift) {
+  if (UseAltGCForwarding) {
+    assert(_sliding_forwarding == nullptr, "only call this once");
+    _sliding_forwarding = new SlidingForwarding(heap, region_size_words_shift);
+  }
+}
+
+void GCForwarding::begin() {
+  if (UseAltGCForwarding) {
+    assert(_sliding_forwarding != nullptr, "expect sliding forwarding initialized");
+    _sliding_forwarding->clear();
+  }
+}
+
+void GCForwarding::end() {
+  if (UseAltGCForwarding) {
+    assert(_sliding_forwarding != nullptr, "expect sliding forwarding initialized");
+  }
+}

src/hotspot/share/gc/shared/gcForwarding.hpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHARED_GCFORWARDING_HPP
+#define SHARE_GC_SHARED_GCFORWARDING_HPP
+
+#include "memory/allStatic.hpp"
+#include "memory/memRegion.hpp"
+#include "oops/oopsHierarchy.hpp"
+
+class SlidingForwarding;
+
+class GCForwarding : public AllStatic {
+private:
+  static SlidingForwarding* _sliding_forwarding;
+
+public:
+  static void initialize(MemRegion heap);
+  static void initialize(MemRegion heap, size_t region_size_words_shift);
+  static void begin();
+  static void end();
+
+  static inline bool is_forwarded(oop obj);
+  static inline bool is_not_forwarded(oop obj);
+  static inline oop forwardee(oop obj);
+  static inline void forward_to(oop obj, oop fwd);
+};
+
+#endif // SHARE_GC_SHARED_GCFORWARDING_HPP