8267703: runtime/cds/appcds/cacheObject/HeapFragmentationTest.java crashed with OutOfMemory

Reviewed-by: tschatzl, kbarrett

Author: kstefanj

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -680,6 +680,11 @@ void G1ConcurrentMark::cleanup_for_next_mark() {
 
 void G1ConcurrentMark::clear_next_bitmap(WorkGang* workers) {
   assert_at_safepoint_on_vm_thread();
+  // To avoid fragmentation the full collection requesting to clear the bitmap
+  // might use fewer workers than available. To ensure the bitmap is cleared
+  // as efficiently as possible the number of active workers are temporarily
+  // increased to include all currently created workers.
+  WithUpdatedActiveWorkers update(workers, workers->created_workers());
   clear_bitmap(_next_mark_bitmap, workers, false);
 }
 

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -94,10 +94,15 @@ uint G1FullCollector::calc_active_workers() {
   uint current_active_workers = heap->workers()->active_workers();
   uint active_worker_limit = WorkerPolicy::calc_active_workers(max_worker_count, current_active_workers, 0);
 
+  // Finally consider the amount of used regions.
+  uint used_worker_limit = heap->num_used_regions();
+  assert(used_worker_limit > 0, "Should never have zero used regions.");
+
   // Update active workers to the lower of the limits.
-  uint worker_count = MIN2(heap_waste_worker_limit, active_worker_limit);
-  log_debug(gc, task)("Requesting %u active workers for full compaction (waste limited workers: %u, adaptive workers: %u)",
-                      worker_count, heap_waste_worker_limit, active_worker_limit);
+  uint worker_count = MIN3(heap_waste_worker_limit, active_worker_limit, used_worker_limit);
+  log_debug(gc, task)("Requesting %u active workers for full compaction (waste limited workers: %u, "
+                      "adaptive workers: %u, used limited workers: %u)",
+                      worker_count, heap_waste_worker_limit, active_worker_limit, used_worker_limit);
   worker_count = heap->workers()->update_active_workers(worker_count);
   log_info(gc, task)("Using %u workers of %u for full compaction", worker_count, max_worker_count);