8357449: ZGC: Multiple medium page sizes

Reviewed-by: stefank, jsikstro

Author: xmas92

src/hotspot/share/gc/z/vmStructs_z.hpp

@@ -120,13 +120,13 @@ typedef ZValue<ZPerNUMAStorage, ZPartition> ZPerNUMAZPartition;
   declare_constant(ZPageType::small)                                                                 \
   declare_constant(ZPageType::medium)                                                                \
   declare_constant(ZPageType::large)                                                                 \
+  declare_constant(ZPageSizeSmallShift)                                                              \
+  declare_constant(ZPageSizeMediumMaxShift)                                                          \
   declare_constant(ZObjectAlignmentMediumShift)                                                      \
   declare_constant(ZObjectAlignmentLargeShift)
 
 #define VM_LONG_CONSTANTS_Z(declare_constant)                                                        \
   declare_constant(ZGranuleSizeShift)                                                                \
-  declare_constant(ZPageSizeSmallShift)                                                              \
-  declare_constant(ZPageSizeMediumShift)                                                             \
   declare_constant(ZAddressOffsetShift)                                                              \
   declare_constant(ZAddressOffsetBits)                                                               \
   declare_constant(ZAddressOffsetMask)                                                               \

src/hotspot/share/gc/z/zAllocationFlags.hpp

@@ -31,22 +31,25 @@
 // Allocation flags layout
 // -----------------------
 //
-//   7      1 0
-//  +------+-+-+
-//  |000000|1|1|
-//  +------+-+-+
+//   7     2 1 0
+//  +-----+-+-+-+
+//  |00000|1|1|1|
+//  +-----+-+-+-+
+//  |      | | |
+//  |      | | * 0-0 Non-Blocking Flag (1-bit)
 //  |      | |
-//  |      | * 0-0 Non-Blocking Flag (1-bit)
+//  |      | * 1-1 GC Relocation Flag (1-bit)
 //  |      |
-//  |      * 1-1 GC Relocation Flag (1-bit)
+//  |      * 2-2 Fast Medium Flag (1-bit)
 //  |
-//  * 7-2 Unused (6-bits)
+//  * 7-3 Unused (5-bits)
 //
 
 class ZAllocationFlags {
 private:
   typedef ZBitField<uint8_t, bool, 0, 1> field_non_blocking;
   typedef ZBitField<uint8_t, bool, 1, 1> field_gc_relocation;
+  typedef ZBitField<uint8_t, bool, 2, 1> field_fast_medium;
 
   uint8_t _flags;
 
@@ -62,13 +65,21 @@ class ZAllocationFlags {
     _flags |= field_gc_relocation::encode(true);
   }
 
+  void set_fast_medium() {
+    _flags |= field_fast_medium::encode(true);
+  }
+
   bool non_blocking() const {
     return field_non_blocking::decode(_flags);
   }
 
   bool gc_relocation() const {
     return field_gc_relocation::decode(_flags);
   }
+
+  bool fast_medium() const {
+    return field_fast_medium::decode(_flags);
+  }
 };
 
 #endif // SHARE_GC_Z_ZALLOCATIONFLAGS_HPP

src/hotspot/share/gc/z/zArguments.cpp

@@ -162,9 +162,8 @@ void ZArguments::initialize() {
     uint tenuring_threshold;
     for (tenuring_threshold = 0; tenuring_threshold < MaxTenuringThreshold; ++tenuring_threshold) {
       // Reduce the number of object ages, if the resulting garbage is too high
-      const size_t medium_page_overhead = ZPageSizeMedium * tenuring_threshold;
-      const size_t small_page_overhead = ZPageSizeSmall * ConcGCThreads * tenuring_threshold;
-      if (small_page_overhead + medium_page_overhead >= ZHeuristics::significant_young_overhead()) {
+      const size_t per_age_overhead = ZHeuristics::relocation_headroom();
+      if (per_age_overhead * tenuring_threshold >= ZHeuristics::significant_young_overhead()) {
         break;
       }
     }

src/hotspot/share/gc/z/zGlobals.cpp

@@ -23,8 +23,10 @@
 
 #include "gc/z/zGlobals.hpp"
 
-size_t     ZPageSizeMediumShift;
-size_t     ZPageSizeMedium;
+int        ZPageSizeMediumMaxShift;
+size_t     ZPageSizeMediumMax;
+size_t     ZPageSizeMediumMin;
+bool       ZPageSizeMediumEnabled;
 
 size_t     ZObjectSizeLimitMedium;
 

src/hotspot/share/gc/z/zGlobals.hpp

@@ -42,12 +42,14 @@ const size_t      ZVirtualToPhysicalRatio       = 16; // 16:1
 const size_t      ZMaxVirtualReservations       = 100; // Each reservation at least 1% of total
 
 // Page size shifts
-const size_t      ZPageSizeSmallShift           = ZGranuleSizeShift;
-extern size_t     ZPageSizeMediumShift;
+const int         ZPageSizeSmallShift           = (int)ZGranuleSizeShift;
+extern int        ZPageSizeMediumMaxShift;
 
 // Page sizes
 const size_t      ZPageSizeSmall                = (size_t)1 << ZPageSizeSmallShift;
-extern size_t     ZPageSizeMedium;
+extern size_t     ZPageSizeMediumMax;
+extern size_t     ZPageSizeMediumMin;
+extern bool       ZPageSizeMediumEnabled;
 
 // Object size limits
 const size_t      ZObjectSizeLimitSmall         = ZPageSizeSmall / 8; // 12.5% max waste

src/hotspot/share/gc/z/zHeap.cpp

@@ -364,7 +364,15 @@ void ZHeap::print_globals_on(outputStream* st) const {
   st->print_cr(" Old Collection:     %s/%u", ZGeneration::old()->phase_to_string(), ZGeneration::old()->seqnum());
   st->print_cr(" Offset Max:         " EXACTFMT " (" PTR_FORMAT ")", EXACTFMTARGS(ZAddressOffsetMax), ZAddressOffsetMax);
   st->print_cr(" Page Size Small:    %zuM", ZPageSizeSmall / M);
-  st->print_cr(" Page Size Medium:   %zuM", ZPageSizeMedium / M);
+  if (ZPageSizeMediumEnabled) {
+    if (ZPageSizeMediumMin == ZPageSizeMediumMax) {
+      st->print_cr(" Page Size Medium: %zuM", ZPageSizeMediumMax / M);
+    } else {
+      st->print_cr(" Page Size Medium: Range [%zuM, %zuM]", ZPageSizeMediumMin / M, ZPageSizeMediumMax / M);
+    }
+  } else {
+    st->print_cr(" Page Size Medium: N/A");
+  }
   st->cr();
   st->print_cr("ZGC Metadata Bits:");
   st->print_cr(" LoadGood:           " PTR_FORMAT, ZPointerLoadGoodMask);

src/hotspot/share/gc/z/zHeuristics.cpp

@@ -28,6 +28,7 @@
 #include "gc/z/zHeuristics.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/os.hpp"
+#include "utilities/align.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/powerOfTwo.hpp"
 
@@ -44,18 +45,22 @@ void ZHeuristics::set_medium_page_size() {
 
   if (size > ZPageSizeSmall) {
     // Enable medium pages
-    ZPageSizeMedium             = size;
-    ZPageSizeMediumShift        = (size_t)log2i_exact(ZPageSizeMedium);
-    ZObjectSizeLimitMedium      = ZPageSizeMedium / 8;
-    ZObjectAlignmentMediumShift = (int)ZPageSizeMediumShift - 13;
+    ZPageSizeMediumMax          = size;
+    ZPageSizeMediumMaxShift     = log2i_exact(ZPageSizeMediumMax);
+    ZObjectSizeLimitMedium      = ZPageSizeMediumMax / 8;
+    ZObjectAlignmentMediumShift = ZPageSizeMediumMaxShift - 13;
     ZObjectAlignmentMedium      = 1 << ZObjectAlignmentMediumShift;
+    ZPageSizeMediumEnabled      = true;
+    ZPageSizeMediumMin          = ZUseMediumPageSizeRange
+                                ? align_up(ZObjectSizeLimitMedium, ZGranuleSize)
+                                : ZPageSizeMediumMax;
   }
 }
 
 size_t ZHeuristics::relocation_headroom() {
   // Calculate headroom needed to avoid in-place relocation. Each worker will try
   // to allocate a small page, and all workers will share a single medium page.
-  return (ConcGCThreads * ZPageSizeSmall) + ZPageSizeMedium;
+  return (ConcGCThreads * ZPageSizeSmall) + ZPageSizeMediumMax;
 }
 
 bool ZHeuristics::use_per_cpu_shared_small_pages() {

src/hotspot/share/gc/z/zMappedCache.cpp

@@ -414,6 +414,11 @@ ZVirtualMemory ZMappedCache::remove_vmem(ZMappedCacheEntry* const entry, size_t
 template <typename SelectFunction, typename ConsumeFunction>
 bool ZMappedCache::try_remove_vmem_size_class(size_t min_size, SelectFunction select, ConsumeFunction consume) {
 new_max_size:
+  if (_size < min_size) {
+    // Not enough left in cache to satisfy the min_size
+    return false;
+  }
+
   // Query the max select size possible given the size of the cache
   const size_t max_size = select(_size);
 
@@ -648,6 +653,37 @@ ZVirtualMemory ZMappedCache::remove_contiguous(size_t size) {
   return result;
 }
 
+ZVirtualMemory ZMappedCache::remove_contiguous_power_of_2(size_t min_size, size_t max_size) {
+  precond(is_aligned(min_size, ZGranuleSize));
+  precond(is_power_of_2(min_size));
+  precond(is_aligned(max_size, ZGranuleSize));
+  precond(is_power_of_2(max_size));
+  precond(min_size <= max_size);
+
+  ZVirtualMemory result;
+
+  const auto select_size_fn = [&](size_t size) {
+    // Always select a power of 2 within the [min_size, max_size] interval.
+    return clamp(round_down_power_of_2(size), min_size, max_size);
+  };
+
+  const auto consume_vmem_fn = [&](ZVirtualMemory vmem) {
+    assert(result.is_null(), "only consume once");
+    assert(min_size <= vmem.size() && vmem.size() <= max_size,
+      "Must be %zu <= %zu <= %zu", min_size, vmem.size(), max_size);
+    assert(is_power_of_2(vmem.size()), "Must be power_of_2(%zu)", vmem.size());
+
+    result = vmem;
+
+    // Only require one vmem
+    return true;
+  };
+
+  scan_remove_vmem<RemovalStrategy::SizeClasses>(min_size, select_size_fn, consume_vmem_fn);
+
+  return result;
+}
+
 size_t ZMappedCache::remove_discontiguous(size_t size, ZArray<ZVirtualMemory>* out) {
   return remove_discontiguous_with_strategy<RemovalStrategy::SizeClasses>(size, out);
 }

src/hotspot/share/gc/z/zMappedCache.hpp

@@ -129,6 +129,7 @@ class ZMappedCache {
   void insert(const ZVirtualMemory& vmem);
 
   ZVirtualMemory remove_contiguous(size_t size);
+  ZVirtualMemory remove_contiguous_power_of_2(size_t min_size, size_t max_size);
   size_t remove_discontiguous(size_t size, ZArray<ZVirtualMemory>* out);
 
   size_t reset_min();

src/hotspot/share/gc/z/zObjectAllocator.cpp

@@ -140,13 +140,26 @@ zaddress ZObjectAllocator::alloc_object_in_medium_page(size_t size,
     ZAllocationFlags non_blocking_flags = flags;
     non_blocking_flags.set_non_blocking();
 
-    addr = alloc_object_in_shared_page(shared_medium_page, ZPageType::medium, ZPageSizeMedium, size, non_blocking_flags);
+    if (ZPageSizeMediumMin != ZPageSizeMediumMax) {
+      assert(ZPageSizeMediumEnabled, "must be enabled");
+      // We attempt a fast medium allocations first. Which will only succeed
+      // if a page in the range [ZPageSizeMediumMin, ZPageSizeMediumMax] can
+      // be allocated without any expensive syscalls, directly from the cache.
+      ZAllocationFlags fast_medium_flags = non_blocking_flags;
+      fast_medium_flags.set_fast_medium();
+      addr = alloc_object_in_shared_page(shared_medium_page, ZPageType::medium, ZPageSizeMediumMax, size, fast_medium_flags);
+    }
+
+    if (is_null(addr)) {
+      addr = alloc_object_in_shared_page(shared_medium_page, ZPageType::medium, ZPageSizeMediumMax, size, non_blocking_flags);
+    }
+
   }
 
   if (is_null(addr) && !flags.non_blocking()) {
     // The above allocation attempts failed and this allocation should stall
     // until memory is available. Redo the allocation with blocking enabled.
-    addr = alloc_object_in_shared_page(shared_medium_page, ZPageType::medium, ZPageSizeMedium, size, flags);
+    addr = alloc_object_in_shared_page(shared_medium_page, ZPageType::medium, ZPageSizeMediumMax, size, flags);
   }
 
   return addr;