8312132: Add tracking of multiple address spaces in NMT

Co-authored-by: Thomas Stuefe <[email protected]>
Reviewed-by: stefank, stuefe

Author: jdksjolen

src/hotspot/share/gc/z/zInitialize.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
 #include "gc/z/zJNICritical.hpp"
 #include "gc/z/zLargePages.hpp"
 #include "gc/z/zMarkStackAllocator.hpp"
+#include "gc/z/zNMT.hpp"
 #include "gc/z/zNUMA.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zThreadLocalAllocBuffer.hpp"
@@ -46,6 +47,7 @@ ZInitialize::ZInitialize(ZBarrierSet* barrier_set) {
                      VM_Version::jdk_debug_level());
 
   // Early initialization
+  ZNMT::initialize();
   ZGlobalsPointers::initialize();
   ZNUMA::initialize();
   ZCPU::initialize();

src/hotspot/share/gc/z/zNMT.cpp

@@ -28,92 +28,31 @@
 #include "gc/z/zVirtualMemory.hpp"
 #include "nmt/memflags.hpp"
 #include "nmt/memTracker.hpp"
+#include "nmt/memoryFileTracker.hpp"
 #include "utilities/nativeCallStack.hpp"
 
-ZNMT::Reservation ZNMT::_reservations[ZMaxVirtualReservations] = {};
-size_t ZNMT::_num_reservations = 0;
+MemoryFileTracker::MemoryFile* ZNMT::_device = nullptr;
 
-size_t ZNMT::reservation_index(zoffset offset, size_t* offset_in_reservation) {
-  assert(_num_reservations > 0, "at least one reservation must exist");
-
-  size_t index = 0;
-  *offset_in_reservation = untype(offset);
-  for (; index < _num_reservations; ++index) {
-    const size_t reservation_size = _reservations[index]._size;
-    if (*offset_in_reservation < reservation_size) {
-      break;
-    }
-    *offset_in_reservation -= reservation_size;
-  }
-
-  assert(index != _num_reservations, "failed to find reservation index");
-  return index;
-}
-
-void ZNMT::process_fake_mapping(zoffset offset, size_t size, bool commit) {
-  // In order to satisfy NTM's requirement of an 1:1 mapping between committed
-  // and reserved addresses, a fake mapping from the offset into the reservation
-  // is used.
-  //
-  // These mappings from
-  //   [offset, offset + size) -> {[virtual address range], ...}
-  // are stable after the heap has been reserved. No commits proceed any
-  // reservations. Committing and uncommitting the same [offset, offset + size)
-  // range will result in same virtual memory ranges.
-
-  size_t left_to_process = size;
-  size_t offset_in_reservation;
-  for (size_t i = reservation_index(offset, &offset_in_reservation); i < _num_reservations; ++i) {
-    const zaddress_unsafe reservation_start = _reservations[i]._start;
-    const size_t reservation_size = _reservations[i]._size;
-    const size_t sub_range_size = MIN2(left_to_process, reservation_size - offset_in_reservation);
-    const uintptr_t sub_range_addr = untype(reservation_start) + offset_in_reservation;
-
-    // commit / uncommit memory
-    if (commit) {
-      MemTracker::record_virtual_memory_commit((void*)sub_range_addr, sub_range_size, CALLER_PC);
-    } else {
-      ThreadCritical tc;
-      MemTracker::record_virtual_memory_uncommit((address)sub_range_addr, sub_range_size);
-    }
-
-    left_to_process -= sub_range_size;
-    if (left_to_process == 0) {
-      // Processed all nmt registrations
-      return;
-    }
-
-    offset_in_reservation = 0;
-  }
-
-  assert(left_to_process == 0, "everything was not commited");
+void ZNMT::initialize() {
+  _device = MemTracker::register_file("ZGC heap backing file");
 }
 
 void ZNMT::reserve(zaddress_unsafe start, size_t size) {
-  assert(_num_reservations < ZMaxVirtualReservations, "too many reservations");
-  // Keep track of the reservations made in order to create fake mappings
-  // between the reserved and commited memory.
-  // See details in ZNMT::process_fake_mapping
-  _reservations[_num_reservations++] = {start, size};
-
-  MemTracker::record_virtual_memory_reserve((void*)untype(start), size, CALLER_PC, mtJavaHeap);
+  MemTracker::record_virtual_memory_reserve((address)untype(start), size, CALLER_PC, mtJavaHeap);
 }
 
 void ZNMT::commit(zoffset offset, size_t size) {
-  // NMT expects a 1-to-1 mapping between virtual and physical memory.
-  // ZGC can temporarily have multiple virtual addresses pointing to
-  // the same physical memory.
-  //
-  // When this function is called we don't know where in the virtual memory
-  // this physical memory will be mapped. So we fake the virtual memory
-  // address by mapping the physical offset into offsets in the reserved
-  // memory space.
-  process_fake_mapping(offset, size, true);
+  MemTracker::allocate_memory_in(ZNMT::_device, untype(offset), size, CALLER_PC, mtJavaHeap);
 }
 
 void ZNMT::uncommit(zoffset offset, size_t size) {
-  // We fake the virtual memory address by mapping the physical offset
-  // into offsets in the reserved memory space.
-  // See comment in ZNMT::commit
-  process_fake_mapping(offset, size, false);
+  MemTracker::free_memory_in(ZNMT::_device, untype(offset), size);
+}
+
+void ZNMT::map(zaddress_unsafe addr, size_t size, zoffset offset) {
+  // NMT doesn't track mappings at the moment.
+}
+
+void ZNMT::unmap(zaddress_unsafe addr, size_t size) {
+  // NMT doesn't track mappings at the moment.
 }

src/hotspot/share/gc/z/zNMT.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,25 +29,24 @@
 #include "gc/z/zMemory.hpp"
 #include "gc/z/zVirtualMemory.hpp"
 #include "memory/allStatic.hpp"
+#include "nmt/memTracker.hpp"
+#include "nmt/memoryFileTracker.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/nativeCallStack.hpp"
 
 class ZNMT : public AllStatic {
 private:
-  struct Reservation {
-    zaddress_unsafe _start;
-    size_t          _size;
-  };
-  static Reservation _reservations[ZMaxVirtualReservations];
-  static size_t      _num_reservations;
-
-  static size_t reservation_index(zoffset offset, size_t* offset_in_reservation);
-  static void process_fake_mapping(zoffset offset, size_t size, bool commit);
+  static MemoryFileTracker::MemoryFile* _device;
 
 public:
+  static void initialize();
+
   static void reserve(zaddress_unsafe start, size_t size);
   static void commit(zoffset offset, size_t size);
   static void uncommit(zoffset offset, size_t size);
+
+  static void map(zaddress_unsafe addr, size_t size, zoffset offset);
+  static void unmap(zaddress_unsafe addr, size_t size);
 };
 
 #endif // SHARE_GC_Z_ZNMT_HPP

src/hotspot/share/nmt/memBaseline.cpp

@@ -137,10 +137,14 @@ class VirtualMemoryAllocationWalker : public VirtualMemoryWalker {
   }
 };
 
-
 void MemBaseline::baseline_summary() {
   MallocMemorySummary::snapshot(&_malloc_memory_snapshot);
   VirtualMemorySummary::snapshot(&_virtual_memory_snapshot);
+  {
+    MemoryFileTracker::Instance::Locker lock;
+    MemoryFileTracker::Instance::summary_snapshot(&_virtual_memory_snapshot);
+  }
+
   _metaspace_stats = MetaspaceUtils::get_combined_statistics();
 }
 
@@ -189,7 +193,6 @@ void MemBaseline::baseline(bool summaryOnly) {
     baseline_allocation_sites();
     _baseline_type = Detail_baselined;
   }
-
 }
 
 int compare_allocation_site(const VirtualMemoryAllocationSite& s1,

src/hotspot/share/nmt/memReporter.cpp

@@ -28,6 +28,7 @@
 #include "nmt/mallocTracker.hpp"
 #include "nmt/memflags.hpp"
 #include "nmt/memReporter.hpp"
+#include "nmt/memoryFileTracker.hpp"
 #include "nmt/threadStackTracker.hpp"
 #include "nmt/virtualMemoryTracker.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -882,4 +883,13 @@ void MemDetailDiffReporter::diff_virtual_memory_site(const NativeCallStack* stac
   }
 
   out->print_cr(")\n");
- }
+}
+
+void MemDetailReporter::report_memory_file_allocations() {
+  stringStream st;
+  {
+    MemoryFileTracker::Instance::Locker lock;
+    MemoryFileTracker::Instance::print_all_reports_on(&st, scale());
+  }
+  output()->print_raw(st.freeze());
+}

src/hotspot/share/nmt/memReporter.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,6 @@
 #include "nmt/memBaseline.hpp"
 #include "nmt/nmtCommon.hpp"
 #include "nmt/virtualMemoryTracker.hpp"
-#include "oops/instanceKlass.hpp"
 
 /*
  * Base class that provides helpers
@@ -165,6 +164,7 @@ class MemDetailReporter : public MemSummaryReporter {
   virtual void report() {
     MemSummaryReporter::report();
     report_virtual_memory_map();
+    report_memory_file_allocations();
     report_detail();
   }
 
@@ -173,6 +173,8 @@ class MemDetailReporter : public MemSummaryReporter {
   void report_detail();
   // Report virtual memory map
   void report_virtual_memory_map();
+  // Report all physical devices
+  void report_memory_file_allocations();
   // Report malloc allocation sites; returns number of omitted sites
   int report_malloc_sites();
   // Report virtual memory reservation sites; returns number of omitted sites

src/hotspot/share/nmt/memTracker.cpp

@@ -67,6 +67,7 @@ void MemTracker::initialize() {
 
   if (level > NMT_off) {
     if (!MallocTracker::initialize(level) ||
+        !MemoryFileTracker::Instance::initialize(level) ||
         !VirtualMemoryTracker::initialize(level)) {
       assert(false, "NMT initialization failed");
       level = NMT_off;

src/hotspot/share/nmt/memTracker.hpp

@@ -27,6 +27,7 @@
 
 #include "nmt/mallocTracker.hpp"
 #include "nmt/nmtCommon.hpp"
+#include "nmt/memoryFileTracker.hpp"
 #include "nmt/threadStackTracker.hpp"
 #include "nmt/virtualMemoryTracker.hpp"
 #include "runtime/mutexLocker.hpp"
@@ -166,6 +167,39 @@ class MemTracker : AllStatic {
     }
   }
 
+  static inline MemoryFileTracker::MemoryFile* register_file(const char* descriptive_name) {
+    assert_post_init();
+    if (!enabled()) return nullptr;
+    MemoryFileTracker::Instance::Locker lock;
+    return MemoryFileTracker::Instance::make_file(descriptive_name);
+  }
+
+  static inline void remove_file(MemoryFileTracker::MemoryFile* file) {
+    assert_post_init();
+    if (!enabled()) return;
+    assert(file != nullptr, "must be");
+    MemoryFileTracker::Instance::Locker lock;
+    MemoryFileTracker::Instance::free_file(file);
+  }
+
+  static inline void allocate_memory_in(MemoryFileTracker::MemoryFile* file, size_t offset, size_t size,
+                                       const NativeCallStack& stack, MEMFLAGS flag) {
+    assert_post_init();
+    if (!enabled()) return;
+    assert(file != nullptr, "must be");
+    MemoryFileTracker::Instance::Locker lock;
+    MemoryFileTracker::Instance::allocate_memory(file, offset, size, stack, flag);
+  }
+
+  static inline void free_memory_in(MemoryFileTracker::MemoryFile* file,
+                                        size_t offset, size_t size) {
+    assert_post_init();
+    if (!enabled()) return;
+    assert(file != nullptr, "must be");
+    MemoryFileTracker::Instance::Locker lock;
+    MemoryFileTracker::Instance::free_memory(file, offset, size);
+  }
+
   // Given an existing memory mapping registered with NMT and a splitting
   //  address, split the mapping in two. The memory region is supposed to
   //  be fully uncommitted.