8293291: Simplify relocation of native pointers in archive heap

Reviewed-by: ccheung, coleenp

Author: iklam

src/hotspot/share/cds/archiveBuilder.cpp

@@ -1166,8 +1166,8 @@ void ArchiveBuilder::clean_up_src_obj_table() {
 void ArchiveBuilder::write_archive(FileMapInfo* mapinfo,
                                    GrowableArray<MemRegion>* closed_heap_regions,
                                    GrowableArray<MemRegion>* open_heap_regions,
-                                   GrowableArray<ArchiveHeapOopmapInfo>* closed_heap_oopmaps,
-                                   GrowableArray<ArchiveHeapOopmapInfo>* open_heap_oopmaps) {
+                                   GrowableArray<ArchiveHeapBitmapInfo>* closed_heap_bitmaps,
+                                   GrowableArray<ArchiveHeapBitmapInfo>* open_heap_bitmaps) {
   // Make sure NUM_CDS_REGIONS (exported in cds.h) agrees with
   // MetaspaceShared::n_regions (internal to hotspot).
   assert(NUM_CDS_REGIONS == MetaspaceShared::n_regions, "sanity");
@@ -1176,18 +1176,18 @@ void ArchiveBuilder::write_archive(FileMapInfo* mapinfo,
   write_region(mapinfo, MetaspaceShared::ro, &_ro_region, /*read_only=*/true, /*allow_exec=*/false);
 
   size_t bitmap_size_in_bytes;
-  char* bitmap = mapinfo->write_bitmap_region(ArchivePtrMarker::ptrmap(), closed_heap_oopmaps, open_heap_oopmaps,
+  char* bitmap = mapinfo->write_bitmap_region(ArchivePtrMarker::ptrmap(), closed_heap_bitmaps, open_heap_bitmaps,
                                               bitmap_size_in_bytes);
 
   if (closed_heap_regions != NULL) {
     _total_closed_heap_region_size = mapinfo->write_heap_regions(
                                         closed_heap_regions,
-                                        closed_heap_oopmaps,
+                                        closed_heap_bitmaps,
                                         MetaspaceShared::first_closed_heap_region,
                                         MetaspaceShared::max_num_closed_heap_regions);
     _total_open_heap_region_size = mapinfo->write_heap_regions(
                                         open_heap_regions,
-                                        open_heap_oopmaps,
+                                        open_heap_bitmaps,
                                         MetaspaceShared::first_open_heap_region,
                                         MetaspaceShared::max_num_open_heap_regions);
   }

src/hotspot/share/cds/archiveBuilder.hpp

@@ -36,7 +36,7 @@
 #include "utilities/resizeableResourceHash.hpp"
 #include "utilities/resourceHash.hpp"
 
-struct ArchiveHeapOopmapInfo;
+struct ArchiveHeapBitmapInfo;
 class CHeapBitMap;
 class FileMapInfo;
 class Klass;
@@ -406,8 +406,8 @@ class ArchiveBuilder : public StackObj {
   void write_archive(FileMapInfo* mapinfo,
                      GrowableArray<MemRegion>* closed_heap_regions,
                      GrowableArray<MemRegion>* open_heap_regions,
-                     GrowableArray<ArchiveHeapOopmapInfo>* closed_heap_oopmaps,
-                     GrowableArray<ArchiveHeapOopmapInfo>* open_heap_oopmaps);
+                     GrowableArray<ArchiveHeapBitmapInfo>* closed_heap_oopmaps,
+                     GrowableArray<ArchiveHeapBitmapInfo>* open_heap_oopmaps);
   void write_region(FileMapInfo* mapinfo, int region_idx, DumpRegion* dump_region,
                     bool read_only,  bool allow_exec);
 

src/hotspot/share/cds/archiveHeapLoader.cpp

@@ -72,15 +72,14 @@ void ArchiveHeapLoader::fixup_regions() {
   if (is_mapped()) {
     mapinfo->fixup_mapped_heap_regions();
   } else if (_loading_failed) {
-    fill_failed_loaded_region();
+    fill_failed_loaded_heap();
   }
   if (is_fully_available()) {
     if (!MetaspaceShared::use_full_module_graph()) {
       // Need to remove all the archived java.lang.Module objects from HeapShared::roots().
       ClassLoaderDataShared::clear_archived_oops();
     }
   }
-  SystemDictionaryShared::update_archived_mirror_native_pointers();
 }
 
 // ------------------ Support for Region MAPPING -----------------------------------------
@@ -171,7 +170,7 @@ struct LoadedArchiveHeapRegion {
 };
 
 void ArchiveHeapLoader::init_loaded_heap_relocation(LoadedArchiveHeapRegion* loaded_regions,
-                                             int num_loaded_regions) {
+                                                    int num_loaded_regions) {
   _dumptime_base_0 = loaded_regions[0]._dumptime_base;
   _dumptime_base_1 = loaded_regions[1]._dumptime_base;
   _dumptime_base_2 = loaded_regions[2]._dumptime_base;
@@ -314,7 +313,7 @@ bool ArchiveHeapLoader::load_regions(FileMapInfo* mapinfo, LoadedArchiveHeapRegi
 
     if (!mapinfo->read_region(ri->_region_index, (char*)load_address, r->used(), /* do_commit = */ false)) {
       // There's no easy way to free the buffer, so we will fill it with zero later
-      // in fill_failed_loaded_region(), and it will eventually be GC'ed.
+      // in fill_failed_loaded_heap(), and it will eventually be GC'ed.
       log_warning(cds)("Loading of heap region %d has failed. Archived objects are disabled", i);
       _loading_failed = true;
       return false;
@@ -339,6 +338,7 @@ bool ArchiveHeapLoader::load_regions(FileMapInfo* mapinfo, LoadedArchiveHeapRegi
       bm.iterate(&patcher);
     }
 
+    r->set_mapped_base((char*)load_address);
     load_address += r->used();
   }
 
@@ -392,17 +392,24 @@ class VerifyLoadedHeapEmbeddedPointers: public BasicOopIterateClosure {
 
 void ArchiveHeapLoader::finish_initialization() {
   if (is_loaded()) {
-    HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
-    HeapWord* top    = (HeapWord*)_loaded_heap_top;
-
-    MemRegion archive_space = MemRegion(bottom, top);
-    Universe::heap()->complete_loaded_archive_space(archive_space);
+    // These operations are needed only when the heap is loaded (not mapped).
+    finish_loaded_heap();
+    if (VerifyArchivedFields > 0) {
+      verify_loaded_heap();
+    }
   }
+  patch_native_pointers();
+}
 
-  if (VerifyArchivedFields <= 0 || !is_loaded()) {
-    return;
-  }
+void ArchiveHeapLoader::finish_loaded_heap() {
+  HeapWord* bottom = (HeapWord*)_loaded_heap_bottom;
+  HeapWord* top    = (HeapWord*)_loaded_heap_top;
+
+  MemRegion archive_space = MemRegion(bottom, top);
+  Universe::heap()->complete_loaded_archive_space(archive_space);
+}
 
+void ArchiveHeapLoader::verify_loaded_heap() {
   log_info(cds, heap)("Verify all oops and pointers in loaded heap");
 
   ResourceMark rm;
@@ -424,7 +431,7 @@ void ArchiveHeapLoader::finish_initialization() {
   }
 }
 
-void ArchiveHeapLoader::fill_failed_loaded_region() {
+void ArchiveHeapLoader::fill_failed_loaded_heap() {
   assert(_loading_failed, "must be");
   if (_loaded_heap_bottom != 0) {
     assert(_loaded_heap_top != 0, "must be");
@@ -434,4 +441,37 @@ void ArchiveHeapLoader::fill_failed_loaded_region() {
   }
 }
 
+class PatchNativePointers: public BitMapClosure {
+  Metadata** _start;
+
+ public:
+  PatchNativePointers(Metadata** start) : _start(start) {}
+
+  bool do_bit(size_t offset) {
+    Metadata** p = _start + offset;
+    *p = (Metadata*)(address(*p) + MetaspaceShared::relocation_delta());
+    // Currently we have only Klass pointers in heap objects.
+    // This needs to be relaxed when we support other types of native
+    // pointers such as Method.
+    assert(((Klass*)(*p))->is_klass(), "must be");
+    return true;
+  }
+};
+
+void ArchiveHeapLoader::patch_native_pointers() {
+  if (MetaspaceShared::relocation_delta() == 0) {
+    return;
+  }
+
+  for (int i = MetaspaceShared::first_archive_heap_region;
+       i <= MetaspaceShared::last_archive_heap_region; i++) {
+    FileMapRegion* r = FileMapInfo::current_info()->space_at(i);
+    if (r->mapped_base() != NULL && r->has_ptrmap()) {
+      log_info(cds, heap)("Patching native pointers in heap region %d", i);
+      BitMapView bm = r->ptrmap_view();
+      PatchNativePointers patcher((Metadata**)r->mapped_base());
+      bm.iterate(&patcher);
+    }
+  }
+}
 #endif // INCLUDE_CDS_JAVA_HEAP

src/hotspot/share/cds/archiveHeapLoader.hpp

@@ -146,7 +146,10 @@ class ArchiveHeapLoader : AllStatic {
                            int num_loaded_regions, uintptr_t buffer);
   static void init_loaded_heap_relocation(LoadedArchiveHeapRegion* reloc_info,
                                           int num_loaded_regions);
-  static void fill_failed_loaded_region();
+  static void patch_native_pointers();
+  static void finish_loaded_heap();
+  static void verify_loaded_heap();
+  static void fill_failed_loaded_heap();
 
   static bool is_in_loaded_heap(uintptr_t o) {
     return (_loaded_heap_bottom <= o && o < _loaded_heap_top);

src/hotspot/share/cds/filemap.cpp

@@ -1492,6 +1492,29 @@ void FileMapRegion::init(int region_index, size_t mapping_offset, size_t size, b
   _mapped_base = NULL;
 }
 
+void FileMapRegion::init_bitmaps(ArchiveHeapBitmapInfo oopmap, ArchiveHeapBitmapInfo ptrmap) {
+  _oopmap_offset = oopmap._bm_region_offset;
+  _oopmap_size_in_bits = oopmap._size_in_bits;
+
+  _ptrmap_offset = ptrmap._bm_region_offset;
+  _ptrmap_size_in_bits = ptrmap._size_in_bits;
+}
+
+BitMapView FileMapRegion::bitmap_view(bool is_oopmap) {
+  char* bitmap_base = FileMapInfo::current_info()->map_bitmap_region();
+  bitmap_base += is_oopmap ? _oopmap_offset : _ptrmap_offset;
+  size_t size_in_bits = is_oopmap ? _oopmap_size_in_bits : _ptrmap_size_in_bits;
+  return BitMapView((BitMap::bm_word_t*)(bitmap_base), size_in_bits);
+}
+
+BitMapView FileMapRegion::oopmap_view() {
+  return bitmap_view(true);
+}
+
+BitMapView FileMapRegion::ptrmap_view() {
+  assert(has_ptrmap(), "must be");
+  return bitmap_view(false);
+}
 
 static const char* region_name(int region_index) {
   static const char* names[] = {
@@ -1566,41 +1589,41 @@ void FileMapInfo::write_region(int region, char* base, size_t size,
   }
 }
 
-size_t FileMapInfo::set_oopmaps_offset(GrowableArray<ArchiveHeapOopmapInfo>* oopmaps, size_t curr_size) {
-  for (int i = 0; i < oopmaps->length(); i++) {
-    oopmaps->at(i)._offset = curr_size;
-    curr_size += oopmaps->at(i)._oopmap_size_in_bytes;
+size_t FileMapInfo::set_bitmaps_offset(GrowableArray<ArchiveHeapBitmapInfo>* bitmaps, size_t curr_size) {
+  for (int i = 0; i < bitmaps->length(); i++) {
+    bitmaps->at(i)._bm_region_offset = curr_size;
+    curr_size += bitmaps->at(i)._size_in_bytes;
   }
   return curr_size;
 }
 
-size_t FileMapInfo::write_oopmaps(GrowableArray<ArchiveHeapOopmapInfo>* oopmaps, size_t curr_offset, char* buffer) {
-  for (int i = 0; i < oopmaps->length(); i++) {
-    memcpy(buffer + curr_offset, oopmaps->at(i)._oopmap, oopmaps->at(i)._oopmap_size_in_bytes);
-    curr_offset += oopmaps->at(i)._oopmap_size_in_bytes;
+size_t FileMapInfo::write_bitmaps(GrowableArray<ArchiveHeapBitmapInfo>* bitmaps, size_t curr_offset, char* buffer) {
+  for (int i = 0; i < bitmaps->length(); i++) {
+    memcpy(buffer + curr_offset, bitmaps->at(i)._map, bitmaps->at(i)._size_in_bytes);
+    curr_offset += bitmaps->at(i)._size_in_bytes;
   }
   return curr_offset;
 }
 
 char* FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap,
-                                       GrowableArray<ArchiveHeapOopmapInfo>* closed_oopmaps,
-                                       GrowableArray<ArchiveHeapOopmapInfo>* open_oopmaps,
+                                       GrowableArray<ArchiveHeapBitmapInfo>* closed_bitmaps,
+                                       GrowableArray<ArchiveHeapBitmapInfo>* open_bitmaps,
                                        size_t &size_in_bytes) {
   size_t size_in_bits = ptrmap->size();
   size_in_bytes = ptrmap->size_in_bytes();
 
-  if (closed_oopmaps != NULL && open_oopmaps != NULL) {
-    size_in_bytes = set_oopmaps_offset(closed_oopmaps, size_in_bytes);
-    size_in_bytes = set_oopmaps_offset(open_oopmaps, size_in_bytes);
+  if (closed_bitmaps != NULL && open_bitmaps != NULL) {
+    size_in_bytes = set_bitmaps_offset(closed_bitmaps, size_in_bytes);
+    size_in_bytes = set_bitmaps_offset(open_bitmaps, size_in_bytes);
   }
 
   char* buffer = NEW_C_HEAP_ARRAY(char, size_in_bytes, mtClassShared);
   ptrmap->write_to((BitMap::bm_word_t*)buffer, ptrmap->size_in_bytes());
   header()->set_ptrmap_size_in_bits(size_in_bits);
 
-  if (closed_oopmaps != NULL && open_oopmaps != NULL) {
-    size_t curr_offset = write_oopmaps(closed_oopmaps, ptrmap->size_in_bytes(), buffer);
-    write_oopmaps(open_oopmaps, curr_offset, buffer);
+  if (closed_bitmaps != NULL && open_bitmaps != NULL) {
+    size_t curr_offset = write_bitmaps(closed_bitmaps, ptrmap->size_in_bytes(), buffer);
+    write_bitmaps(open_bitmaps, curr_offset, buffer);
   }
 
   write_region(MetaspaceShared::bm, (char*)buffer, size_in_bytes, /*read_only=*/true, /*allow_exec=*/false);
@@ -1639,7 +1662,7 @@ char* FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap,
 //             |
 //             +-- gap
 size_t FileMapInfo::write_heap_regions(GrowableArray<MemRegion>* regions,
-                                       GrowableArray<ArchiveHeapOopmapInfo>* oopmaps,
+                                       GrowableArray<ArchiveHeapBitmapInfo>* bitmaps,
                                        int first_region_id, int max_num_regions) {
   assert(max_num_regions <= 2, "Only support maximum 2 memory regions");
 
@@ -1665,8 +1688,10 @@ size_t FileMapInfo::write_heap_regions(GrowableArray<MemRegion>* regions,
     int region_idx = i + first_region_id;
     write_region(region_idx, start, size, false, false);
     if (size > 0) {
-      space_at(region_idx)->init_oopmap(oopmaps->at(i)._offset,
-                                        oopmaps->at(i)._oopmap_size_in_bits);
+      int oopmap_idx = i * 2;
+      int ptrmap_idx = i * 2 + 1;
+      space_at(region_idx)->init_bitmaps(bitmaps->at(oopmap_idx),
+                                         bitmaps->at(ptrmap_idx));
     }
   }
   return total_size;
@@ -2307,6 +2332,8 @@ bool FileMapInfo::map_heap_regions(int first, int max,  bool is_open_archive,
       log_info(cds)("UseSharedSpaces: mapped heap regions are corrupt");
       return false;
     }
+
+    si->set_mapped_base(base);
   }
 
   cleanup._aborted = false;

src/hotspot/share/cds/filemap.hpp

@@ -39,6 +39,7 @@
 
 static const int JVM_IDENT_MAX = 256;
 
+class BitMapView;
 class CHeapBitMap;
 class ClassFileStream;
 class ClassLoaderData;
@@ -102,11 +103,11 @@ class SharedClassPathEntry {
   }
 };
 
-struct ArchiveHeapOopmapInfo {
-  address _oopmap;               // bitmap for relocating embedded oops
-  size_t  _offset;               // this oopmap is stored at this offset from the bottom of the BM region
-  size_t  _oopmap_size_in_bits;
-  size_t  _oopmap_size_in_bytes;
+struct ArchiveHeapBitmapInfo {
+  address _map;               // bitmap for relocating embedded oops
+  size_t  _bm_region_offset;  // this bitmap is stored at this offset from the bottom of the BM region
+  size_t  _size_in_bits;
+  size_t  _size_in_bytes;
 };
 
 class SharedPathTable {
@@ -137,6 +138,7 @@ class SharedPathTable {
 
 
 class FileMapRegion: private CDSFileMapRegion {
+  BitMapView bitmap_view(bool is_oopmap);
 public:
   void assert_is_heap_region() const {
     assert(_is_heap_region, "must be heap region");
@@ -156,7 +158,7 @@ class FileMapRegion: private CDSFileMapRegion {
   size_t mapping_end_offset()       const { return _mapping_offset + used_aligned(); }
   size_t used()                     const { return _used; }
   size_t used_aligned()             const; // aligned up to MetaspaceShared::core_region_alignment()
-  char*  mapped_base()              const { assert_is_not_heap_region(); return _mapped_base; }
+  char*  mapped_base()              const { return _mapped_base; }
   char*  mapped_end()               const { return mapped_base()        + used_aligned(); }
   bool   read_only()                const { return _read_only != 0; }
   bool   allow_exec()               const { return _allow_exec != 0; }
@@ -170,11 +172,10 @@ class FileMapRegion: private CDSFileMapRegion {
   void set_mapped_from_file(bool v)  { _mapped_from_file = v; }
   void init(int region_index, size_t mapping_offset, size_t size, bool read_only,
             bool allow_exec, int crc);
-
-  void init_oopmap(size_t oopmap_offset, size_t size_in_bits) {
-    _oopmap_offset = oopmap_offset;
-    _oopmap_size_in_bits = size_in_bits;
-  }
+  void init_bitmaps(ArchiveHeapBitmapInfo oopmap, ArchiveHeapBitmapInfo ptrmap);
+  BitMapView oopmap_view();
+  BitMapView ptrmap_view();
+  bool has_ptrmap()                  { return _ptrmap_size_in_bits != 0; }
 
   void print(outputStream* st, int region_index);
 };
@@ -444,11 +445,11 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   void  write_region(int region, char* base, size_t size,
                      bool read_only, bool allow_exec);
   char* write_bitmap_region(const CHeapBitMap* ptrmap,
-                            GrowableArray<ArchiveHeapOopmapInfo>* closed_oopmaps,
-                            GrowableArray<ArchiveHeapOopmapInfo>* open_oopmaps,
+                            GrowableArray<ArchiveHeapBitmapInfo>* closed_bitmaps,
+                            GrowableArray<ArchiveHeapBitmapInfo>* open_bitmaps,
                             size_t &size_in_bytes);
   size_t write_heap_regions(GrowableArray<MemRegion>* regions,
-                            GrowableArray<ArchiveHeapOopmapInfo>* oopmaps,
+                            GrowableArray<ArchiveHeapBitmapInfo>* bitmaps,
                             int first_region_id, int max_num_regions);
   void  write_bytes(const void* buffer, size_t count);
   void  write_bytes_aligned(const void* buffer, size_t count);
@@ -573,8 +574,8 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   void  map_heap_regions_impl() NOT_CDS_JAVA_HEAP_RETURN;
   MapArchiveResult map_region(int i, intx addr_delta, char* mapped_base_address, ReservedSpace rs);
   bool  relocate_pointers_in_core_regions(intx addr_delta);
-  static size_t set_oopmaps_offset(GrowableArray<ArchiveHeapOopmapInfo> *oopmaps, size_t curr_size);
-  static size_t write_oopmaps(GrowableArray<ArchiveHeapOopmapInfo> *oopmaps, size_t curr_offset, char* buffer);
+  static size_t set_bitmaps_offset(GrowableArray<ArchiveHeapBitmapInfo> *bitmaps, size_t curr_size);
+  static size_t write_bitmaps(GrowableArray<ArchiveHeapBitmapInfo> *bitmaps, size_t curr_offset, char* buffer);
 
   address decode_start_address(FileMapRegion* spc, bool with_current_oop_encoding_mode);