8348420: Shenandoah: Check is_reserved before using ReservedSpace instances

Reviewed-by: ysr

Author: William Kemper

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -166,7 +166,11 @@ static ReservedSpace reserve(size_t size, size_t preferred_page_size) {
     size = align_up(size, alignment);
   }
 
-  return MemoryReserver::reserve(size, alignment, preferred_page_size);
+  const ReservedSpace reserved = MemoryReserver::reserve(size, alignment, preferred_page_size);
+  if (!reserved.is_reserved()) {
+    vm_exit_during_initialization("Could not reserve space");
+  }
+  return reserved;
 }
 
 jint ShenandoahHeap::initialize() {
@@ -386,8 +390,9 @@ jint ShenandoahHeap::initialize() {
 
     if (_collection_set == nullptr) {
       cset_rs = MemoryReserver::reserve(cset_size, cset_align, os::vm_page_size());
-
-      // Maybe Shenandoah wants to check the the memory got reserved here?
+      if (!cset_rs.is_reserved()) {
+        vm_exit_during_initialization("Cannot reserve memory for collection set");
+      }
 
       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
     }