8332865: ubsan: os::attempt_reserve_memory_between reports overflow

Reviewed-by: stuefe, clanger

Author: MBaesken

src/hotspot/share/runtime/os.cpp

@@ -1935,7 +1935,11 @@ char* os::attempt_reserve_memory_between(char* min, char* max, size_t bytes, siz
     return nullptr; // overflow
   }
 
-  char* const hi_att = align_down(MIN2(max, absolute_max) - bytes, alignment_adjusted);
+  char* const hi_end = MIN2(max, absolute_max);
+  if ((uintptr_t)hi_end < bytes) {
+    return nullptr; // no need to go on
+  }
+  char* const hi_att = align_down(hi_end - bytes, alignment_adjusted);
   if (hi_att > max) {
     return nullptr; // overflow
   }