8227766: CheckUnhandledOops is broken in MemAllocator

Save oop created in handle more eagerly, so CheckUnhandledOops doesn't bash it.

Reviewed-by: phh
Backport-of: d35c154

Author: shipilev

src/hotspot/share/classfile/stringTable.cpp

@@ -324,7 +324,7 @@ oop StringTable::intern(Handle string_or_null_h, jchar* name, int len, TRAPS) {
     return found_string;
   }
   return StringTable::the_table()->do_intern(string_or_null_h, name, len,
-                                             hash, CHECK_NULL);
+                                             hash, THREAD);
 }
 
 class StringTableCreateEntry : public StackObj {

src/hotspot/share/gc/shared/memAllocator.cpp

@@ -377,6 +377,10 @@ oop MemAllocator::allocate() const {
     HeapWord* mem = mem_allocate(allocation);
     if (mem != NULL) {
       obj = initialize(mem);
+    } else {
+      // The unhandled oop detector will poison local variable obj,
+      // so reset it to NULL if mem is NULL.
+      obj = NULL;
     }
   }
   return obj;

src/hotspot/share/runtime/javaCalls.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -349,9 +349,6 @@ void JavaCalls::call_helper(JavaValue* result, const methodHandle& method, JavaC
   assert(!SafepointSynchronize::is_at_safepoint(), "call to Java code during VM operation");
   assert(!thread->handle_area()->no_handle_mark_active(), "cannot call out to Java here");
 
-
-  CHECK_UNHANDLED_OOPS_ONLY(thread->clear_unhandled_oops();)
-
 #if INCLUDE_JVMCI
   // Gets the nmethod (if any) that should be called instead of normal target
   nmethod* alternative_target = args->alternative_target();
@@ -398,10 +395,6 @@ void JavaCalls::call_helper(JavaValue* result, const methodHandle& method, JavaC
   BasicType result_type = runtime_type_from(result);
   bool oop_result_flag = (result->get_type() == T_OBJECT || result->get_type() == T_ARRAY);
 
-  // NOTE: if we move the computation of the result_val_address inside
-  // the call to call_stub, the optimizer produces wrong code.
-  intptr_t* result_val_address = (intptr_t*)(result->get_value_addr());
-
   // Find receiver
   Handle receiver = (!method->is_static()) ? args->receiver() : Handle();
 
@@ -439,14 +432,19 @@ void JavaCalls::call_helper(JavaValue* result, const methodHandle& method, JavaC
   { JavaCallWrapper link(method, receiver, result, CHECK);
     { HandleMark hm(thread);  // HandleMark used by HandleMarkCleaner
 
+      // NOTE: if we move the computation of the result_val_address inside
+      // the call to call_stub, the optimizer produces wrong code.
+      intptr_t* result_val_address = (intptr_t*)(result->get_value_addr());
+      intptr_t* parameter_address = args->parameters();
+
       StubRoutines::call_stub()(
         (address)&link,
         // (intptr_t*)&(result->_value), // see NOTE above (compiler problem)
         result_val_address,          // see NOTE above (compiler problem)
         result_type,
         method(),
         entry_point,
-        args->parameters(),
+        parameter_address,
         args->size_of_parameters(),
         CHECK
       );

src/hotspot/share/runtime/unhandledOops.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -56,15 +56,15 @@ void UnhandledOops::dump_oops(UnhandledOops *list) {
 
 // For debugging unhandled oop detector _in the debugger_
 // You don't want to turn it on in compiled code here.
-static bool unhandled_oop_print=0;
+static Thread* unhandled_oop_print = NULL;
 
 void UnhandledOops::register_unhandled_oop(oop* op, address pc) {
   if (!_thread->is_in_stack((address)op))
     return;
 
-  _level ++;
-  if (unhandled_oop_print) {
-    for (int i=0; i<_level; i++) tty->print(" ");
+  _level++;
+  if (unhandled_oop_print == _thread) {
+    for (int i=0; i < _level; i++) tty->print(" ");
     tty->print_cr("r " INTPTR_FORMAT, p2i(op));
   }
   UnhandledOopEntry entry(op, pc);
@@ -99,11 +99,11 @@ void UnhandledOops::allow_unhandled_oop(oop* op) {
 void UnhandledOops::unregister_unhandled_oop(oop* op) {
   if (!_thread->is_in_stack((address)op)) return;
 
-  _level --;
-  if (unhandled_oop_print) {
-    for (int i=0; i<_level; i++) tty->print(" ");
+  if (unhandled_oop_print == _thread) {
+    for (int i=0; i < _level; i++) tty->print(" ");
     tty->print_cr("u " INTPTR_FORMAT, p2i(op));
   }
+  _level--;
 
   int i = _oop_list->find_from_end(op, match_oop_entry);
   assert(i!=-1, "oop not in unhandled_oop_list");

src/hotspot/share/services/gcNotifier.cpp

@@ -159,7 +159,7 @@ static Handle createGcInfo(GCMemoryManager *gcManager, GCStatInfo *gcStatInfo,TR
                           gcInfoklass,
                           vmSymbols::com_sun_management_GcInfo_constructor_signature(),
                           &constructor_args,
-                          CHECK_NH);
+                          THREAD);
 }
 
 void GCNotifier::sendNotification(TRAPS) {

test/hotspot/jtreg/runtime/CheckUnhandledOops/TestOutOfMemory.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8227766
+ * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:+CheckUnhandledOops -Xmx100m TestOutOfMemory
+ */
+
+public class TestOutOfMemory {
+    public static void main(java.lang.String[] unused) {
+        final int BIG = 0x100000;
+        // Getting OOM breaks the unhandled oop detector
+        try {
+            int[][] X = new int[BIG][];
+            for (int i = 0; i < BIG; i++) {
+                X[i] = new int[BIG];
+                System.out.println("length = " + X.length);
+            }
+         } catch (OutOfMemoryError oom) {
+            System.out.println("OOM expected");
+         }
+    }
+}