8318694: [JVMCI] disable can_call_java in most contexts for libjvmci compiler threads

Reviewed-by: dholmes, never

Author: Doug Simon

src/hotspot/share/classfile/javaClasses.cpp

@@ -2453,7 +2453,7 @@ void java_lang_Throwable::print_stack_trace(Handle throwable, outputStream* st)
       BacktraceElement bte = iter.next(THREAD);
       print_stack_element_to_stream(st, bte._mirror, bte._method_id, bte._version, bte._bci, bte._name);
     }
-    {
+    if (THREAD->can_call_java()) {
       // Call getCause() which doesn't necessarily return the _cause field.
       ExceptionMark em(THREAD);
       JavaValue cause(T_OBJECT);
@@ -2475,6 +2475,9 @@ void java_lang_Throwable::print_stack_trace(Handle throwable, outputStream* st)
           st->cr();
         }
       }
+    } else {
+      st->print_raw_cr("<<cannot call Java to get cause>>");
+      return;
     }
   }
 }

src/hotspot/share/classfile/systemDictionary.cpp

@@ -611,7 +611,7 @@ InstanceKlass* SystemDictionary::resolve_instance_class_or_null(Symbol* name,
   InstanceKlass* loaded_class = nullptr;
   SymbolHandle superclassname; // Keep alive while loading in parallel thread.
 
-  assert(THREAD->can_call_java(),
+  guarantee(THREAD->can_call_java(),
          "can not load classes with compiler thread: class=%s, classloader=%s",
          name->as_C_string(),
          class_loader.is_null() ? "null" : class_loader->klass()->name()->as_C_string());
@@ -2056,7 +2056,7 @@ Method* SystemDictionary::find_method_handle_invoker(Klass* klass,
                                                      Klass* accessing_klass,
                                                      Handle* appendix_result,
                                                      TRAPS) {
-  assert(THREAD->can_call_java() ,"");
+  guarantee(THREAD->can_call_java(), "");
   Handle method_type =
     SystemDictionary::find_method_handle_type(signature, accessing_klass, CHECK_NULL);
 

src/hotspot/share/compiler/compilerThread.cpp

@@ -39,6 +39,7 @@ CompilerThread::CompilerThread(CompileQueue* queue,
   _queue = queue;
   _counters = counters;
   _buffer_blob = nullptr;
+  _can_call_java = false;
   _compiler = nullptr;
   _arena_stat = CompilationMemoryStatistic::enabled() ? new ArenaStatCounter : nullptr;
 
@@ -56,15 +57,17 @@ CompilerThread::~CompilerThread() {
   delete _arena_stat;
 }
 
+void CompilerThread::set_compiler(AbstractCompiler* c) {
+  // Only jvmci compiler threads can call Java
+  _can_call_java = c != nullptr && c->is_jvmci();
+  _compiler = c;
+}
+
 void CompilerThread::thread_entry(JavaThread* thread, TRAPS) {
   assert(thread->is_Compiler_thread(), "must be compiler thread");
   CompileBroker::compiler_thread_loop();
 }
 
-bool CompilerThread::can_call_java() const {
-  return _compiler != nullptr && _compiler->is_jvmci();
-}
-
 // Hide native compiler threads from external view.
 bool CompilerThread::is_hidden_from_external_view() const {
   return _compiler == nullptr || _compiler->is_hidden_from_external_view();

src/hotspot/share/compiler/compilerThread.hpp

@@ -31,18 +31,17 @@ class AbstractCompiler;
 class ArenaStatCounter;
 class BufferBlob;
 class ciEnv;
-class CompileThread;
+class CompilerThread;
 class CompileLog;
 class CompileTask;
 class CompileQueue;
 class CompilerCounters;
 class IdealGraphPrinter;
-class JVMCIEnv;
-class JVMCIPrimitiveArray;
 
 // A thread used for Compilation.
 class CompilerThread : public JavaThread {
   friend class VMStructs;
+  JVMCI_ONLY(friend class CompilerThreadCanCallJava;)
  private:
   CompilerCounters* _counters;
 
@@ -51,6 +50,7 @@ class CompilerThread : public JavaThread {
   CompileTask* volatile _task;  // print_threads_compiling can read this concurrently.
   CompileQueue*         _queue;
   BufferBlob*           _buffer_blob;
+  bool                  _can_call_java;
 
   AbstractCompiler*     _compiler;
   TimeStamp             _idle_time;
@@ -73,13 +73,13 @@ class CompilerThread : public JavaThread {
 
   bool is_Compiler_thread() const                { return true; }
 
-  virtual bool can_call_java() const;
+  virtual bool can_call_java() const             { return _can_call_java; }
 
   // Returns true if this CompilerThread is hidden from JVMTI and FlightRecorder.  C1 and C2 are
   // always hidden but JVMCI compiler threads might be hidden.
   virtual bool is_hidden_from_external_view() const;
 
-  void set_compiler(AbstractCompiler* c)         { _compiler = c; }
+  void set_compiler(AbstractCompiler* c);
   AbstractCompiler* compiler() const             { return _compiler; }
 
   CompileQueue* queue()        const             { return _queue; }

src/hotspot/share/jvmci/jvmci.cpp

@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "classfile/systemDictionary.hpp"
+#include "compiler/abstractCompiler.hpp"
 #include "compiler/compileTask.hpp"
 #include "compiler/compilerThread.hpp"
 #include "gc/shared/collectedHeap.hpp"
@@ -53,6 +54,29 @@ volatile intx JVMCI::_fatal_log_init_thread = -1;
 volatile int JVMCI::_fatal_log_fd = -1;
 const char* JVMCI::_fatal_log_filename = nullptr;
 
+CompilerThreadCanCallJava::CompilerThreadCanCallJava(JavaThread* current, bool new_state) {
+  _current = nullptr;
+  if (current->is_Compiler_thread()) {
+    CompilerThread* ct = CompilerThread::cast(current);
+    if (ct->_can_call_java != new_state &&
+        ct->_compiler != nullptr &&
+        ct->_compiler->is_jvmci())
+    {
+      // Only enter a new context if the ability of the
+      // current thread to call Java actually changes
+      _reset_state = ct->_can_call_java;
+      ct->_can_call_java = new_state;
+      _current = ct;
+    }
+  }
+}
+
+CompilerThreadCanCallJava::~CompilerThreadCanCallJava() {
+  if (_current != nullptr) {
+    _current->_can_call_java = _reset_state;
+  }
+}
+
 void jvmci_vmStructs_init() NOT_DEBUG_RETURN;
 
 bool JVMCI::can_initialize_JVMCI() {
@@ -176,6 +200,10 @@ void JVMCI::ensure_box_caches_initialized(TRAPS) {
     java_lang_Long_LongCache::symbol()
   };
 
+  // Class resolution and initialization below
+  // requires calling into Java
+  CompilerThreadCanCallJava ccj(THREAD, true);
+
   for (unsigned i = 0; i < sizeof(box_classes) / sizeof(Symbol*); i++) {
     Klass* k = SystemDictionary::resolve_or_fail(box_classes[i], true, CHECK);
     InstanceKlass* ik = InstanceKlass::cast(k);

src/hotspot/share/jvmci/jvmci.hpp

@@ -29,6 +29,7 @@
 #include "utilities/exceptions.hpp"
 
 class BoolObjectClosure;
+class CompilerThread;
 class constantPoolHandle;
 class JavaThread;
 class JVMCIEnv;
@@ -46,6 +47,34 @@ typedef FormatStringEventLog<256> StringEventLog;
 struct _jmetadata;
 typedef struct _jmetadata *jmetadata;
 
+// A stack object that manages a scope in which the current thread, if
+// it's a CompilerThread, can have its CompilerThread::_can_call_java
+// field changed. This allows restricting libjvmci better in terms
+// of when it can make Java calls. If a Java call on a CompilerThread
+// reaches a clinit, there's a risk of dead-lock when async compilation
+// is disabled (e.g. -Xbatch or -Xcomp) as the non-CompilerThread thread
+// waiting for the blocking compilation may hold the clinit lock.
+//
+// This scope is primarily used to disable Java calls when libjvmci enters
+// the VM via a C2V (i.e. CompilerToVM) native method.
+class CompilerThreadCanCallJava : StackObj {
+ private:
+  CompilerThread* _current; // Only non-null if state of thread changed
+  bool _reset_state;        // Value prior to state change, undefined
+                            // if no state change.
+public:
+  // Enters a scope in which the ability of the current CompilerThread
+  // to call Java is specified by `new_state`. This call only makes a
+  // change if the current thread is a CompilerThread associated with
+  // a JVMCI compiler whose CompilerThread::_can_call_java is not
+  // currently `new_state`.
+  CompilerThreadCanCallJava(JavaThread* current, bool new_state);
+
+  // Resets CompilerThread::_can_call_java of the current thread if the
+  // constructor changed it.
+  ~CompilerThreadCanCallJava();
+};
+
 class JVMCI : public AllStatic {
   friend class JVMCIRuntime;
   friend class JVMCIEnv;

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -165,14 +165,19 @@ Handle JavaArgumentUnboxer::next_arg(BasicType expectedType) {
   MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, thread));       \
   ThreadInVMfromNative __tiv(thread);                             \
   HandleMarkCleaner __hm(thread);                                 \
-  JavaThread* THREAD = thread;                                        \
+  JavaThread* THREAD = thread;                                    \
   debug_only(VMNativeEntryWrapper __vew;)
 
 // Native method block that transitions current thread to '_thread_in_vm'.
-#define C2V_BLOCK(result_type, name, signature)      \
-  JVMCI_VM_ENTRY_MARK;                               \
-  ResourceMark rm;                                   \
-  JVMCIENV_FROM_JNI(JVMCI::compilation_tick(thread), env);
+// Note: CompilerThreadCanCallJava must precede JVMCIENV_FROM_JNI so that
+// the translation of an uncaught exception in the JVMCIEnv does not make
+// a Java call when __is_hotspot == false.
+#define C2V_BLOCK(result_type, name, signature)            \
+  JVMCI_VM_ENTRY_MARK;                                     \
+  ResourceMark rm;                                         \
+  bool __is_hotspot = env == thread->jni_environment();    \
+  CompilerThreadCanCallJava ccj(thread, __is_hotspot);     \
+  JVMCIENV_FROM_JNI(JVMCI::compilation_tick(thread), env); \
 
 static JavaThread* get_current_thread(bool allow_null=true) {
   Thread* thread = Thread::current_or_null_safe();
@@ -188,7 +193,7 @@ static JavaThread* get_current_thread(bool allow_null=true) {
 #define C2V_VMENTRY(result_type, name, signature)        \
   JNIEXPORT result_type JNICALL c2v_ ## name signature { \
   JavaThread* thread = get_current_thread();             \
-  if (thread == nullptr) {                                  \
+  if (thread == nullptr) {                               \
     env->ThrowNew(JNIJVMCI::InternalError::clazz(),      \
         err_msg("Cannot call into HotSpot from JVMCI shared library without attaching current thread")); \
     return;                                              \
@@ -199,7 +204,7 @@ static JavaThread* get_current_thread(bool allow_null=true) {
 #define C2V_VMENTRY_(result_type, name, signature, result) \
   JNIEXPORT result_type JNICALL c2v_ ## name signature { \
   JavaThread* thread = get_current_thread();             \
-  if (thread == nullptr) {                                  \
+  if (thread == nullptr) {                               \
     env->ThrowNew(JNIJVMCI::InternalError::clazz(),      \
         err_msg("Cannot call into HotSpot from JVMCI shared library without attaching current thread")); \
     return result;                                       \
@@ -221,15 +226,15 @@ static JavaThread* get_current_thread(bool allow_null=true) {
 #define JNI_THROW(caller, name, msg) do {                                         \
     jint __throw_res = env->ThrowNew(JNIJVMCI::name::clazz(), msg);               \
     if (__throw_res != JNI_OK) {                                                  \
-      tty->print_cr("Throwing " #name " in " caller " returned %d", __throw_res); \
+      JVMCI_event_1("Throwing " #name " in " caller " returned %d", __throw_res); \
     }                                                                             \
     return;                                                                       \
   } while (0);
 
 #define JNI_THROW_(caller, name, msg, result) do {                                \
     jint __throw_res = env->ThrowNew(JNIJVMCI::name::clazz(), msg);               \
     if (__throw_res != JNI_OK) {                                                  \
-      tty->print_cr("Throwing " #name " in " caller " returned %d", __throw_res); \
+      JVMCI_event_1("Throwing " #name " in " caller " returned %d", __throw_res); \
     }                                                                             \
     return result;                                                                \
   } while (0)
@@ -579,6 +584,7 @@ C2V_VMENTRY_0(jboolean, shouldInlineMethod,(JNIEnv* env, jobject, ARGUMENT_PAIR(
 C2V_END
 
 C2V_VMENTRY_NULL(jobject, lookupType, (JNIEnv* env, jobject, jstring jname, ARGUMENT_PAIR(accessing_klass), jint accessing_klass_loader, jboolean resolve))
+  CompilerThreadCanCallJava canCallJava(thread, resolve); // Resolution requires Java calls
   JVMCIObject name = JVMCIENV->wrap(jname);
   const char* str = JVMCIENV->as_utf8_string(name);
   TempNewSymbol class_name = SymbolTable::new_symbol(str);
@@ -592,7 +598,7 @@ C2V_VMENTRY_NULL(jobject, lookupType, (JNIEnv* env, jobject, jstring jname, ARGU
   if (val != nullptr) {
     if (strstr(val, "<trace>") != nullptr) {
       tty->print_cr("CompilerToVM.lookupType: %s", str);
-    } else if (strstr(val, str) != nullptr) {
+    } else if (strstr(str, val) != nullptr) {
       THROW_MSG_0(vmSymbols::java_lang_Exception(),
                   err_msg("lookupTypeException: %s", str));
     }
@@ -938,6 +944,17 @@ C2V_VMENTRY_NULL(jobject, resolveFieldInPool, (JNIEnv* env, jobject, ARGUMENT_PA
   Bytecodes::Code code = (Bytecodes::Code)(((int) opcode) & 0xFF);
   fieldDescriptor fd;
   methodHandle mh(THREAD, UNPACK_PAIR(Method, method));
+
+  Bytecodes::Code bc = (Bytecodes::Code) (((int) opcode) & 0xFF);
+  int holder_index = cp->klass_ref_index_at(index, bc);
+  if (!cp->tag_at(holder_index).is_klass() && !THREAD->can_call_java()) {
+    // If the holder is not resolved in the constant pool and the current
+    // thread cannot call Java, return null. This avoids a Java call
+    // in LinkInfo to load the holder.
+    Symbol* klass_name = cp->klass_ref_at_noresolve(index, bc);
+    return nullptr;
+  }
+
   LinkInfo link_info(cp, index, mh, code, CHECK_NULL);
   LinkResolver::resolve_field(fd, link_info, Bytecodes::java_code(code), false, CHECK_NULL);
   JVMCIPrimitiveArray info = JVMCIENV->wrap(info_handle);
@@ -2726,6 +2743,7 @@ C2V_VMENTRY_0(jlong, translate, (JNIEnv* env, jobject, jobject obj_handle, jbool
     return 0L;
   }
   PEER_JVMCIENV_FROM_THREAD(THREAD, !JVMCIENV->is_hotspot());
+  CompilerThreadCanCallJava canCallJava(thread, PEER_JVMCIENV->is_hotspot());
   PEER_JVMCIENV->check_init(JVMCI_CHECK_0);
 
   JVMCIEnv* thisEnv = JVMCIENV;
@@ -2945,18 +2963,21 @@ static jbyteArray get_encoded_annotation_data(InstanceKlass* holder, AnnotationA
 
 C2V_VMENTRY_NULL(jbyteArray, getEncodedClassAnnotationData, (JNIEnv* env, jobject, ARGUMENT_PAIR(klass),
                  jobject filter, jint filter_length, jlong filter_klass_pointers))
+  CompilerThreadCanCallJava canCallJava(thread, true); // Requires Java support
   InstanceKlass* holder = InstanceKlass::cast(UNPACK_PAIR(Klass, klass));
   return get_encoded_annotation_data(holder, holder->class_annotations(), true, filter_length, filter_klass_pointers, THREAD, JVMCIENV);
 C2V_END
 
 C2V_VMENTRY_NULL(jbyteArray, getEncodedExecutableAnnotationData, (JNIEnv* env, jobject, ARGUMENT_PAIR(method),
                  jobject filter, jint filter_length, jlong filter_klass_pointers))
+  CompilerThreadCanCallJava canCallJava(thread, true); // Requires Java support
   methodHandle method(THREAD, UNPACK_PAIR(Method, method));
   return get_encoded_annotation_data(method->method_holder(), method->annotations(), false, filter_length, filter_klass_pointers, THREAD, JVMCIENV);
 C2V_END
 
 C2V_VMENTRY_NULL(jbyteArray, getEncodedFieldAnnotationData, (JNIEnv* env, jobject, ARGUMENT_PAIR(klass), jint index,
                  jobject filter, jint filter_length, jlong filter_klass_pointers))
+  CompilerThreadCanCallJava canCallJava(thread, true); // Requires Java support
   InstanceKlass* holder = check_field(InstanceKlass::cast(UNPACK_PAIR(Klass, klass)), index, JVMCIENV);
   fieldDescriptor fd(holder, index);
   return get_encoded_annotation_data(holder, fd.annotations(), false, filter_length, filter_klass_pointers, THREAD, JVMCIENV);
@@ -3013,6 +3034,7 @@ C2V_VMENTRY_0(jboolean, addFailedSpeculation, (JNIEnv* env, jobject, jlong faile
 C2V_END
 
 C2V_VMENTRY(void, callSystemExit, (JNIEnv* env, jobject, jint status))
+  CompilerThreadCanCallJava canCallJava(thread, true);
   JavaValue result(T_VOID);
   JavaCallArguments jargs(1);
   jargs.push_int(status);

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -448,6 +448,15 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
  private:
   const Handle& _throwable;
 
+  char* print_throwable_to_buffer(Handle throwable, jlong buffer, int buffer_size) {
+    char* char_buffer = (char*) buffer + 4;
+    stringStream st(char_buffer, (size_t) buffer_size - 4);
+    java_lang_Throwable::print_stack_trace(throwable, &st);
+    u4 len = (u4) st.size();
+    *((u4*) buffer) = len;
+    return char_buffer;
+  }
+
   bool handle_pending_exception(JavaThread* THREAD, jlong buffer, int buffer_size) {
     if (HAS_PENDING_EXCEPTION) {
       Handle throwable = Handle(THREAD, PENDING_EXCEPTION);
@@ -457,11 +466,7 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
         JVMCI_event_1("error translating exception: OutOfMemoryError");
         decode(THREAD, _encode_oome_fail, 0L);
       } else {
-        char* char_buffer = (char*) buffer + 4;
-        stringStream st(char_buffer, (size_t) buffer_size - 4);
-        java_lang_Throwable::print_stack_trace(throwable, &st);
-        u4 len = (u4) st.size();
-        *((u4*) buffer) = len;
+        char* char_buffer = print_throwable_to_buffer(throwable, buffer, buffer_size);
         JVMCI_event_1("error translating exception: %s", char_buffer);
         decode(THREAD, _encode_fail, buffer);
       }
@@ -471,6 +476,13 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
   }
 
   int encode(JavaThread* THREAD, jlong buffer, int buffer_size) {
+    if (!THREAD->can_call_java()) {
+      char* char_buffer = print_throwable_to_buffer(_throwable, buffer, buffer_size);
+      const char* detail = log_is_enabled(Info, exceptions) ? "" : " (-Xlog:exceptions may give more detail)";
+      JVMCI_event_1("cannot call Java to translate exception%s: %s", detail, char_buffer);
+      decode(THREAD, _encode_fail, buffer);
+      return 0;
+    }
     Klass* vmSupport = SystemDictionary::resolve_or_fail(vmSymbols::jdk_internal_vm_VMSupport(), true, THREAD);
     if (handle_pending_exception(THREAD, buffer, buffer_size)) {
       return 0;
@@ -1311,6 +1323,7 @@ JVMCIObject JVMCIEnv::get_jvmci_type(const JVMCIKlassHandle& klass, JVMCI_TRAPS)
   JavaThread* THREAD = JVMCI::compilation_tick(JavaThread::current()); // For exception macros.
   jboolean exception = false;
   if (is_hotspot()) {
+    CompilerThreadCanCallJava ccj(THREAD, true);
     JavaValue result(T_OBJECT);
     JavaCallArguments args;
     args.push_long(pointer);