8331735: UpcallLinker::on_exit races with GC when copying frame anchor

8343144: UpcallLinker::on_entry racingly clears pending exception with GC safepoints
8286875: ProgrammableUpcallHandler::on_entry/on_exit access thread fields from native

Reviewed-by: dholmes, eosterlund, aboldtch

Author: JornVernee

src/hotspot/share/prims/upcallLinker.cpp

@@ -84,16 +84,16 @@ JavaThread* UpcallLinker::on_entry(UpcallStub::FrameData* context) {
   // since it can potentially block.
   context->new_handles = JNIHandleBlock::allocate_block(thread);
 
-  // clear any pending exception in thread (native calls start with no exception pending)
-  thread->clear_pending_exception();
-
   // The call to transition_from_native below contains a safepoint check
   // which needs the code cache to be writable.
   MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, thread));
 
   // After this, we are officially in Java Code. This needs to be done before we change any of the thread local
   // info, since we cannot find oops before the new information is set up completely.
-  ThreadStateTransition::transition_from_native(thread, _thread_in_Java, true /* check_asyncs */);
+  ThreadStateTransition::transition_from_native(thread, _thread_in_Java, false /* check_asyncs */);
+
+  // clear any pending exception in thread, in case someone forgot to check it after a JNI API call.
+  thread->clear_pending_exception();
 
   context->old_handles = thread->active_handles();
 
@@ -119,20 +119,16 @@ void UpcallLinker::on_exit(UpcallStub::FrameData* context) {
   // restore previous handle block
   thread->set_active_handles(context->old_handles);
 
-  thread->frame_anchor()->zap();
-
   debug_only(thread->dec_java_call_counter());
 
+  thread->frame_anchor()->copy(&context->jfa);
+
   // Old thread-local info. has been restored. We are now back in native code.
   ThreadStateTransition::transition_from_java(thread, _thread_in_native);
 
-  thread->frame_anchor()->copy(&context->jfa);
-
   // Release handles after we are marked as being in native code again, since this
   // operation might block
   JNIHandleBlock::release_block(context->new_handles, thread);
-
-  assert(!thread->has_pending_exception(), "Upcall can not throw an exception");
 }
 
 void UpcallLinker::handle_uncaught_exception(oop exception) {