AArch64: Add cross modify fence verification

Author: a74nh

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -4403,10 +4403,15 @@ void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype)
 // Read the polling page.  The address of the polling page must
 // already be in r.
 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
-  InstructionMark im(this);
-  code_section()->relocate(inst_mark(), rtype);
-  ldrw(zr, Address(r, 0));
-  return inst_mark();
+  address mark;
+  {
+    InstructionMark im(this);
+    code_section()->relocate(inst_mark(), rtype);
+    ldrw(zr, Address(r, 0));
+    mark = inst_mark();
+  }
+  verify_cross_modify_fence_not_required();
+  return mark;
 }
 
 void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset) {
@@ -4471,6 +4476,7 @@ void MacroAssembler::build_frame(int framesize) {
       sub(sp, sp, rscratch1);
     }
   }
+  verify_cross_modify_fence_not_required();
 }
 
 void MacroAssembler::remove_frame(int framesize) {
@@ -5298,3 +5304,29 @@ void MacroAssembler::verify_ptrue() {
   stop("Error: the preserved predicate register (p7) elements are not all true");
   bind(verify_ok);
 }
+
+void MacroAssembler::safepoint_isb() {
+  isb();
+#ifndef PRODUCT
+  if (VerifyCrossModifyFence) {
+    // Clear the thread state.
+    strb(zr, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset())));
+  }
+#endif
+}
+
+#ifndef PRODUCT
+void MacroAssembler::verify_cross_modify_fence_not_required() {
+  if (VerifyCrossModifyFence) {
+    // Check if thread needs a cross modify fence.
+    ldrb(rscratch1, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset())));
+    Label fence_not_required;
+    cbz(rscratch1, fence_not_required);
+    // If it does then fail.
+    lea(rscratch1, CAST_FROM_FN_PTR(address, JavaThread::verify_cross_modify_fence_failure));
+    mov(c_rarg0, rthread);
+    blr(rscratch1);
+    bind(fence_not_required);
+  }
+}
+#endif

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1307,7 +1307,7 @@ class MacroAssembler: public Assembler {
   void mul_add(Register out, Register in, Register offs, Register len, Register k);
 
   // Place an ISB after code may have been modified due to a safepoint.
-  void safepoint_isb() { isb(); }
+  void safepoint_isb();
 
 private:
   // Return the effective address r + (r1 << ext) + offset.
@@ -1383,6 +1383,11 @@ class MacroAssembler: public Assembler {
   }
   void cache_wb(Address line);
   void cache_wbsync(bool is_pre);
+
+private:
+  // Check the current thread doesn't need a cross modify fence.
+  void verify_cross_modify_fence_not_required() PRODUCT_RETURN;
+
 };
 
 #ifdef ASSERT

src/hotspot/share/runtime/globals.hpp

@@ -2460,7 +2460,11 @@ const intx ObjectAlignmentInBytes = 8;
                 "Allow allocating fields in empty slots of super-classes")  \
                                                                             \
   product(bool, DeoptimizeNMethodBarriersALot, false, DIAGNOSTIC,           \
-                "Make nmethod barriers deoptimise a lot.")
+                "Make nmethod barriers deoptimise a lot.")                  \
+                                                                            \
+  develop(bool, VerifyCrossModifyFence, false,                              \
+             "Mark all threads after a safepoint, and clear on a modify "   \
+             "fence. Add cleanliness checks.")                              \
 
 // end of RUNTIME_FLAGS
 

src/hotspot/share/runtime/orderAccess.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,10 @@
 #include "precompiled.hpp"
 #include "runtime/orderAccess.hpp"
 #include "runtime/stubRoutines.hpp"
+
+#ifndef PRODUCT
 #include "runtime/thread.hpp"
+#endif
 
 void OrderAccess::StubRoutines_fence() {
   // Use a stub if it exists.  It may not exist during bootstrap so do
@@ -38,3 +41,11 @@ void OrderAccess::StubRoutines_fence() {
   }
   assert(Threads::number_of_threads() == 0, "for bootstrap only");
 }
+
+#ifndef PRODUCT
+void OrderAccess::cross_modify_fence_verify() {
+    if (VerifyCrossModifyFence) {
+      JavaThread::current()->set_requires_cross_modify_fence(false);
+    }
+}
+#endif

src/hotspot/share/runtime/orderAccess.hpp

@@ -258,6 +258,7 @@ class OrderAccess : public AllStatic {
 
   static void     cross_modify_fence() {
     cross_modify_fence_impl();
+    cross_modify_fence_verify();
   }
 
   // Processors which are not multi-copy-atomic require a full fence
@@ -277,6 +278,8 @@ class OrderAccess : public AllStatic {
   static void StubRoutines_fence();
 
   static void cross_modify_fence_impl();
+
+  static void cross_modify_fence_verify() PRODUCT_RETURN;
 };
 
 #include OS_CPU_HEADER(orderAccess)

src/hotspot/share/runtime/safepoint.cpp

@@ -381,6 +381,14 @@ void SafepointSynchronize::begin() {
   assert(_waiting_to_block == 0, "No thread should be running");
 
 #ifndef PRODUCT
+  // Mark all threads
+  if (VerifyCrossModifyFence) {
+    JavaThreadIteratorWithHandle jtiwh;
+    for (; JavaThread *cur = jtiwh.next(); ) {
+      cur->set_requires_cross_modify_fence(true);
+    }
+  }
+
   if (safepoint_limit_time != 0) {
     jlong current_time = os::javaTimeNanos();
     if (safepoint_limit_time < current_time) {

src/hotspot/share/runtime/thread.cpp

@@ -1775,6 +1775,9 @@ JavaThread::JavaThread() :
   ThreadSafepointState::create(this);
 
   SafepointMechanism::initialize_header(this);
+
+  set_requires_cross_modify_fence(false);
+
   pd_initialize();
   assert(deferred_card_mark().is_empty(), "Default MemRegion ctor");
 }
@@ -4959,3 +4962,9 @@ void Threads::verify() {
   VMThread* thread = VMThread::vm_thread();
   if (thread != NULL) thread->verify();
 }
+
+#ifndef PRODUCT
+void JavaThread::verify_cross_modify_fence_failure(JavaThread *thread) {
+   report_vm_error(__FILE__, __LINE__, "Cross modify fence failure", "%p", thread);
+}
+#endif

src/hotspot/share/runtime/thread.hpp

@@ -1109,6 +1109,7 @@ class JavaThread: public Thread {
  private:
   ThreadSafepointState* _safepoint_state;        // Holds information about a thread during a safepoint
   address               _saved_exception_pc;     // Saved pc of instruction where last implicit exception happened
+  NOT_PRODUCT(bool      _requires_cross_modify_fence;) // State used by VerifyCrossModifyFence
 
   // JavaThread termination support
   enum TerminatedTypes {
@@ -1339,6 +1340,8 @@ class JavaThread: public Thread {
 
   SafepointMechanism::ThreadData* poll_data() { return &_poll_data; }
 
+  void set_requires_cross_modify_fence(bool val) PRODUCT_RETURN NOT_PRODUCT({ _requires_cross_modify_fence = val; })
+
  private:
   // Support for thread handshake operations
   HandshakeState _handshake;
@@ -1634,6 +1637,7 @@ class JavaThread: public Thread {
     return byte_offset_of(JavaThread, _should_post_on_exceptions_flag);
   }
   static ByteSize doing_unsafe_access_offset() { return byte_offset_of(JavaThread, _doing_unsafe_access); }
+  NOT_PRODUCT(static ByteSize requires_cross_modify_fence_offset()  { return byte_offset_of(JavaThread, _requires_cross_modify_fence); })
 
   // Returns the jni environment for this thread
   JNIEnv* jni_environment()                      { return &_jni_environment; }
@@ -1923,6 +1927,8 @@ class JavaThread: public Thread {
   bool is_interrupted(bool clear_interrupted);
 
   static OopStorage* thread_oop_storage();
+
+  static void verify_cross_modify_fence_failure(JavaThread *thread) PRODUCT_RETURN;
 };
 
 // Inline implementation of JavaThread::current