[GR-51330] Support per-thread suspend/resume.

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -49,6 +49,7 @@
 import com.oracle.svm.core.jfr.JfrTicks;
 import com.oracle.svm.core.jfr.events.SafepointBeginEvent;
 import com.oracle.svm.core.jfr.events.SafepointEndEvent;
+import com.oracle.svm.core.locks.VMCondition;
 import com.oracle.svm.core.log.Log;
 import com.oracle.svm.core.nodes.CFunctionEpilogueNode;
 import com.oracle.svm.core.nodes.CFunctionPrologueNode;
@@ -173,11 +174,11 @@ public static class Options {
     }
 
     private static long getSafepointPromptnessWarningNanos() {
-        return Options.SafepointPromptnessWarningNanos.getValue().longValue();
+        return Options.SafepointPromptnessWarningNanos.getValue();
     }
 
     private static long getSafepointPromptnessFailureNanos() {
-        return Options.SafepointPromptnessFailureNanos.getValue().longValue();
+        return Options.SafepointPromptnessFailureNanos.getValue();
     }
 
     @Uninterruptible(reason = "Must not contain safepoint checks.")
@@ -219,7 +220,7 @@ private static void slowPathSafepointCheck0(int newStatus, boolean callerHasJava
             assert !ThreadingSupportImpl.isRecurringCallbackRegistered(myself) || ThreadingSupportImpl.isRecurringCallbackPaused();
         } else {
             do {
-                if (Master.singleton().getRequestingThread().isNonNull()) {
+                if (Master.singleton().getRequestingThread().isNonNull() || suspendedTL.getVolatile() > 0) {
                     Statistics.incFrozen();
                     freezeAtSafepoint(newStatus, callerHasJavaFrameAnchor);
                     SafepointListenerSupport.singleton().afterFreezeAtSafepoint();
@@ -343,6 +344,9 @@ private static void freezeAtSafepoint(int newStatus, boolean callerHasJavaFrameA
     @Uninterruptible(reason = "Must not contain safepoint checks.")
     private static void notInlinedLockNoTransition() {
         VMThreads.THREAD_MUTEX.lockNoTransition();
+        while (suspendedTL.get() > 0) {
+            COND_SUSPEND.blockNoTransition();
+        }
     }
 
     /**
@@ -404,6 +408,16 @@ public static int getThreadLocalSafepointRequestedOffset() {
         return VMThreadLocalInfos.getOffset(safepointRequested);
     }
 
+    /**
+     * The possible value is {@code 0} (not suspended), or positive (suspended, possibly blocked on
+     * {@link #COND_SUSPEND}). This counter may only be modified while holding the
+     * {@link VMThreads#THREAD_MUTEX}.
+     */
+    static final FastThreadLocalInt suspendedTL = FastThreadLocalFactory.createInt("Safepoint.suspended");
+
+    /** Condition on which to block when suspended. */
+    static final VMCondition COND_SUSPEND = new VMCondition(VMThreads.THREAD_MUTEX);
+
     /** Foreign call: {@link #ENTER_SLOW_PATH_SAFEPOINT_CHECK}. */
     @SubstrateForeignCallTarget(stubCallingConvention = true)
     @Uninterruptible(reason = "Must not contain safepoint checks")
@@ -581,7 +595,7 @@ public static Master singleton() {
          * safepoint and Java allocations are disabled as well.
          */
         @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "The safepoint logic must not allocate.")
-        protected boolean freeze(String reason) {
+        boolean freeze(String reason) {
             assert VMOperationControl.mayExecuteVmOperations();
             long startTicks = JfrTicks.elapsedTicks();
 
@@ -607,11 +621,11 @@ protected boolean freeze(String reason) {
 
         /** Let all threads proceed from their safepoint. */
         @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "The safepoint logic must not allocate.")
-        protected void thaw(String reason, boolean unlock) {
+        void thaw(boolean unlock) {
             assert VMOperationControl.mayExecuteVmOperations();
             long startTicks = JfrTicks.elapsedTicks();
             safepointState = NOT_AT_SAFEPOINT;
-            releaseSafepoints(reason);
+            releaseSafepoints();
             SafepointEndEvent.emit(getSafepointId(), startTicks);
             ImageSingletons.lookup(Heap.class).endSafepoint();
             Statistics.setThawedNanos();
@@ -819,18 +833,18 @@ private static void waitForSafepoints(String reason) {
         }
 
         /** Release each thread at a safepoint. */
-        private static void releaseSafepoints(String reason) {
-            final Log trace = Log.noopLog().string("[Safepoint.Master.releaseSafepoints:").string("  reason: ").string(reason).newline();
+        private static void releaseSafepoints() {
             assert VMThreads.THREAD_MUTEX.isOwner() : "Must hold mutex when releasing safepoints.";
             // Set all the thread statuses that are at safepoint back to being in native code.
             for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) {
                 if (!isMyself(vmThread) && !SafepointBehavior.ignoresSafepoints(vmThread)) {
-                    if (trace.isEnabled()) {
-                        trace.string("  vmThread status: ").string(StatusSupport.getStatusString(vmThread));
-                    }
-
                     restoreSafepointRequestedValue(vmThread);
 
+                    /* Skip suspended threads so that they remain in STATUS_IN_SAFEPOINT. */
+                    if (suspendedTL.get(vmThread) > 0) {
+                        continue;
+                    }
+
                     /*
                      * Release the thread back to native code. Most threads will transition from
                      * safepoint to native; but some threads will already be in native code if they
@@ -839,21 +853,17 @@ private static void releaseSafepoints(String reason) {
                      */
                     StatusSupport.setStatusNative(vmThread);
                     Statistics.incReleased();
-                    if (trace.isEnabled()) {
-                        trace.string("  ->  ").string(StatusSupport.getStatusString(vmThread)).newline();
-                    }
                 }
             }
-            trace.string("]").newline();
         }
 
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-        protected IsolateThread getRequestingThread() {
+        IsolateThread getRequestingThread() {
             return requestingThread;
         }
 
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-        protected boolean isFrozen() {
+        boolean isFrozen() {
             return safepointState == AT_SAFEPOINT;
         }
 
@@ -1040,7 +1050,7 @@ public static void incSlowPathThawed() {
             }
         }
 
-        public static Log toLog(Log log, boolean newLine, String prefix) {
+        public static void toLog(Log log, boolean newLine, String prefix) {
             if (log.isEnabled() && Options.GatherSafepointStatistics.getValue()) {
                 if (newLine) {
                     log.newline();
@@ -1057,7 +1067,6 @@ public static Log toLog(Log log, boolean newLine, String prefix) {
                 log.string("  slowPathFrozen: ").signed(getSlowPathFrozen()).newline();
                 log.string("  slowPathThawed: ").signed(getSlowPathThawed()).string("]").newline();
             }
-            return log;
         }
     }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.thread;
+
+import org.graalvm.nativeimage.IsolateThread;
+
+import com.oracle.svm.core.heap.VMOperationInfos;
+import com.oracle.svm.core.thread.VMThreads.StatusSupport;
+import com.oracle.svm.core.util.VMError;
+
+/**
+ * Support for suspending and resuming threads.
+ * <p>
+ * Once a thread is suspended, we guarantee that it won't execute any Java code until it is resumed
+ * explicitly. However, suspended threads may still execute native code (including uninterruptible
+ * AOT-compiled Native Image code). Such threads will get blocked when they try to change their
+ * thread status to {@link StatusSupport#STATUS_IN_JAVA} (forced safepoint slow-path).
+ * <p>
+ * After a safepoint, suspended threads remain in {@link StatusSupport#STATUS_IN_SAFEPOINT}, even
+ * though there is no global safepoint at the moment. The suspends and resumes are counted. A thread
+ * needs to be resumed the same number of times it was suspended to continue again.
+ * <p>
+ * Note that there are races possible, for example:
+ * <ul>
+ * <li>Thread A starts detaching.</li>
+ * <li>Thread B suspends thread A.</li>
+ * <li>Thread A exits without ever being suspended because it is already in uninterruptible code and
+ * does not enter the safepoint slowpath anymore.</li>
+ * </ul>
+ *
+ * or
+ *
+ * <ul>
+ * <li>Thread A wants to suspend thread B and requests a VM operation.</li>
+ * <li>Thread B exits before the VM operation starts.</li>
+ * </ul>
+ */
+public final class ThreadSuspendSupport {
+    private ThreadSuspendSupport() {
+    }
+
+    /**
+     * Suspends a thread. If the thread was already suspended, only the suspension counter will be
+     * incremented.
+     */
+    public static void suspend(Thread thread) {
+        VMError.guarantee(!thread.isVirtual(), "must not be called for virtual threads");
+        ThreadSuspendOperation op = new ThreadSuspendOperation(thread);
+        op.enqueue();
+    }
+
+    /**
+     * Decrements the suspension counter of the thread and resumes the thread if the counter reaches
+     * 0.
+     */
+    public static void resume(Thread thread) {
+        VMError.guarantee(!thread.isVirtual(), "must not be called for virtual threads");
+        ThreadResumeOperation op = new ThreadResumeOperation(thread);
+        op.enqueue();
+    }
+
+    private static void suspend(IsolateThread isolateThread) {
+        VMThreads.guaranteeOwnsThreadMutex("Must own the THREAD_MUTEX to prevent races.");
+
+        int newValue = Safepoint.suspendedTL.get(isolateThread) + 1;
+        VMError.guarantee(newValue > 0, "Too many thread suspends.");
+        Safepoint.suspendedTL.set(isolateThread, newValue);
+    }
+
+    private static void resume(IsolateThread isolateThread) {
+        VMThreads.guaranteeOwnsThreadMutex("Must own the THREAD_MUTEX to prevent races.");
+
+        int newValue = Safepoint.suspendedTL.get(isolateThread) - 1;
+        VMError.guarantee(newValue >= 0, "Only a suspended thread can be resumed.");
+        Safepoint.suspendedTL.set(isolateThread, newValue);
+
+        if (newValue == 0) {
+            Safepoint.COND_SUSPEND.broadcast();
+        }
+    }
+
+    private static class ThreadSuspendOperation extends JavaVMOperation {
+        private final Thread thread;
+
+        ThreadSuspendOperation(Thread thread) {
+            super(VMOperationInfos.get(ThreadSuspendOperation.class, "Thread Suspend", SystemEffect.SAFEPOINT));
+            this.thread = thread;
+        }
+
+        @Override
+        protected void operate() {
+            IsolateThread isolateThread = PlatformThreads.getIsolateThread(thread);
+            if (isolateThread.isNonNull()) {
+                ThreadSuspendSupport.suspend(isolateThread);
+            }
+        }
+    }
+
+    private static class ThreadResumeOperation extends JavaVMOperation {
+        private final Thread thread;
+
+        ThreadResumeOperation(Thread thread) {
+            super(VMOperationInfos.get(ThreadResumeOperation.class, "Thread Resume", SystemEffect.SAFEPOINT));
+            this.thread = thread;
+        }
+
+        @Override
+        protected void operate() {
+            IsolateThread isolateThread = PlatformThreads.getIsolateThread(thread);
+            if (isolateThread.isNonNull()) {
+                ThreadSuspendSupport.resume(isolateThread);
+            }
+        }
+    }
+}

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMOperationControl.java

@@ -576,7 +576,7 @@ private void executeAllQueuedVMOperations() {
                     drain(javaSafepointOperations);
                 } finally {
                     if (startedSafepoint) {
-                        master.thaw(safepointReason, lockedForSafepoint);
+                        master.thaw(lockedForSafepoint);
                     }
                 }
             }