From 38aa07e0a5853ab0c254a20aa4f2775ff21ab923 Mon Sep 17 00:00:00 2001 From: Martin Entlicher Date: Fri, 19 Jan 2024 09:13:10 +0100 Subject: [PATCH 1/2] Support for per-thread suspend/resume on a safepoint. (GR-51330) --- .../com/oracle/svm/core/thread/Safepoint.java | 21 ++++- .../svm/core/thread/ThreadSuspendSupport.java | 76 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 598ef797ad10..635c41157401 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -49,6 +49,7 @@ import com.oracle.svm.core.jfr.JfrTicks; import com.oracle.svm.core.jfr.events.SafepointBeginEvent; import com.oracle.svm.core.jfr.events.SafepointEndEvent; +import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; @@ -219,7 +220,7 @@ private static void slowPathSafepointCheck0(int newStatus, boolean callerHasJava assert !ThreadingSupportImpl.isRecurringCallbackRegistered(myself) || ThreadingSupportImpl.isRecurringCallbackPaused(); } else { do { - if (Master.singleton().getRequestingThread().isNonNull()) { + if (Master.singleton().getRequestingThread().isNonNull() || safepointSuspend.getVolatile() > 0) { Statistics.incFrozen(); freezeAtSafepoint(newStatus, callerHasJavaFrameAnchor); SafepointListenerSupport.singleton().afterFreezeAtSafepoint(); @@ -343,6 +344,9 @@ private static void freezeAtSafepoint(int newStatus, boolean callerHasJavaFrameA @Uninterruptible(reason = "Must not contain safepoint checks.") private static void notInlinedLockNoTransition() { VMThreads.THREAD_MUTEX.lockNoTransition(); + while (safepointSuspend.get() > 0) { + COND_SUSPEND.blockNoTransition(); + } } /** @@ -404,6 +408,15 @@ public static int getThreadLocalSafepointRequestedOffset() { return VMThreadLocalInfos.getOffset(safepointRequested); } + /** + * Per-thread counter for safepoint suspends. The possible value is {@code 0} (not suspended), + * or positive (blocked on {@link #COND_SUSPEND}). + */ + static final FastThreadLocalInt safepointSuspend = FastThreadLocalFactory.createInt("Safepoint.safepointSuspend"); + + /** Condition on which to block when suspended. */ + static final VMCondition COND_SUSPEND = new VMCondition(VMThreads.THREAD_MUTEX); + /** Foreign call: {@link #ENTER_SLOW_PATH_SAFEPOINT_CHECK}. */ @SubstrateForeignCallTarget(stubCallingConvention = true) @Uninterruptible(reason = "Must not contain safepoint checks") @@ -829,6 +842,10 @@ private static void releaseSafepoints(String reason) { trace.string(" vmThread status: ").string(StatusSupport.getStatusString(vmThread)); } + if (safepointSuspend.get() > 0) { + continue; + } + restoreSafepointRequestedValue(vmThread); /* diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java new file mode 100644 index 000000000000..10b237d90fe6 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.thread; + +import com.oracle.svm.core.thread.VMThreads.StatusSupport; +import com.oracle.svm.core.util.VMError; + +import org.graalvm.nativeimage.IsolateThread; + +/** + * Per-thread safepoint suspend/resume. The thread suspends on the current, or the first met + * safepoint. + *

+ * Suspended threads remain in {@link StatusSupport#STATUS_IN_SAFEPOINT} (even though there is no + * global safepoint at the moment). Threads that get suspended while they execute native code will + * only stop once they enter Java code again (forced safepoint slow-path). + *

+ * The suspends and resumes are counted. The thread needs to be resumed the same number of times it + * was suspended to continue again. The suspension counter is only modified in a safepoint, i.e., + * all modifications are protected by the {@link VMThreads#THREAD_MUTEX}. + */ +public final class ThreadSuspendSupport { + + private ThreadSuspendSupport() { + } + + /** + * Suspend the thread on a safepoint, or increase the suspend count when suspended already. Need + * to be called under the {@link VMThreads#THREAD_MUTEX}. + */ + public static void suspend(IsolateThread vmThread) { + VMThreads.guaranteeOwnsThreadMutex("Must own the THREAD_MUTEX to prevent races."); + int suspendCounter = Safepoint.safepointSuspend.get(vmThread); + if (suspendCounter == Integer.MAX_VALUE) { + throw VMError.shouldNotReachHere("Too many thread suspends."); + } + Safepoint.safepointSuspend.set(vmThread, suspendCounter + 1); + } + + /** + * Decrements suspend count of the thread and resume when decremented to 0. Need to be called + * under the {@link VMThreads#THREAD_MUTEX}. + */ + public static void resume(IsolateThread vmThread) { + VMThreads.guaranteeOwnsThreadMutex("Must own the THREAD_MUTEX to prevent races."); + int suspendCounter = Safepoint.safepointSuspend.get(vmThread); + if (suspendCounter == 0) { + // Is resumed already + throw VMError.shouldNotReachHere("The thread is not suspended."); + } + Safepoint.safepointSuspend.set(vmThread, suspendCounter - 1); + Safepoint.COND_SUSPEND.broadcast(); + } +} From 1ee6cf324a80e5109e61a8c135f297589e16a097 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 2 Feb 2024 20:40:14 +0100 Subject: [PATCH 2/2] Cleanups and a small fix. --- .../com/oracle/svm/core/thread/Safepoint.java | 44 +++---- .../svm/core/thread/ThreadSuspendSupport.java | 119 +++++++++++++----- .../svm/core/thread/VMOperationControl.java | 2 +- 3 files changed, 109 insertions(+), 56 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 635c41157401..03a374d4f949 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -174,11 +174,11 @@ public static class Options { } private static long getSafepointPromptnessWarningNanos() { - return Options.SafepointPromptnessWarningNanos.getValue().longValue(); + return Options.SafepointPromptnessWarningNanos.getValue(); } private static long getSafepointPromptnessFailureNanos() { - return Options.SafepointPromptnessFailureNanos.getValue().longValue(); + return Options.SafepointPromptnessFailureNanos.getValue(); } @Uninterruptible(reason = "Must not contain safepoint checks.") @@ -220,7 +220,7 @@ private static void slowPathSafepointCheck0(int newStatus, boolean callerHasJava assert !ThreadingSupportImpl.isRecurringCallbackRegistered(myself) || ThreadingSupportImpl.isRecurringCallbackPaused(); } else { do { - if (Master.singleton().getRequestingThread().isNonNull() || safepointSuspend.getVolatile() > 0) { + if (Master.singleton().getRequestingThread().isNonNull() || suspendedTL.getVolatile() > 0) { Statistics.incFrozen(); freezeAtSafepoint(newStatus, callerHasJavaFrameAnchor); SafepointListenerSupport.singleton().afterFreezeAtSafepoint(); @@ -344,7 +344,7 @@ private static void freezeAtSafepoint(int newStatus, boolean callerHasJavaFrameA @Uninterruptible(reason = "Must not contain safepoint checks.") private static void notInlinedLockNoTransition() { VMThreads.THREAD_MUTEX.lockNoTransition(); - while (safepointSuspend.get() > 0) { + while (suspendedTL.get() > 0) { COND_SUSPEND.blockNoTransition(); } } @@ -409,10 +409,11 @@ public static int getThreadLocalSafepointRequestedOffset() { } /** - * Per-thread counter for safepoint suspends. The possible value is {@code 0} (not suspended), - * or positive (blocked on {@link #COND_SUSPEND}). + * The possible value is {@code 0} (not suspended), or positive (suspended, possibly blocked on + * {@link #COND_SUSPEND}). This counter may only be modified while holding the + * {@link VMThreads#THREAD_MUTEX}. */ - static final FastThreadLocalInt safepointSuspend = FastThreadLocalFactory.createInt("Safepoint.safepointSuspend"); + static final FastThreadLocalInt suspendedTL = FastThreadLocalFactory.createInt("Safepoint.suspended"); /** Condition on which to block when suspended. */ static final VMCondition COND_SUSPEND = new VMCondition(VMThreads.THREAD_MUTEX); @@ -594,7 +595,7 @@ public static Master singleton() { * safepoint and Java allocations are disabled as well. */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "The safepoint logic must not allocate.") - protected boolean freeze(String reason) { + boolean freeze(String reason) { assert VMOperationControl.mayExecuteVmOperations(); long startTicks = JfrTicks.elapsedTicks(); @@ -620,11 +621,11 @@ protected boolean freeze(String reason) { /** Let all threads proceed from their safepoint. */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "The safepoint logic must not allocate.") - protected void thaw(String reason, boolean unlock) { + void thaw(boolean unlock) { assert VMOperationControl.mayExecuteVmOperations(); long startTicks = JfrTicks.elapsedTicks(); safepointState = NOT_AT_SAFEPOINT; - releaseSafepoints(reason); + releaseSafepoints(); SafepointEndEvent.emit(getSafepointId(), startTicks); ImageSingletons.lookup(Heap.class).endSafepoint(); Statistics.setThawedNanos(); @@ -832,22 +833,18 @@ private static void waitForSafepoints(String reason) { } /** Release each thread at a safepoint. */ - private static void releaseSafepoints(String reason) { - final Log trace = Log.noopLog().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); + private static void releaseSafepoints() { assert VMThreads.THREAD_MUTEX.isOwner() : "Must hold mutex when releasing safepoints."; // Set all the thread statuses that are at safepoint back to being in native code. for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { if (!isMyself(vmThread) && !SafepointBehavior.ignoresSafepoints(vmThread)) { - if (trace.isEnabled()) { - trace.string(" vmThread status: ").string(StatusSupport.getStatusString(vmThread)); - } + restoreSafepointRequestedValue(vmThread); - if (safepointSuspend.get() > 0) { + /* Skip suspended threads so that they remain in STATUS_IN_SAFEPOINT. */ + if (suspendedTL.get(vmThread) > 0) { continue; } - restoreSafepointRequestedValue(vmThread); - /* * Release the thread back to native code. Most threads will transition from * safepoint to native; but some threads will already be in native code if they @@ -856,21 +853,17 @@ private static void releaseSafepoints(String reason) { */ StatusSupport.setStatusNative(vmThread); Statistics.incReleased(); - if (trace.isEnabled()) { - trace.string(" -> ").string(StatusSupport.getStatusString(vmThread)).newline(); - } } } - trace.string("]").newline(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected IsolateThread getRequestingThread() { + IsolateThread getRequestingThread() { return requestingThread; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected boolean isFrozen() { + boolean isFrozen() { return safepointState == AT_SAFEPOINT; } @@ -1057,7 +1050,7 @@ public static void incSlowPathThawed() { } } - public static Log toLog(Log log, boolean newLine, String prefix) { + public static void toLog(Log log, boolean newLine, String prefix) { if (log.isEnabled() && Options.GatherSafepointStatistics.getValue()) { if (newLine) { log.newline(); @@ -1074,7 +1067,6 @@ public static Log toLog(Log log, boolean newLine, String prefix) { log.string(" slowPathFrozen: ").signed(getSlowPathFrozen()).newline(); log.string(" slowPathThawed: ").signed(getSlowPathThawed()).string("]").newline(); } - return log; } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java index 10b237d90fe6..b41276ab6288 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadSuspendSupport.java @@ -24,53 +24,114 @@ */ package com.oracle.svm.core.thread; +import org.graalvm.nativeimage.IsolateThread; + +import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.thread.VMThreads.StatusSupport; import com.oracle.svm.core.util.VMError; -import org.graalvm.nativeimage.IsolateThread; - /** - * Per-thread safepoint suspend/resume. The thread suspends on the current, or the first met - * safepoint. + * Support for suspending and resuming threads. *

- * Suspended threads remain in {@link StatusSupport#STATUS_IN_SAFEPOINT} (even though there is no - * global safepoint at the moment). Threads that get suspended while they execute native code will - * only stop once they enter Java code again (forced safepoint slow-path). + * Once a thread is suspended, we guarantee that it won't execute any Java code until it is resumed + * explicitly. However, suspended threads may still execute native code (including uninterruptible + * AOT-compiled Native Image code). Such threads will get blocked when they try to change their + * thread status to {@link StatusSupport#STATUS_IN_JAVA} (forced safepoint slow-path). *

- * The suspends and resumes are counted. The thread needs to be resumed the same number of times it - * was suspended to continue again. The suspension counter is only modified in a safepoint, i.e., - * all modifications are protected by the {@link VMThreads#THREAD_MUTEX}. + * After a safepoint, suspended threads remain in {@link StatusSupport#STATUS_IN_SAFEPOINT}, even + * though there is no global safepoint at the moment. The suspends and resumes are counted. A thread + * needs to be resumed the same number of times it was suspended to continue again. + *

+ * Note that there are races possible, for example: + *

+ * + * or + * + * */ public final class ThreadSuspendSupport { - private ThreadSuspendSupport() { } /** - * Suspend the thread on a safepoint, or increase the suspend count when suspended already. Need - * to be called under the {@link VMThreads#THREAD_MUTEX}. + * Suspends a thread. If the thread was already suspended, only the suspension counter will be + * incremented. */ - public static void suspend(IsolateThread vmThread) { - VMThreads.guaranteeOwnsThreadMutex("Must own the THREAD_MUTEX to prevent races."); - int suspendCounter = Safepoint.safepointSuspend.get(vmThread); - if (suspendCounter == Integer.MAX_VALUE) { - throw VMError.shouldNotReachHere("Too many thread suspends."); - } - Safepoint.safepointSuspend.set(vmThread, suspendCounter + 1); + public static void suspend(Thread thread) { + VMError.guarantee(!thread.isVirtual(), "must not be called for virtual threads"); + ThreadSuspendOperation op = new ThreadSuspendOperation(thread); + op.enqueue(); } /** - * Decrements suspend count of the thread and resume when decremented to 0. Need to be called - * under the {@link VMThreads#THREAD_MUTEX}. + * Decrements the suspension counter of the thread and resumes the thread if the counter reaches + * 0. */ - public static void resume(IsolateThread vmThread) { + public static void resume(Thread thread) { + VMError.guarantee(!thread.isVirtual(), "must not be called for virtual threads"); + ThreadResumeOperation op = new ThreadResumeOperation(thread); + op.enqueue(); + } + + private static void suspend(IsolateThread isolateThread) { + VMThreads.guaranteeOwnsThreadMutex("Must own the THREAD_MUTEX to prevent races."); + + int newValue = Safepoint.suspendedTL.get(isolateThread) + 1; + VMError.guarantee(newValue > 0, "Too many thread suspends."); + Safepoint.suspendedTL.set(isolateThread, newValue); + } + + private static void resume(IsolateThread isolateThread) { VMThreads.guaranteeOwnsThreadMutex("Must own the THREAD_MUTEX to prevent races."); - int suspendCounter = Safepoint.safepointSuspend.get(vmThread); - if (suspendCounter == 0) { - // Is resumed already - throw VMError.shouldNotReachHere("The thread is not suspended."); + + int newValue = Safepoint.suspendedTL.get(isolateThread) - 1; + VMError.guarantee(newValue >= 0, "Only a suspended thread can be resumed."); + Safepoint.suspendedTL.set(isolateThread, newValue); + + if (newValue == 0) { + Safepoint.COND_SUSPEND.broadcast(); + } + } + + private static class ThreadSuspendOperation extends JavaVMOperation { + private final Thread thread; + + ThreadSuspendOperation(Thread thread) { + super(VMOperationInfos.get(ThreadSuspendOperation.class, "Thread Suspend", SystemEffect.SAFEPOINT)); + this.thread = thread; + } + + @Override + protected void operate() { + IsolateThread isolateThread = PlatformThreads.getIsolateThread(thread); + if (isolateThread.isNonNull()) { + ThreadSuspendSupport.suspend(isolateThread); + } + } + } + + private static class ThreadResumeOperation extends JavaVMOperation { + private final Thread thread; + + ThreadResumeOperation(Thread thread) { + super(VMOperationInfos.get(ThreadResumeOperation.class, "Thread Resume", SystemEffect.SAFEPOINT)); + this.thread = thread; + } + + @Override + protected void operate() { + IsolateThread isolateThread = PlatformThreads.getIsolateThread(thread); + if (isolateThread.isNonNull()) { + ThreadSuspendSupport.resume(isolateThread); + } } - Safepoint.safepointSuspend.set(vmThread, suspendCounter - 1); - Safepoint.COND_SUSPEND.broadcast(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMOperationControl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMOperationControl.java index 62db88280b05..410f5ae151a5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMOperationControl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMOperationControl.java @@ -576,7 +576,7 @@ private void executeAllQueuedVMOperations() { drain(javaSafepointOperations); } finally { if (startedSafepoint) { - master.thaw(safepointReason, lockedForSafepoint); + master.thaw(lockedForSafepoint); } } }