diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 5c71693f5802..fe55d597931f 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -428,6 +428,9 @@ "com.oracle.svm.core.graal.aarch64", ], "requiresConcealed" : { + "java.base" : [ + "jdk.internal.misc", + ], "jdk.internal.vm.ci" : [ "jdk.vm.ci.code", ], diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java index ee48a8560878..e740e9f931ff 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java @@ -147,6 +147,9 @@ public interface pthread_key_tPointer extends PointerBase { @CFunction(transition = Transition.NO_TRANSITION) public static native int pthread_mutex_init(pthread_mutex_t mutex, pthread_mutexattr_t mutexattr); + @CFunction(value = "pthread_mutex_trylock", transition = Transition.NO_TRANSITION) + public static native int pthread_mutex_trylock_no_transition(pthread_mutex_t mutex); + @CFunction(transition = Transition.TO_NATIVE) public static native int pthread_mutex_lock(pthread_mutex_t mutex); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadConditionUtils.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadConditionUtils.java index f601be67bf33..ecba0af3c381 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadConditionUtils.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadConditionUtils.java @@ -63,7 +63,7 @@ public static int initCondition(Pthread.pthread_cond_t cond) { return Pthread.pthread_cond_init(cond, attr); } - @Uninterruptible(reason = "Called from uninterruptible code.") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void getAbsoluteTimeNanos(timespec result) { /* * We need the real-time clock to compute absolute deadlines when a conditional wait should @@ -88,15 +88,15 @@ private static void getAbsoluteTimeNanos(timespec result) { } } - /** Turn a delay in nanoseconds into a deadline in a Time.timespec. */ - @Uninterruptible(reason = "Called from uninterruptible code.") - public static void delayNanosToDeadlineTimespec(long delayNanos, Time.timespec result) { + /** Turn a duration in nanoseconds into a deadline in a Time.timespec. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void durationNanosToDeadlineTimespec(long durationNanos, Time.timespec result) { timespec currentTimespec = StackValue.get(timespec.class); getAbsoluteTimeNanos(currentTimespec); - assert delayNanos >= 0; - long sec = TimeUtils.addOrMaxValue(currentTimespec.tv_sec(), TimeUtils.divideNanosToSeconds(delayNanos)); - long nsec = currentTimespec.tv_nsec() + TimeUtils.remainderNanosToSeconds(delayNanos); + assert durationNanos >= 0; + long sec = TimeUtils.addOrMaxValue(currentTimespec.tv_sec(), TimeUtils.divideNanosToSeconds(durationNanos)); + long nsec = currentTimespec.tv_nsec() + TimeUtils.remainderNanosToSeconds(durationNanos); if (nsec >= TimeUtils.nanosPerSecond) { sec = TimeUtils.addOrMaxValue(sec, 1); nsec -= TimeUtils.nanosPerSecond; @@ -108,7 +108,7 @@ public static void delayNanosToDeadlineTimespec(long delayNanos, Time.timespec r } @Uninterruptible(reason = "Called from uninterruptible code.") - public static long deadlineTimespecToDelayNanos(Time.timespec deadlineTimespec) { + public static long deadlineTimespecToDurationNanos(Time.timespec deadlineTimespec) { timespec currentTimespec = StackValue.get(timespec.class); getAbsoluteTimeNanos(currentTimespec); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java index 52f962f09a26..a396eb15cdc5 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java @@ -315,7 +315,7 @@ public void blockNoTransitionUnspecifiedOwner() { @Override public long block(long waitNanos) { Time.timespec deadlineTimespec = UnsafeStackValue.get(Time.timespec.class); - PthreadConditionUtils.delayNanosToDeadlineTimespec(waitNanos, deadlineTimespec); + PthreadConditionUtils.durationNanosToDeadlineTimespec(waitNanos, deadlineTimespec); mutex.clearCurrentThreadOwner(); final int timedWaitResult = Pthread.pthread_cond_timedwait(getStructPointer(), ((PthreadVMMutex) getMutex()).getStructPointer(), deadlineTimespec); @@ -326,14 +326,14 @@ public long block(long waitNanos) { } /* Check for other errors from the timed wait. */ PthreadVMLockSupport.checkResult(timedWaitResult, "pthread_cond_timedwait"); - return PthreadConditionUtils.deadlineTimespecToDelayNanos(deadlineTimespec); + return PthreadConditionUtils.deadlineTimespecToDurationNanos(deadlineTimespec); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", callerMustBe = true) public long blockNoTransition(long waitNanos) { Time.timespec deadlineTimespec = StackValue.get(Time.timespec.class); - PthreadConditionUtils.delayNanosToDeadlineTimespec(waitNanos, deadlineTimespec); + PthreadConditionUtils.durationNanosToDeadlineTimespec(waitNanos, deadlineTimespec); mutex.clearCurrentThreadOwner(); final int timedwaitResult = Pthread.pthread_cond_timedwait_no_transition(getStructPointer(), ((PthreadVMMutex) getMutex()).getStructPointer(), deadlineTimespec); @@ -344,7 +344,7 @@ public long blockNoTransition(long waitNanos) { } /* Check for other errors from the timed wait. */ PthreadVMLockSupport.checkResult(timedwaitResult, "pthread_cond_timedwait"); - return PthreadConditionUtils.deadlineTimespecToDelayNanos(deadlineTimespec); + return PthreadConditionUtils.deadlineTimespecToDurationNanos(deadlineTimespec); } @Override diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index 69340459f932..fd001a58a81e 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -60,6 +60,7 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.monitor.JavaMonitor; import com.oracle.svm.core.os.IsDefined; import com.oracle.svm.core.posix.PosixUtils; import com.oracle.svm.core.posix.headers.Errno; @@ -75,9 +76,12 @@ import com.oracle.svm.core.thread.ParkEvent; import com.oracle.svm.core.thread.ParkEvent.ParkEventFactory; import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.util.TimeUtils; import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; +import jdk.internal.misc.Unsafe; + @AutomaticallyRegisteredImageSingleton(PlatformThreads.class) public final class PosixPlatformThreads extends PlatformThreads { @@ -287,16 +291,32 @@ final class Target_java_lang_Thread { Pthread.pthread_t pthreadIdentifier; } -class PosixParkEvent extends ParkEvent { +/** + * {@link PosixParkEvent} is based on HotSpot class {@code Parker} in {@code os_posix.cpp}, as of + * JDK 19 (git commit hash: f640fc5a1eb876a657d0de011dcd9b9a42b88eec, JDK tag: jdk-19+30). + *

+ * HotSpot has two constructs with a similar purpose: {@code ParkEvent} and {@code Parker}. The + * latter implements JSR 166 synchronization primitives {@link Unsafe#park} and + * {@link Unsafe#unpark}, just like we do here, therefore we base this implementation on + * {@code Parker}. Our implementation of Java object monitors, {@link JavaMonitor}, uses the JSR 166 + * primitives, so it can potentially experience interference from unrelated calls to + * {@link Unsafe#unpark}. This is a difference to HotSpot's {@code ObjectMonitor}, which uses a + * separate HotSpot {@code ParkEvent} instance. Another difference is that {@code Parker} and the + * code below return control to the caller on spurious wakeups, unlike HotSpot's {@code ParkEvent}. + * This does not affect correctness. + */ +final class PosixParkEvent extends ParkEvent { + private static final Unsafe U = Unsafe.getUnsafe(); + private static final long EVENT_OFFSET = U.objectFieldOffset(PosixParkEvent.class, "event"); private Pthread.pthread_mutex_t mutex; private Pthread.pthread_cond_t cond; - /** - * The ticket: false implies unavailable, true implies available. Volatile so it can be safely - * updated in {@link #reset()} without holding the lock. - */ - protected volatile boolean event; + /** Permit: 1 if an unpark is pending, otherwise 0. */ + private volatile int event = 0; + + /** Whether the owner is currently parked. Guarded by {@link #mutex}. */ + private boolean parked = false; PosixParkEvent() { // Allocate mutex and condition in a single step so that they are adjacent in memory. @@ -308,61 +328,79 @@ class PosixParkEvent extends ParkEvent { final Pthread.pthread_mutexattr_t mutexAttr = WordFactory.nullPointer(); PosixUtils.checkStatusIs0(Pthread.pthread_mutex_init(mutex, mutexAttr), "mutex initialization"); PosixUtils.checkStatusIs0(PthreadConditionUtils.initCondition(cond), "condition variable initialization"); + // Note: HotSpot has another pthread_cond_t without CLOCK_MONOTONIC for absolute timed waits } @Override protected void reset() { - event = false; + event = 0; } @Override protected void condWait() { - StackOverflowCheck.singleton().makeYellowZoneAvailable(); - try { - PosixUtils.checkStatusIs0(Pthread.pthread_mutex_lock(mutex), "park(): mutex lock"); - try { - while (!event) { - int status = Pthread.pthread_cond_wait(cond, mutex); - PosixUtils.checkStatusIs0(status, "park(): condition variable wait"); - } - event = false; - } finally { - PosixUtils.checkStatusIs0(Pthread.pthread_mutex_unlock(mutex), "park(): mutex unlock"); - } - } finally { - StackOverflowCheck.singleton().protectYellowZone(); + park(false, 0); + } + + @Override + protected void condTimedWait(long durationNanos) { + if (durationNanos > 0) { + park(false, durationNanos); } } @Override - protected void condTimedWait(long delayNanos) { + protected boolean tryFastPark() { + // We depend on getAndSet having full barrier semantics since we are not locking + return U.getAndSetInt(this, EVENT_OFFSET, 0) != 0; + } + + @Override + protected void park(boolean isAbsolute, long time) { + if (time < 0 || (isAbsolute && time == 0)) { + return; // don't wait at all + } StackOverflowCheck.singleton().makeYellowZoneAvailable(); try { - /* Encode the delay as a deadline in a Time.timespec. */ - Time.timespec deadlineTimespec = UnsafeStackValue.get(Time.timespec.class); - PthreadConditionUtils.delayNanosToDeadlineTimespec(delayNanos, deadlineTimespec); - - PosixUtils.checkStatusIs0(Pthread.pthread_mutex_lock(mutex), "park(long): mutex lock"); + int status = Pthread.pthread_mutex_trylock_no_transition(mutex); + if (status == Errno.EBUSY()) { + return; // can only mean another thread is unparking us: don't wait + } + PosixUtils.checkStatusIs0(status, "park: mutex trylock"); try { - while (!event) { - int status = Pthread.pthread_cond_timedwait(cond, mutex, deadlineTimespec); - if (status == Errno.ETIMEDOUT()) { - break; - } else if (status != 0) { - Log.log().newline() - .string("[PosixParkEvent.condTimedWait(delayNanos: ").signed(delayNanos).string("): Should not reach here.") - .string(" mutex: ").hex(mutex) - .string(" cond: ").hex(cond) - .string(" deadlineTimeSpec.tv_sec: ").signed(deadlineTimespec.tv_sec()) - .string(" deadlineTimespec.tv_nsec: ").signed(deadlineTimespec.tv_nsec()) - .string(" status: ").signed(status).string(" ").string(Errno.strerror(status)) - .string("]").newline(); - PosixUtils.checkStatusIs0(status, "park(long): condition variable timed wait"); + if (event == 0) { + assert !parked; + parked = true; + if (!isAbsolute && time == 0) { + status = Pthread.pthread_cond_wait(cond, mutex); + PosixUtils.checkStatusIs0(status, "park(): condition variable wait"); + } else { + long durationNanos = TimeUtils.durationNanos(isAbsolute, time); + Time.timespec deadlineTimespec = UnsafeStackValue.get(Time.timespec.class); + PthreadConditionUtils.durationNanosToDeadlineTimespec(durationNanos, deadlineTimespec); + + status = Pthread.pthread_cond_timedwait(cond, mutex, deadlineTimespec); + if (status != 0 && status != Errno.ETIMEDOUT()) { + Log.log().newline() + .string("[PosixParkEvent.park(durationNanos: ").signed(durationNanos).string("): Should not reach here.") + .string(" mutex: ").hex(mutex) + .string(" cond: ").hex(cond) + .string(" deadlineTimeSpec.tv_sec: ").signed(deadlineTimespec.tv_sec()) + .string(" deadlineTimespec.tv_nsec: ").signed(deadlineTimespec.tv_nsec()) + .string(" status: ").signed(status).string(" ").string(Errno.strerror(status)) + .string("]").newline(); + PosixUtils.checkStatusIs0(status, "park(boolean, long): condition variable timed wait"); + } } + parked = false; } - event = false; + event = 0; + } finally { - PosixUtils.checkStatusIs0(Pthread.pthread_mutex_unlock(mutex), "park(long): mutex unlock"); + PosixUtils.checkStatusIs0(Pthread.pthread_mutex_unlock(mutex), "park: mutex unlock"); + + // Paranoia to ensure our locked and lock-free paths interact + // correctly with each other and Java-level accesses. + U.fullFence(); } } finally { StackOverflowCheck.singleton().protectYellowZone(); @@ -373,13 +411,23 @@ protected void condTimedWait(long delayNanos) { protected void unpark() { StackOverflowCheck.singleton().makeYellowZoneAvailable(); try { + int s; + boolean p; PosixUtils.checkStatusIs0(Pthread.pthread_mutex_lock(mutex), "PosixParkEvent.unpark(): mutex lock"); try { - event = true; - PosixUtils.checkStatusIs0(Pthread.pthread_cond_broadcast(cond), "PosixParkEvent.unpark(): condition variable broadcast"); + s = event; + event = 1; + p = parked; } finally { PosixUtils.checkStatusIs0(Pthread.pthread_mutex_unlock(mutex), "PosixParkEvent.unpark(): mutex unlock"); } + if (s == 0 && p) { + /* + * Signal without holding the mutex, which is safe and avoids futile wakeups if the + * platform does not implement wait morphing. + */ + PosixUtils.checkStatusIs0(Pthread.pthread_cond_signal(cond), "PosixParkEvent.unpark(): condition variable signal"); + } } finally { StackOverflowCheck.singleton().protectYellowZone(); } @@ -400,9 +448,4 @@ class PosixParkEventFactory implements ParkEventFactory { public ParkEvent acquire() { return new PosixParkEvent(); } - - @Override - public boolean usesParkEventList() { - return false; - } } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsPlatformThreads.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsPlatformThreads.java index 8557f6bcd3c3..454b24230692 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsPlatformThreads.java @@ -231,13 +231,13 @@ protected void condWait() { } @Override - protected void condTimedWait(long delayNanos) { + protected void condTimedWait(long durationNanos) { StackOverflowCheck.singleton().makeYellowZoneAvailable(); try { final int maxTimeout = 0x10_000_000; - long delayMillis = Math.max(0, TimeUtils.roundUpNanosToMillis(delayNanos)); + long durationMillis = Math.max(0, TimeUtils.roundUpNanosToMillis(durationNanos)); do { // at least once to consume potential unpark - int timeout = (delayMillis < maxTimeout) ? (int) delayMillis : maxTimeout; + int timeout = (durationMillis < maxTimeout) ? (int) durationMillis : maxTimeout; int status = SynchAPI.WaitForSingleObject(eventHandle, timeout); if (status == SynchAPI.WAIT_OBJECT_0()) { break; // unparked @@ -246,8 +246,8 @@ protected void condTimedWait(long delayNanos) { Log.log().newline().string("GetLastError returned: ").hex(WinBase.GetLastError()).newline(); throw VMError.shouldNotReachHere("WaitForSingleObject failed"); } - delayMillis -= timeout; - } while (delayMillis > 0); + durationMillis -= timeout; + } while (durationMillis > 0); } finally { StackOverflowCheck.singleton().protectYellowZone(); } @@ -279,9 +279,4 @@ class WindowsParkEventFactory implements ParkEventFactory { public ParkEvent acquire() { return new WindowsParkEvent(); } - - @Override - public boolean usesParkEventList() { - return false; - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ParkEvent.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ParkEvent.java index 6f8fbc8a0ece..b41fea606e9a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ParkEvent.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ParkEvent.java @@ -24,177 +24,82 @@ */ package com.oracle.svm.core.thread; -import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicReference; -import com.oracle.svm.core.util.VMError; +import com.oracle.svm.core.util.TimeUtils; /** - * Each thread has several of these on which to wait. Instances are usually expensive objects - * because they encapsulate native resources. Therefore, lazy initialization is used, see + * Per-thread blocking support. An instance is owned by at most one thread at a time for blocking + * itself from scheduling ("parking") by calling {@link #park}. Any other thread may call + * {@link #unpark} to unblock the owner thread (or make its next attempt to park return instantly). + * + * Each thread may own several of these for parking. Instances are usually expensive objects because + * they encapsulate native resources. Therefore, lazy initialization is used, see * {@link ThreadData}. */ public abstract class ParkEvent { public interface ParkEventFactory { - default ParkEvent acquire() { - return create(); - } - - /** For legacy code that does not implement {@link #acquire}. */ - default ParkEvent create() { - throw VMError.shouldNotReachHere("Must implement acquire()."); - } - - /** - * For legacy code. This should be an implementation detail of {@link #acquire} and - * {@link ParkEvent#release}. - */ - default boolean usesParkEventList() { - return true; - } + ParkEvent acquire(); } /** Currently required by legacy code. */ protected boolean isSleepEvent; - /** - * A cons-cell for putting this ParkEvent on the free list. This must be (a) allocated - * beforehand because I need it when I can not allocate, (b) must not be reused, to avoid an ABA - * problem. - */ - private ParkEventConsCell consCell; - - /** Constructor for subclasses. */ protected ParkEvent() { } - /** - * Resets a pending {@link #unpark()} at the time of the call. - */ + /** Reset a pending {@link #unpark()} at the time of the call. */ protected abstract void reset(); - /* cond_wait. */ + /** {@link #park} indefinitely: {@code park(false, 0);}. */ protected abstract void condWait(); - /** cond_timedwait, similar to {@link #condWait} but with a timeout in nanoseconds. */ - protected abstract void condTimedWait(long delayNanos); - - /** Notify anyone waiting on this event. */ - protected abstract void unpark(); - - /** Use up the cons-cell for this ParkEvent. */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - ParkEventConsCell consumeConsCell() { - assert consCell != null : "Consuming null cons cell."; - ParkEventConsCell result = consCell; - consCell = null; - return result; - } - - /** Acquire a ParkEvent, either by allocating or reusing a previously released event. */ - static ParkEvent acquire(boolean isSleepEvent) { - ParkEventFactory factory = ImageSingletons.lookup(ParkEventFactory.class); - if (!factory.usesParkEventList()) { - return factory.acquire(); - } + /** + * {@link #park} with a duration in nanoseconds: + * {@code if(duration > 0) park(false, duration);}. + */ + protected abstract void condTimedWait(long durationNanos); - ParkEvent result = ParkEventList.getSingleton().pop(); - if (result == null) { - result = factory.acquire(); + /** + * Block the calling thread (which must be the owner of this instance) from being scheduled + * until another thread calls {@link #unpark}, + *

+ * May also return spuriously instead (for no apparent reason). + */ + protected void park(boolean isAbsolute, long time) { + if (!isAbsolute && time == 0) { + condWait(); + } else if (time > 0) { + condTimedWait(TimeUtils.durationNanos(isAbsolute, time)); } - - /* Assign a *new* cons-cell for this ParkEvent. */ - result.consCell = new ParkEventConsCell(result); - result.isSleepEvent = isSleepEvent; - result.reset(); - return result; - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void release() { - assert ImageSingletons.lookup(ParkEventFactory.class).usesParkEventList(); - ParkEventList.getSingleton().push(this); - } -} - -/** - * A free-list of ParkEvents. - * - * Since ParkEvents have to be immortal, they are not garbage collected. Instead, they are put back - * on a free-list. To avoid ABA problems with multi-threaded pops from the list, I make up a new - * cons-cell for each push to the list. - */ -final class ParkEventList { - - private static final ParkEventList SINGLETON = new ParkEventList(); - - @Fold - public static ParkEventList getSingleton() { - return SINGLETON; } - /** The free-list of ParkEvents. */ - private final AtomicReference freeList; - - /** Private constructor: Only the singleton instance. */ - private ParkEventList() { - freeList = new AtomicReference<>(null); + /** Try consuming an unpark without blocking. */ + protected boolean tryFastPark() { + return false; } - /** Push an element on to the free-list. */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void push(ParkEvent element) { - ParkEventConsCell sampleHead; - /* Use up the cons-cell for each attempted push to avoid the ABA problem on pops. */ - ParkEventConsCell nextHead = element.consumeConsCell(); - do { - sampleHead = freeList.get(); - nextHead.setNext(sampleHead); - } while (!freeList.compareAndSet(sampleHead, nextHead)); - } - - /** Return the head of the free-list, or null. */ - public ParkEvent pop() { - ParkEventConsCell sampleHead; - ParkEventConsCell sampleNext; - do { - sampleHead = freeList.get(); - if (sampleHead == null) { - return null; - } - sampleNext = sampleHead.getNext(); - } while (!freeList.compareAndSet(sampleHead, sampleNext)); - return sampleHead.getElement(); - } -} - -/** A cons-cell for the free-list. */ -final class ParkEventConsCell { - - /** Immutable state. */ - private final ParkEvent element; - /** Mutable state, but only until the cons-cell is on the list. */ - private ParkEventConsCell next; - - /** Constructor. */ - ParkEventConsCell(ParkEvent element) { - this.element = element; - this.next = null; - } - - protected ParkEvent getElement() { - return element; - } + /** + * Unblock the owner thread if it parks on this object, or make its next attempt to park on this + * object return immediately. + */ + protected abstract void unpark(); - protected ParkEventConsCell getNext() { - return next; + static ParkEvent acquire(boolean isSleepEvent) { + ParkEventFactory factory = ImageSingletons.lookup(ParkEventFactory.class); + ParkEvent event = factory.acquire(); + event.isSleepEvent = isSleepEvent; + return event; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void setNext(ParkEventConsCell next) { - this.next = next; - } + protected abstract void release(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index da4fdcaeb336..490d3dcbde49 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -852,46 +852,36 @@ static Thread[] getAllThreads() { /** Interruptibly park the current thread. */ static void parkCurrentPlatformOrCarrierThread() { - VMOperationControl.guaranteeOkayToBlock("[PlatformThreads.parkCurrentPlatformOrCarrierThread(): Should not park when it is not okay to block.]"); - Thread thread = currentThread.get(); - if (JavaThreads.isInterrupted(thread)) { // avoid state changes and synchronization - return; - } + parkCurrentPlatformOrCarrierThread(false, 0); + } - ParkEvent parkEvent = getCurrentThreadData().ensureUnsafeParkEvent(); - // Change the Java thread state while parking. - int oldStatus = getThreadStatus(thread); - int newStatus = MonitorSupport.singleton().getParkedThreadStatus(currentThread.get(), false); - setThreadStatus(thread, newStatus); - try { - /* - * If another thread interrupted this thread in the meanwhile, then the call below won't - * block because Thread.interrupt() modifies the ParkEvent. - */ - parkEvent.condWait(); - } finally { - setThreadStatus(thread, oldStatus); + /** Interruptibly park the current thread, indefinitely or with a timeout. */ + static void parkCurrentPlatformOrCarrierThread(boolean isAbsolute, long time) { + VMOperationControl.guaranteeOkayToBlock("[PlatformThreads.parkCurrentPlatformOrCarrierThread: Should not park when it is not okay to block.]"); + + if (time < 0 || (isAbsolute && time == 0)) { + return; // don't wait at all } - } + boolean timed = (time != 0); - /** Interruptibly park the current thread for the given number of nanoseconds. */ - static void parkCurrentPlatformOrCarrierThread(long delayNanos) { - VMOperationControl.guaranteeOkayToBlock("[PlatformThreads.parkCurrentPlatformOrCarrierThread(long): Should not park when it is not okay to block.]"); Thread thread = currentThread.get(); if (JavaThreads.isInterrupted(thread)) { // avoid state changes and synchronization return; } ParkEvent parkEvent = getCurrentThreadData().ensureUnsafeParkEvent(); + if (parkEvent.tryFastPark()) { + return; + } int oldStatus = getThreadStatus(thread); - int newStatus = MonitorSupport.singleton().getParkedThreadStatus(currentThread.get(), true); + int newStatus = MonitorSupport.singleton().getParkedThreadStatus(currentThread.get(), timed); setThreadStatus(thread, newStatus); try { /* * If another thread interrupted this thread in the meanwhile, then the call below won't * block because Thread.interrupt() modifies the ParkEvent. */ - parkEvent.condTimedWait(delayNanos); + parkEvent.park(isAbsolute, time); } finally { setThreadStatus(thread, oldStatus); } @@ -901,7 +891,7 @@ static void parkCurrentPlatformOrCarrierThread(long delayNanos) { * Unpark a Thread. * * @see #parkCurrentPlatformOrCarrierThread() - * @see #parkCurrentPlatformOrCarrierThread(long) + * @see #parkCurrentPlatformOrCarrierThread(boolean, long) */ static void unpark(Thread thread) { assert !isVirtual(thread); @@ -927,7 +917,7 @@ static void sleep(long millis) throws InterruptedException { } } - private static void sleep0(long delayNanos) { + private static void sleep0(long durationNanos) { VMOperationControl.guaranteeOkayToBlock("[PlatformThreads.sleep(long): Should not sleep when it is not okay to block.]"); Thread thread = currentThread.get(); ParkEvent sleepEvent = getCurrentThreadData().ensureSleepParkEvent(); @@ -950,11 +940,19 @@ private static void sleep0(long delayNanos) { final int oldStatus = getThreadStatus(thread); setThreadStatus(thread, ThreadStatus.SLEEPING); try { - /* - * If another thread interrupted this thread in the meanwhile, then the call below won't - * block because Thread.interrupt() modifies the ParkEvent. - */ - sleepEvent.condTimedWait(delayNanos); + long remainingNanos = durationNanos; + long startNanos = System.nanoTime(); + while (remainingNanos > 0) { + /* + * If another thread interrupted this thread in the meanwhile, then the call below + * won't block because Thread.interrupt() modifies the ParkEvent. + */ + sleepEvent.condTimedWait(remainingNanos); + if (JavaThreads.isInterrupted(thread)) { + return; + } + remainingNanos = durationNanos - (System.nanoTime() - startNanos); + } } finally { setThreadStatus(thread, oldStatus); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/SpinLockUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/SpinLockUtils.java index 7ee550c8d26f..bc9bd29f7cdd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/SpinLockUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/SpinLockUtils.java @@ -40,21 +40,21 @@ public class SpinLockUtils { @Uninterruptible(reason = "This method does not do a transition, so the whole critical section must be uninterruptible.", callerMustBe = true) public static void lockNoTransition(Object obj, long intFieldOffset) { - // Fast-path. if (UNSAFE.compareAndSetInt(obj, intFieldOffset, 0, 1)) { - return; + return; // fast-path } - // Slow-path. + int ctr = 0; int yields = 0; while (true) { - while (UNSAFE.getIntVolatile(obj, intFieldOffset) != 0) { + while (UNSAFE.getIntOpaque(obj, intFieldOffset) != 0) { + ctr++; /* - * It would be better to use a more sophisticated logic that takes the number of CPU - * cores into account. However, this is not easily possible because calling - * Runtime.availableProcessors() can be expensive. + * It would be better to take into account if we are on a single-processor machine + * where spinning is futile. However, determining that is expensive in itself. We do + * use fewer successive spins than the equivalent HotSpot code does (0xFFF). */ - if (VMThreads.singleton().supportsNativeYieldAndSleep()) { + if ((ctr & 0xff) == 0 && VMThreads.singleton().supportsNativeYieldAndSleep()) { if (yields > 5) { VMThreads.singleton().nativeSleep(1); } else { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_jdk_internal_misc_Unsafe_JavaThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_jdk_internal_misc_Unsafe_JavaThreads.java index 51ccc33775c3..402217d9805b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_jdk_internal_misc_Unsafe_JavaThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_jdk_internal_misc_Unsafe_JavaThreads.java @@ -33,7 +33,6 @@ import com.oracle.svm.core.annotate.TargetClass; import com.oracle.svm.core.jfr.JfrTicks; import com.oracle.svm.core.jfr.events.ThreadParkEvent; -import com.oracle.svm.core.util.TimeUtils; @TargetClass(className = "jdk.internal.misc.Unsafe") @Platforms(InternalPlatform.NATIVE_ONLY.class) @@ -61,8 +60,7 @@ void park(boolean isAbsolute, long time) { ThreadParkEvent.emit(startTicks, parkBlocker, Long.MIN_VALUE, Long.MIN_VALUE); } else { /* Park with deadline. */ - final long delayNanos = TimeUtils.delayNanos(isAbsolute, time); - PlatformThreads.parkCurrentPlatformOrCarrierThread(delayNanos); + PlatformThreads.parkCurrentPlatformOrCarrierThread(isAbsolute, time); if (isAbsolute) { ThreadParkEvent.emit(startTicks, parkBlocker, Long.MIN_VALUE, time); } else { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TimeUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TimeUtils.java index f92dc6db0635..76efeb8704b5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TimeUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TimeUtils.java @@ -82,15 +82,15 @@ public static boolean nanoTimeLessThan(long leftNanos, long rightNanos) { } /** - * Turn an absolute deadline in milliseconds, or a relative delay in nanoseconds, into a - * relative delay in nanoseconds. + * Turn an absolute deadline in milliseconds, or a relative duration in nanoseconds, into a + * relative duration in nanoseconds. */ - public static long delayNanos(boolean isAbsolute, long time) { + public static long durationNanos(boolean isAbsolute, long time) { if (isAbsolute) { /* Absolute deadline, in milliseconds. */ return millisToNanos(time - System.currentTimeMillis()); } else { - /* Relative delay, in nanoseconds. */ + /* Relative duration, in nanoseconds. */ return time; } }