From 1533d7bd7ee608c15956ddba1722be74f7b80c15 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 17 Feb 2022 16:01:18 +0300 Subject: [PATCH 001/125] Noop thread using volatile flags and busy wait --- .../oracle/svm/core/genscavenge/GCImpl.java | 23 ++- .../oracle/svm/core/SubstrateDiagnostics.java | 7 + .../graal/snippets/CEntryPointSnippets.java | 3 + .../com/oracle/svm/core/heap/ParallelGC.java | 141 ++++++++++++++++++ .../svm/core/stack/JavaStackWalker.java | 17 +++ .../com/oracle/svm/core/thread/Safepoint.java | 33 +++- .../com/oracle/svm/core/thread/VMThreads.java | 14 ++ 7 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 88e1eefa7920..13c65d9ba716 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -29,8 +29,13 @@ import java.lang.ref.Reference; +import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.jfr.JfrTicks; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.thread.Safepoint; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -677,6 +682,9 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { + thawParallelGCThreads(); +// ImageSingletons.lookup(ParallelGC.class).signal(); + /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -768,6 +776,9 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { + thawParallelGCThreads(); + ImageSingletons.lookup(ParallelGC.class).signal(); + /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); @@ -848,7 +859,7 @@ private void blackenStackRoots() { JavaStackWalk walk = StackValue.get(JavaStackWalk.class); JavaStackWalker.initWalk(walk, sp, ip); - walkStack(walk); + walkStack(walk, WordFactory.nullPointer()); if (SubstrateOptions.MultiThreaded.getValue()) { /* @@ -866,7 +877,7 @@ private void blackenStackRoots() { continue; } if (JavaStackWalker.initWalk(walk, vmThread)) { - walkStack(walk); + walkStack(walk, vmThread); } } } @@ -882,7 +893,7 @@ private void blackenStackRoots() { * calls to a stack frame visitor. */ @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) - private void walkStack(JavaStackWalk walk) { + private void walkStack(JavaStackWalk walk, IsolateThread thread) { assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether"; while (true) { @@ -895,7 +906,7 @@ private void walkStack(JavaStackWalk walk) { DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); if (deoptFrame == null) { if (codeInfo.isNull()) { - throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame); + throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame, thread, walk.getAnchor(), walk.getIPCodeInfo(), codeInfo); } CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult); @@ -933,6 +944,10 @@ private void walkStack(JavaStackWalk walk) { } } + private void thawParallelGCThreads() { + Safepoint.Master.releaseParallelGCSafepoints(); + } + private void walkThreadLocals() { if (SubstrateOptions.MultiThreaded.getValue()) { Timer walkThreadLocalsTimer = timers.walkThreadLocals.open(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index d0b83fa3ace9..1c9dd8d108d4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -712,6 +712,13 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } log.string(", stack(").zhex(VMThreads.StackEnd.get(thread)).string(",").zhex(VMThreads.StackBase.get(thread)).string(")"); + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); + if (anchor.isNonNull()) { + log.newline().spaces(21) + .string("anchor=").zhex(anchor) + .string(", sp=").zhex(anchor.getLastJavaSP()) + .string(", ip=").zhex(anchor.getLastJavaIP()); + } log.newline(); printed++; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 96ebfdb87dbc..69c7e8b43e31 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -33,6 +33,7 @@ import java.util.Map; +import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; @@ -291,6 +292,8 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } + ImageSingletons.lookup(ParallelGC.class).startThreads(); + /* * After starting all the necessary threads, we can finally execute complex JDK code or code * that allocates a significant amount of memory. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java new file mode 100644 index 000000000000..5b6cd58962c3 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -0,0 +1,141 @@ +package com.oracle.svm.core.heap; + +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; +import sun.misc.Signal; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +public class ParallelGC { + + private final VMMutex mutex = new VMMutex("pargc"); + private final VMCondition cond = new VMCondition(mutex); + + private Task task = new Task(); + private volatile boolean idle = true; + + public void setTask(Pointer src, Pointer dst, UnsignedWord size) { + mutex.lock(); + task.src = src; + task.dst = dst; + task.size = size; + mutex.unlock(); + + idle = false; + cond.signal(); + Log.log().string("setTask() on ").string(Thread.currentThread().getName()).newline(); + } + + public boolean isIdle() { + return idle; + } + + public void _startThreads() { + Log trace = Log.log(); + Thread t = new Thread(() -> { + trace.string(" started thread ").string(Thread.currentThread().getName()).newline(); + while (true) { + mutex.lock(); + while (idle) { + trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); + cond.block(); + trace.string(" unblocked on ").string(Thread.currentThread().getName()).newline(); + } + trace.string(" copying...").newline(); +// UnmanagedMemoryUtil.copyLongsForward(task.src, task.dst, task.size); + mutex.unlock(); + idle = true; + trace.string(" done").newline(); + } + }); + t.setName("ParallelGC"); + t.setDaemon(true); + trace.string("starting thread...").newline(); + t.start(); + } + + static class Task { + Pointer src, dst; + UnsignedWord size; + } + + private volatile boolean notified; + private volatile boolean stopped; + +// public static Thread parThread; + + public void __startThreads() { + Log trace = Log.log(); + Thread t = new Thread() { + @Override + @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) + public void run() { + VMThreads.ParallelGCSupport.setParallelGCThread(); + try { + while (!stopped) { + waitForNotification(); + } + } catch (Throwable e) { + VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); + } + } + }; + t.setName("ParallelGC-noop"); + t.setDaemon(true); +// parThread = t; + t.start(); + } + + private void waitForNotification() { + Log trace = Log.log(); + trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); + while (!notified); + trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); + notified = false; + } + + public void signal() { + Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); + notified = true; + } + + public void startThreads() { + Log trace = Log.log(); + Thread t = new Thread() { + @Override + public void run() { + VMThreads.ParallelGCSupport.setParallelGCThread(); + try { + while (!stopped) { + waitForNotification(); + } + } catch (Throwable e) { + VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); + } + } + }; + t.setName("ParallelGC-noop"); + t.setDaemon(true); + t.start(); + } +} + +@AutomaticFeature +class ParallelGCFeature implements Feature { +// @Override +// public boolean isInConfiguration(IsInConfigurationAccess access) { +// return SubstrateOptions.UseSerialGC.getValue(); +// } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(ParallelGC.class, new ParallelGC()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java index 96bfa38324f8..c9bc20c9e0b6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java @@ -245,6 +245,23 @@ public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePoi } + @Uninterruptible(reason = "Not really uninterruptible, but we are about to fatally fail.", calleeMustBe = false) + public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePointer ip, DeoptimizedFrame deoptFrame, + IsolateThread thread, JavaFrameAnchor anchor, UntetheredCodeInfo uCodeInfo, CodeInfo codeInfo) + { + Log log = Log.log().string("Stack walk must walk only frames of known code:"); + log.string(" anchor=").hex(anchor).string(" sp=").hex(sp).string(" ip=").hex(ip).newline() + .string(" thread=").hex(thread).string(" codeInfo=").hex(codeInfo) + .string(" untethered=").hex(uCodeInfo); + + if (DeoptimizationSupport.enabled()) { + log.string(" deoptFrame=").object(deoptFrame); + } + log.newline(); + throw VMError.shouldNotReachHere("Stack walk must walk only frames of known code"); + + } + public static boolean walkCurrentThread(Pointer startSP, StackFrameVisitor visitor) { return walkCurrentThread(startSP, visitor, null); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 71596b7cdb32..9b46f41c7109 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -665,7 +665,7 @@ private static boolean isMyself(IsolateThread thread) { /** Send each of the threads (except myself) a request to come to a safepoint. */ private static int requestSafepoints(String reason) { VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex while requesting a safepoint."); - final Log trace = Log.noopLog().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); + final Log trace = Log.log().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); int numOfThreads = 0; // Walk the threads list and ask each thread (except myself) to come to a safepoint. @@ -853,7 +853,7 @@ private static void waitForSafepoints(String reason) { /** Release each thread at a safepoint. */ private static void releaseSafepoints(String reason) { - final Log trace = Log.noopLog().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); + final Log trace = Log.log().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); // Set all the thread statuses that are at safepoint back to being in native code. for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { @@ -880,6 +880,35 @@ private static void releaseSafepoints(String reason) { trace.string("]").newline(); } + /** Release each thread at a safepoint. */ + public static void releaseParallelGCSafepoints() { + final Log trace = Log.log().string("[Safepoint.Master.releasePGCSafepoints").newline(); + VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); + // Set PGC thread statuses that are at safepoint back to being in native code. + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { + if (trace.isEnabled()) { + trace.string(" PGC vmThread status: ").string(StatusSupport.getStatusString(vmThread)); + } + + restoreSafepointRequestedValue(vmThread); + + /* + * Release the thread back to native code. Most threads will transition from + * safepoint to native; but some threads will already be in native code if they + * returned from native code, found the safepoint in progress and blocked on the + * mutex putting themselves back in native code again. + */ + StatusSupport.setStatusNative(vmThread); + Statistics.incReleased(); + if (trace.isEnabled()) { + trace.string(" -> ").string(StatusSupport.getStatusString(vmThread)).newline(); + } + } + } + trace.string("]").newline(); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected IsolateThread getRequestingThread() { return requestingThread; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 9a4b14251f57..f10e1ae9aa94 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -826,6 +826,20 @@ public static int getNewThreadStatus(CFunctionOptions.Transition transition) { } } + public static class ParallelGCSupport { + private static final FastThreadLocalInt isParallelGCThreadTL = FastThreadLocalFactory.createInt("ParallelGCSupport.isParallelGCThreadTL"); + private static final int REGULAR_THREAD = 0; + private static final int PARALLEL_GC_THREAD = 1; + + public static boolean isParallelGCThread(IsolateThread vmThread) { + return isParallelGCThreadTL.getVolatile(vmThread) == PARALLEL_GC_THREAD; + } + + public static void setParallelGCThread() { + isParallelGCThreadTL.setVolatile(PARALLEL_GC_THREAD); + } + } + public static class SafepointBehavior { /** Determines how this thread interacts with the safepoint handling. */ private static final FastThreadLocalInt safepointBehaviorTL = FastThreadLocalFactory.createInt("StatusSupport.safepointBehaviorTL"); From 6195b82dde2f2b47fbd019da0279a23340052380 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 28 Feb 2022 15:26:11 +0300 Subject: [PATCH 002/125] Noop thread with safepoints and synchronization --- .../oracle/svm/core/genscavenge/GCImpl.java | 11 +- .../graal/snippets/CEntryPointSnippets.java | 2 +- .../com/oracle/svm/core/heap/ParallelGC.java | 104 ++++-------------- .../com/oracle/svm/core/thread/Safepoint.java | 12 +- 4 files changed, 36 insertions(+), 93 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 13c65d9ba716..97df1fce19f1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -32,7 +32,6 @@ import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.thread.Safepoint; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; @@ -682,8 +681,8 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - thawParallelGCThreads(); -// ImageSingletons.lookup(ParallelGC.class).signal(); + ParallelGC.thawWorkerThreads(); + ImageSingletons.lookup(ParallelGC.class).signal(); /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -776,7 +775,7 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - thawParallelGCThreads(); + ParallelGC.thawWorkerThreads(); ImageSingletons.lookup(ParallelGC.class).signal(); /* Visit all the Objects promoted since the snapshot, transitively. */ @@ -944,10 +943,6 @@ private void walkStack(JavaStackWalk walk, IsolateThread thread) { } } - private void thawParallelGCThreads() { - Safepoint.Master.releaseParallelGCSafepoints(); - } - private void walkThreadLocals() { if (SubstrateOptions.MultiThreaded.getValue()) { Timer walkThreadLocalsTimer = timers.walkThreadLocals.open(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 69c7e8b43e31..05f89811bce0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -292,7 +292,7 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ImageSingletons.lookup(ParallelGC.class).startThreads(); + ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); /* * After starting all the necessary threads, we can finally execute complex JDK code or code diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 5b6cd58962c3..c2985a4e4874 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -5,73 +5,21 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.Safepoint; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; -import sun.misc.Signal; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; public class ParallelGC { private final VMMutex mutex = new VMMutex("pargc"); private final VMCondition cond = new VMCondition(mutex); - private Task task = new Task(); - private volatile boolean idle = true; - - public void setTask(Pointer src, Pointer dst, UnsignedWord size) { - mutex.lock(); - task.src = src; - task.dst = dst; - task.size = size; - mutex.unlock(); - - idle = false; - cond.signal(); - Log.log().string("setTask() on ").string(Thread.currentThread().getName()).newline(); - } - - public boolean isIdle() { - return idle; - } - - public void _startThreads() { - Log trace = Log.log(); - Thread t = new Thread(() -> { - trace.string(" started thread ").string(Thread.currentThread().getName()).newline(); - while (true) { - mutex.lock(); - while (idle) { - trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); - cond.block(); - trace.string(" unblocked on ").string(Thread.currentThread().getName()).newline(); - } - trace.string(" copying...").newline(); -// UnmanagedMemoryUtil.copyLongsForward(task.src, task.dst, task.size); - mutex.unlock(); - idle = true; - trace.string(" done").newline(); - } - }); - t.setName("ParallelGC"); - t.setDaemon(true); - trace.string("starting thread...").newline(); - t.start(); - } - - static class Task { - Pointer src, dst; - UnsignedWord size; - } - private volatile boolean notified; private volatile boolean stopped; -// public static Thread parThread; - - public void __startThreads() { + public void startWorkerThreads() { Log trace = Log.log(); Thread t = new Thread() { @Override @@ -83,47 +31,43 @@ public void run() { waitForNotification(); } } catch (Throwable e) { - VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); + VMError.shouldNotReachHere(e.getClass().getName()); } } }; t.setName("ParallelGC-noop"); t.setDaemon(true); -// parThread = t; t.start(); } private void waitForNotification() { Log trace = Log.log(); - trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); - while (!notified); - trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); - notified = false; + mutex.lock(); + try { + while (!notified) { + trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); + cond.block(); + trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); + } + notified = false; + } finally { + mutex.unlock(); + } } public void signal() { - Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); - notified = true; + mutex.lock(); + try { + Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); + notified = true; + cond.broadcast(); + } finally { + mutex.unlock(); + } } - public void startThreads() { - Log trace = Log.log(); - Thread t = new Thread() { - @Override - public void run() { - VMThreads.ParallelGCSupport.setParallelGCThread(); - try { - while (!stopped) { - waitForNotification(); - } - } catch (Throwable e) { - VMError.shouldNotReachHere("No exception must by thrown in the JFR recorder thread as this could break file IO operations."); - } - } - }; - t.setName("ParallelGC-noop"); - t.setDaemon(true); - t.start(); + public static void thawWorkerThreads() { + Safepoint.Master.singleton().releaseParallelGCSafepoints(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 9b46f41c7109..75adb6e78596 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.thread; import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.RelevantForCompilationIsolates; +import static com.oracle.svm.core.thread.VMThreads.StatusSupport.STATUS_IN_SAFEPOINT; import java.util.concurrent.atomic.AtomicInteger; @@ -881,16 +882,19 @@ private static void releaseSafepoints(String reason) { } /** Release each thread at a safepoint. */ - public static void releaseParallelGCSafepoints() { - final Log trace = Log.log().string("[Safepoint.Master.releasePGCSafepoints").newline(); + public void releaseParallelGCSafepoints() { + final Log trace = Log.log().string("[Safepoint.Master.release||safepoints").newline(); VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); // Set PGC thread statuses that are at safepoint back to being in native code. for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { if (trace.isEnabled()) { - trace.string(" PGC vmThread status: ").string(StatusSupport.getStatusString(vmThread)); + trace.string(" || vmThread status: ").string(StatusSupport.getStatusString(vmThread)); } + if (VMThreads.StatusSupport.getStatusVolatile(vmThread) != STATUS_IN_SAFEPOINT) { + continue; + } restoreSafepointRequestedValue(vmThread); /* @@ -944,7 +948,7 @@ public static int countingVMOperation() { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; // Check if the thread is at a safepoint or in native code. switch (status) { - case StatusSupport.STATUS_IN_SAFEPOINT: + case STATUS_IN_SAFEPOINT: atSafepoint += 1; break; default: From 3dfd353912677340fec46c76622f04336a1a6f7b Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 2 Mar 2022 11:29:26 +0300 Subject: [PATCH 003/125] OldGeneration.scanGreyObjects() runs on worker thread (but logging crashes) --- .../oracle/svm/core/genscavenge/GCImpl.java | 13 +++- .../svm/core/genscavenge/OldGeneration.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 55 +++++++++++++++++ .../com/oracle/svm/core/heap/ParallelGC.java | 60 ++++--------------- 4 files changed, 78 insertions(+), 52 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 97df1fce19f1..3d2d2e897261 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -682,7 +682,7 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { ParallelGC.thawWorkerThreads(); - ImageSingletons.lookup(ParallelGC.class).signal(); + parallelScanGreyObjects(false); /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -704,6 +704,17 @@ private void cheneyScanFromRoots() { } } + private void parallelScanGreyObjects(boolean isIncremental) { + Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); + try { + ParallelGC pgc = ImageSingletons.lookup(ParallelGC.class); + pgc.signal(false); + pgc.waitForNotification(false); + } finally { + scanGreyObjectsTimer.close(); + } + } + private void cheneyScanFromDirtyRoots() { Timer cheneyScanFromDirtyRootsTimer = timers.cheneyScanFromDirtyRoots.open(); try { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 8b3e042e34f1..1979a446eb32 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -106,7 +106,7 @@ void prepareForPromotion() { toGreyObjectsWalker.setScanStart(getToSpace()); } - boolean scanGreyObjects() { + public boolean scanGreyObjects() { if (!toGreyObjectsWalker.haveGreyObjects()) { return false; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java new file mode 100644 index 000000000000..3ba09508a08f --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -0,0 +1,55 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.genscavenge.HeapImpl; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.Safepoint; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.Feature; + +public class ParallelGCImpl extends ParallelGC { + + @Override + public void startWorkerThreads() { + Log trace = Log.log(); + Thread t = new Thread() { + @Override + @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) + public void run() { + VMThreads.ParallelGCSupport.setParallelGCThread(); + try { + while (!stopped) { + waitForNotification(true); + trace.string(" doing work on ").string(Thread.currentThread().getName()).newline(); + HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); + signal(true); + } + } catch (Throwable e) { + VMError.shouldNotReachHere(e.getClass().getName()); + } + } + }; + t.setName("ParallelGC-noop"); + t.setDaemon(true); + t.start(); + } +} + +@AutomaticFeature +class ParallelGCFeature implements Feature { +// @Override +// public boolean isInConfiguration(IsInConfigurationAccess access) { +// return SubstrateOptions.UseSerialGC.getValue(); +// } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(ParallelGC.class, new ParallelGCImpl()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index c2985a4e4874..9834c5e62a31 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,65 +1,38 @@ package com.oracle.svm.core.heap; -import com.oracle.svm.core.annotate.AutomaticFeature; -import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.Safepoint; -import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.util.VMError; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.hosted.Feature; -public class ParallelGC { +public abstract class ParallelGC { + protected final VMMutex mutex = new VMMutex("pargc"); + protected final VMCondition cond = new VMCondition(mutex); - private final VMMutex mutex = new VMMutex("pargc"); - private final VMCondition cond = new VMCondition(mutex); + protected volatile boolean workerThreadActive; + protected volatile boolean stopped; - private volatile boolean notified; - private volatile boolean stopped; + public abstract void startWorkerThreads(); - public void startWorkerThreads() { - Log trace = Log.log(); - Thread t = new Thread() { - @Override - @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) - public void run() { - VMThreads.ParallelGCSupport.setParallelGCThread(); - try { - while (!stopped) { - waitForNotification(); - } - } catch (Throwable e) { - VMError.shouldNotReachHere(e.getClass().getName()); - } - } - }; - t.setName("ParallelGC-noop"); - t.setDaemon(true); - t.start(); - } - - private void waitForNotification() { + public void waitForNotification(boolean onWorkerThread) { Log trace = Log.log(); mutex.lock(); try { - while (!notified) { + while (workerThreadActive != onWorkerThread) { trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); cond.block(); trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); } - notified = false; } finally { mutex.unlock(); } } - public void signal() { + public void signal(boolean onWorkerThread) { mutex.lock(); try { Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); - notified = true; + workerThreadActive = !onWorkerThread; cond.broadcast(); } finally { mutex.unlock(); @@ -70,16 +43,3 @@ public static void thawWorkerThreads() { Safepoint.Master.singleton().releaseParallelGCSafepoints(); } } - -@AutomaticFeature -class ParallelGCFeature implements Feature { -// @Override -// public boolean isInConfiguration(IsInConfigurationAccess access) { -// return SubstrateOptions.UseSerialGC.getValue(); -// } - - @Override - public void afterRegistration(AfterRegistrationAccess access) { - ImageSingletons.add(ParallelGC.class, new ParallelGC()); - } -} From ce6b1532086e7a768d4db7d062b853f26a265ff1 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 21 Apr 2022 10:47:21 +0300 Subject: [PATCH 004/125] Worker thread synchronizing via a queue, never put on safepoint --- .../oracle/svm/core/genscavenge/GCImpl.java | 19 +------ .../svm/core/genscavenge/OldGeneration.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 14 +++-- .../genscavenge/parallel/ParallelGCImpl.java | 12 ++-- .../core/genscavenge/parallel/TaskQueue.java | 57 +++++++++++++++++++ .../com/oracle/svm/core/heap/ParallelGC.java | 5 -- .../com/oracle/svm/core/thread/Safepoint.java | 34 +---------- 7 files changed, 78 insertions(+), 65 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 3d2d2e897261..2a21f0999204 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -29,6 +29,7 @@ import java.lang.ref.Reference; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; import com.oracle.svm.core.heap.ParallelGC; @@ -681,9 +682,6 @@ private void cheneyScanFromRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGC.thawWorkerThreads(); - parallelScanGreyObjects(false); - /* Visit all the Objects promoted since the snapshot. */ scanGreyObjects(false); @@ -704,17 +702,6 @@ private void cheneyScanFromRoots() { } } - private void parallelScanGreyObjects(boolean isIncremental) { - Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); - try { - ParallelGC pgc = ImageSingletons.lookup(ParallelGC.class); - pgc.signal(false); - pgc.waitForNotification(false); - } finally { - scanGreyObjectsTimer.close(); - } - } - private void cheneyScanFromDirtyRoots() { Timer cheneyScanFromDirtyRootsTimer = timers.cheneyScanFromDirtyRoots.open(); try { @@ -786,8 +773,8 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGC.thawWorkerThreads(); - ImageSingletons.lookup(ParallelGC.class).signal(); + ParallelGCImpl pargc = (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + pargc.QUEUE.put(getCollectionEpoch()); /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 1979a446eb32..8b3e042e34f1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -106,7 +106,7 @@ void prepareForPromotion() { toGreyObjectsWalker.setScanStart(getToSpace()); } - public boolean scanGreyObjects() { + boolean scanGreyObjects() { if (!toGreyObjectsWalker.haveGreyObjects()) { return false; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index c7ba62e9fcc7..3bad969edce6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -202,9 +202,10 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue()) { - VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); - } + ///assert owns mutex || pargc thread ? +// if (SubstrateOptions.MultiThreaded.getValue()) { +// VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); +// } appendAlignedHeapChunkUninterruptibly(aChunk); accounting.noteAlignedHeapChunk(); } @@ -255,9 +256,10 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue()) { - VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); - } + ///assert owns mutex || pargc thread ? +// if (SubstrateOptions.MultiThreaded.getValue()) { +// VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); +// } appendUnalignedHeapChunkUninterruptibly(uChunk); accounting.noteUnalignedHeapChunk(uChunk); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 3ba09508a08f..64a60b1dc1f2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -12,9 +12,12 @@ import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.UnsignedWord; public class ParallelGCImpl extends ParallelGC { + public final TaskQueue QUEUE = new TaskQueue("pargc"); + @Override public void startWorkerThreads() { Log trace = Log.log(); @@ -25,17 +28,15 @@ public void run() { VMThreads.ParallelGCSupport.setParallelGCThread(); try { while (!stopped) { - waitForNotification(true); - trace.string(" doing work on ").string(Thread.currentThread().getName()).newline(); - HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); - signal(true); + UnsignedWord token = QUEUE.get(); + trace.string(" got token ").unsigned(token).string(" on ").string(Thread.currentThread().getName()).newline(); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); } } }; - t.setName("ParallelGC-noop"); + t.setName("ParallelGC"); t.setDaemon(true); t.start(); } @@ -43,6 +44,7 @@ public void run() { @AutomaticFeature class ParallelGCFeature implements Feature { +/// // @Override // public boolean isInConfiguration(IsInConfigurationAccess access) { // return SubstrateOptions.UseSerialGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java new file mode 100644 index 000000000000..a0a29b376d45 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -0,0 +1,57 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +public class TaskQueue { + private static final UnsignedWord ZERO = WordFactory.zero(); + + private final VMMutex mutex; + private final VMCondition cond; + private UnsignedWord item; + + public TaskQueue(String name) { + mutex = new VMMutex(name + "-queue"); + cond = new VMCondition(mutex); + } + + public UnsignedWord get() { + try { + mutex.lock(); + while (item.equal(ZERO)) { + cond.block(); + } + return item; + } finally { + item = ZERO; + mutex.unlock(); + cond.signal(); + } + } + + public void put(UnsignedWord obj) { + try { + mutex.lock(); + while (item.notEqual(ZERO)) { + cond.block(); + } + item = obj; + } finally { + mutex.unlock(); + cond.signal(); + } + } + + public void waitUntilEmpty() { + try { + mutex.lock(); + while (item.notEqual(ZERO)) { + cond.block(); + } + } finally { + mutex.unlock(); + } + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 9834c5e62a31..0275910defab 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -3,7 +3,6 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.Safepoint; public abstract class ParallelGC { protected final VMMutex mutex = new VMMutex("pargc"); @@ -38,8 +37,4 @@ public void signal(boolean onWorkerThread) { mutex.unlock(); } } - - public static void thawWorkerThreads() { - Safepoint.Master.singleton().releaseParallelGCSafepoints(); - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 75adb6e78596..daa172b4b904 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -767,6 +767,8 @@ private static void waitForSafepoints(String reason) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints++; + } else if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { + ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; switch (status) { @@ -881,38 +883,6 @@ private static void releaseSafepoints(String reason) { trace.string("]").newline(); } - /** Release each thread at a safepoint. */ - public void releaseParallelGCSafepoints() { - final Log trace = Log.log().string("[Safepoint.Master.release||safepoints").newline(); - VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); - // Set PGC thread statuses that are at safepoint back to being in native code. - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { - if (trace.isEnabled()) { - trace.string(" || vmThread status: ").string(StatusSupport.getStatusString(vmThread)); - } - - if (VMThreads.StatusSupport.getStatusVolatile(vmThread) != STATUS_IN_SAFEPOINT) { - continue; - } - restoreSafepointRequestedValue(vmThread); - - /* - * Release the thread back to native code. Most threads will transition from - * safepoint to native; but some threads will already be in native code if they - * returned from native code, found the safepoint in progress and blocked on the - * mutex putting themselves back in native code again. - */ - StatusSupport.setStatusNative(vmThread); - Statistics.incReleased(); - if (trace.isEnabled()) { - trace.string(" -> ").string(StatusSupport.getStatusString(vmThread)).newline(); - } - } - } - trace.string("]").newline(); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected IsolateThread getRequestingThread() { return requestingThread; From 674df609d3fcc8c675c999637f5e538578f8f918 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 22 Apr 2022 12:17:54 +0300 Subject: [PATCH 005/125] Two parallel workers synchronizing via queue --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 3 +-- .../svm/core/genscavenge/YoungGeneration.java | 2 +- .../core/genscavenge/parallel/ParallelGCImpl.java | 13 ++++++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 2a21f0999204..bdbf529e0cf5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -773,8 +773,7 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGCImpl pargc = (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); - pargc.QUEUE.put(getCollectionEpoch()); + ParallelGCImpl.QUEUE.put(getCollectionEpoch()); /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 209fbad2731d..261420c69035 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -171,7 +171,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); + Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 64a60b1dc1f2..154dce3c6dd7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -14,12 +14,19 @@ import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.UnsignedWord; +import java.util.stream.IntStream; + public class ParallelGCImpl extends ParallelGC { - public final TaskQueue QUEUE = new TaskQueue("pargc"); + public static final int WORKERS_COUNT = 2; + public static final TaskQueue QUEUE = new TaskQueue("pargc"); @Override public void startWorkerThreads() { + IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); + } + + public void startWorkerThread(int n) { Log trace = Log.log(); Thread t = new Thread() { @Override @@ -29,14 +36,14 @@ public void run() { try { while (!stopped) { UnsignedWord token = QUEUE.get(); - trace.string(" got token ").unsigned(token).string(" on ").string(Thread.currentThread().getName()).newline(); + trace.string(" got token ").unsigned(token).string(" on PGCWorker-").unsigned(n).newline(); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); } } }; - t.setName("ParallelGC"); + t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); } From ae4f77f9bca76111928d3ad1e180a1109204288c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 31 May 2022 14:10:52 +0300 Subject: [PATCH 006/125] `releaseSpaces` stage working on 2 background threads --- .../oracle/svm/core/genscavenge/GCImpl.java | 5 +- .../svm/core/genscavenge/OldGeneration.java | 3 +- .../oracle/svm/core/genscavenge/Space.java | 35 ++++++- .../svm/core/genscavenge/YoungGeneration.java | 13 ++- .../genscavenge/parallel/ObjectQueue.java | 97 +++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 34 ++++--- .../core/genscavenge/parallel/TaskQueue.java | 1 + .../com/oracle/svm/core/thread/Safepoint.java | 9 +- 8 files changed, 170 insertions(+), 27 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index bdbf529e0cf5..1d6a0a09d38b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -32,10 +32,8 @@ import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; -import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -773,8 +771,6 @@ private void cheneyScanFromDirtyRoots() { startTicks = JfrGCEvents.startGCPhasePause(); try { - ParallelGCImpl.QUEUE.put(getCollectionEpoch()); - /* Visit all the Objects promoted since the snapshot, transitively. */ scanGreyObjects(true); @@ -1144,6 +1140,7 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } + ParallelGCImpl.waitForIdle(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 8b3e042e34f1..ebf45ded2431 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -99,7 +99,8 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig } void releaseSpaces(ChunkReleaser chunkReleaser) { - getFromSpace().releaseChunks(chunkReleaser); +// getFromSpace().setInd(1000); + getFromSpace().releaseChunksParallel(chunkReleaser); } void prepareForPromotion() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3bad969edce6..005cb7d0a26b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,6 +28,8 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import com.oracle.svm.core.annotate.AlwaysInline; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.locks.VMMutex; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -36,7 +38,6 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.MemoryWalker; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; @@ -45,7 +46,6 @@ import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; -import com.oracle.svm.core.thread.VMThreads; /** * A Space is a collection of HeapChunks. @@ -185,6 +185,37 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } + private int ind;///rm + + private final VMMutex mutex = new VMMutex("Space.ind.mutex"); + + private void setInd(int ind) { + try { + mutex.lock(); + if (ind < 10) { + Log.log().string("## ind ").unsigned(this.ind).string(" -> ").unsigned(ind).newline(); + } + this.ind = ind; + } finally { + mutex.unlock(); + } + } + + private int getInd() { + try { + mutex.lock(); + return ind; + } finally { + mutex.unlock(); + } + } + + public void releaseChunksParallel(ChunkReleaser chunkReleaser) { +// Log.log().string(" put space ").object(this) +// .string(", chunkReleaser ").object(chunkReleaser).newline(); + ParallelGCImpl.QUEUE.put(this, chunkReleaser); + } + public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 261420c69035..4c515d4019f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -136,13 +136,22 @@ private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { } void releaseSpaces(ChunkReleaser chunkReleaser) { - getEden().releaseChunks(chunkReleaser); +// getEden().setInd(333); + getEden().releaseChunksParallel(chunkReleaser); for (int i = 0; i < maxSurvivorSpaces; i++) { - getSurvivorFromSpaceAt(i).releaseChunks(chunkReleaser); +// getSurvivorFromSpaceAt(i).setInd(100+i); + getSurvivorFromSpaceAt(i).releaseChunksParallel(chunkReleaser); } } void swapSpaces() { +// try (Log trace = Log.log()) { +// trace.string(">>> eden ind=").unsigned(getEden().getInd()).newline(); +// for (int i = 0; i < maxSurvivorSpaces; i++) { +// trace.string(">>> ss").unsigned(i).string(" ind=") +// .unsigned(getSurvivorFromSpaceAt(i).getInd()).newline(); +// } +// } for (int i = 0; i < maxSurvivorSpaces; i++) { assert getSurvivorFromSpaceAt(i).isEmpty() : "Survivor fromSpace should be empty."; assert getSurvivorFromSpaceAt(i).getChunkBytes().equal(0) : "Chunk bytes must be 0"; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java new file mode 100644 index 000000000000..5b11cf0e19df --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java @@ -0,0 +1,97 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import java.util.function.BiConsumer; + +public class ObjectQueue { + private final VMMutex mutex; + private final VMCondition cond; + private final Object[] item; + private boolean empty; + private volatile int idleCount; + + public ObjectQueue(String name) { + mutex = new VMMutex(name + "-queue"); + cond = new VMCondition(mutex); + item = new Object[2]; + empty = true; + } + + private Object[] get() { +// Log.log().string(" >>> OQ.get() called").newline(); + try { + mutex.lock(); + idleCount++; + while (empty) { +// Log.log().string(" >>> OQ.get() WAIT").newline(); + cond.block(); + } +// Log.log().string(" >>> OQ.get() returns").newline(); + return item; + } finally { + empty = true; + idleCount--; + mutex.unlock(); + cond.broadcast(); + } + } + + public void put(Object value0, Object value1) { +// Log.log().string(" >>> OQ.put() called").newline(); + try { + mutex.lock(); + while (!empty) { +// Log.log().string(" >>> OQ.put() WAIT").newline(); + cond.block(); + } + item[0] = value0; + item[1] = value1; +// Log.log().string(" >>> OQ.put() returns").newline(); + } finally { + empty = false; + mutex.unlock(); + cond.broadcast(); + } + } + + public void consume(BiConsumer consumer) { + Object val0, val1; +// Log.log().string(" >>> OQ.consume() called").newline(); + try { + mutex.lock(); + idleCount++; + while (empty) { +// Log.log().string(" >>> OQ.consume() WAIT").newline(); + cond.block(); + } +// Log.log().string(" >>> OQ.consume() unblocks").newline(); + val0 = item[0]; + val1 = item[1]; + } finally { + empty = true; + idleCount--; + mutex.unlock(); + cond.broadcast(); + } + consumer.accept(val0, val1); + } + + public void waitUntilIdle(int expectedIdleCount) { + try { + mutex.lock(); +// Log.log().string(">>> OQ.wait() empty=").bool(empty) +// .string(", idle=").signed(idleCount).newline(); + while (!empty) { + cond.block(); + } + } finally { + mutex.unlock(); + } + while (idleCount < expectedIdleCount); + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 154dce3c6dd7..5ff308229c9d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,25 +1,23 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; -import com.oracle.svm.core.annotate.Uninterruptible; -import com.oracle.svm.core.genscavenge.HeapImpl; +import com.oracle.svm.core.genscavenge.GCImpl; +import com.oracle.svm.core.genscavenge.Space; import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.Safepoint; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.word.UnsignedWord; +import java.util.function.BiConsumer; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { + /// static -> ImageSingletons public static final int WORKERS_COUNT = 2; - public static final TaskQueue QUEUE = new TaskQueue("pargc"); + public static final ObjectQueue QUEUE = new ObjectQueue("pargc"); @Override public void startWorkerThreads() { @@ -27,16 +25,24 @@ public void startWorkerThreads() { } public void startWorkerThread(int n) { - Log trace = Log.log(); + final Log trace = Log.log(); + final BiConsumer releaseChunks = (s, cr) -> { + Space space = (Space) s; + GCImpl.ChunkReleaser chunkReleaser = (GCImpl.ChunkReleaser) cr; +// trace.string(" got space ").object(space) +// .string(", chunkReleaser ").object(chunkReleaser) +// .string(" on PGCWorker-").unsigned(n).newline(); + space.releaseChunks(chunkReleaser); +// space.setInd(n); + }; Thread t = new Thread() { @Override - @Uninterruptible(reason = "Trying to ignore safepoints", calleeMustBe = false) public void run() { - VMThreads.ParallelGCSupport.setParallelGCThread(); +// VMThreads.ParallelGCSupport.setParallelGCThread(); + VMThreads.SafepointBehavior.markThreadAsCrashed(); try { while (!stopped) { - UnsignedWord token = QUEUE.get(); - trace.string(" got token ").unsigned(token).string(" on PGCWorker-").unsigned(n).newline(); + QUEUE.consume(releaseChunks); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); @@ -47,6 +53,10 @@ public void run() { t.setDaemon(true); t.start(); } + + public static void waitForIdle() { + QUEUE.waitUntilIdle(WORKERS_COUNT); + } } @AutomaticFeature diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index a0a29b376d45..597a5d3276a9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -5,6 +5,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +///rm public class TaskQueue { private static final UnsignedWord ZERO = WordFactory.zero(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index daa172b4b904..71596b7cdb32 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -25,7 +25,6 @@ package com.oracle.svm.core.thread; import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.RelevantForCompilationIsolates; -import static com.oracle.svm.core.thread.VMThreads.StatusSupport.STATUS_IN_SAFEPOINT; import java.util.concurrent.atomic.AtomicInteger; @@ -666,7 +665,7 @@ private static boolean isMyself(IsolateThread thread) { /** Send each of the threads (except myself) a request to come to a safepoint. */ private static int requestSafepoints(String reason) { VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex while requesting a safepoint."); - final Log trace = Log.log().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); + final Log trace = Log.noopLog().string("[Safepoint.Master.requestSafepoints: reason: ").string(reason); int numOfThreads = 0; // Walk the threads list and ask each thread (except myself) to come to a safepoint. @@ -767,8 +766,6 @@ private static void waitForSafepoints(String reason) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints++; - } else if (VMThreads.ParallelGCSupport.isParallelGCThread(vmThread)) { - ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; switch (status) { @@ -856,7 +853,7 @@ private static void waitForSafepoints(String reason) { /** Release each thread at a safepoint. */ private static void releaseSafepoints(String reason) { - final Log trace = Log.log().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); + final Log trace = Log.noopLog().string("[Safepoint.Master.releaseSafepoints:").string(" reason: ").string(reason).newline(); VMThreads.THREAD_MUTEX.assertIsOwner("Must hold mutex when releasing safepoints."); // Set all the thread statuses that are at safepoint back to being in native code. for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { @@ -918,7 +915,7 @@ public static int countingVMOperation() { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; // Check if the thread is at a safepoint or in native code. switch (status) { - case STATUS_IN_SAFEPOINT: + case StatusSupport.STATUS_IN_SAFEPOINT: atSafepoint += 1; break; default: From 7db5c4928ab7767e68b2b5223ed20f4e6ce2952e Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 2 Jun 2022 15:10:19 +0300 Subject: [PATCH 007/125] Added a custom Timer to HeapImpl --- .../src/com/oracle/svm/core/genscavenge/HeapImpl.java | 1 + .../src/com/oracle/svm/core/genscavenge/Timers.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index d9d2b1ab8b31..342b077c949e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -106,6 +106,7 @@ public final class HeapImpl extends Heap { private final RuntimeCodeInfoGCSupportImpl runtimeCodeInfoGcSupport; private final ImageHeapInfo imageHeapInfo = new ImageHeapInfo(); private final HeapAccounting accounting = new HeapAccounting(); + public final Timer timer = new Timer("promoteUnalignedHeapChunk()"); /** Head of the linked list of currently pending (ready to be enqueued) {@link Reference}s. */ private Reference refPendingList; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index bd5b1cd255fd..e00701d54eaf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -150,6 +150,7 @@ void resetAllExceptMutator() { referenceObjects.reset(); releaseSpaces.reset(); verifyAfter.reset(); + HeapImpl.getHeapImpl().timer.reset(); /* The mutator timer is *not* reset here. */ trace.string("]").newline(); } @@ -171,6 +172,7 @@ void logAfterCollection(Log log) { logOneTimer(log, " ", blackenImageHeapRoots); logOneTimer(log, " ", blackenDirtyCardRoots); logOneTimer(log, " ", scanGreyObjects); + logOneTimer(log, " ", HeapImpl.getHeapImpl().timer); logOneTimer(log, " ", cleanCodeCache); logOneTimer(log, " ", referenceObjects); logOneTimer(log, " ", releaseSpaces); From aef1e38f567be73101232348651e0339da90d845 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Jun 2022 11:02:21 +0300 Subject: [PATCH 008/125] Parallel `dirtyCardIfNecessary` --- .../oracle/svm/core/genscavenge/GCImpl.java | 58 ++++++++++++++++- .../genscavenge/GreyToBlackObjRefVisitor.java | 27 ++++---- .../svm/core/genscavenge/OldGeneration.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 6 -- .../svm/core/genscavenge/YoungGeneration.java | 4 +- .../genscavenge/parallel/ParallelGCImpl.java | 37 +++++++---- .../core/genscavenge/parallel/TaskQueue.java | 65 ++++++++++++++----- 7 files changed, 149 insertions(+), 50 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 1d6a0a09d38b..9cf6a305b7fe 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -30,9 +30,12 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.TaskQueue; +import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -1039,6 +1042,7 @@ private static void prepareForPromotion(boolean isIncremental) { } private void scanGreyObjects(boolean isIncremental) { + ParallelGCImpl.setEnabled(true); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { if (isIncremental) { @@ -1049,6 +1053,8 @@ private void scanGreyObjects(boolean isIncremental) { } finally { scanGreyObjectsTimer.close(); } + ParallelGCImpl.setEnabled(false); + ParallelGCImpl.waitForIdle(); } private static void scanGreyObjectsLoop() { @@ -1096,6 +1102,57 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } + @AlwaysInline("GC performance") + @SuppressWarnings("static-method") + boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + /// from promObj + HeapImpl heap = HeapImpl.getHeapImpl(); + boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); + Header originalChunk = getChunk(original, isAligned); + Space originalSpace = HeapChunk.getSpace(originalChunk); + if (!originalSpace.isFromSpace() || !isAligned || innerOffset != 0 || !ParallelGCImpl.isEnabled()) { + return false; + } + + Object result = null; + if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { + result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + if (result == null) { + accounting.onSurvivorOverflowed(); + } + } + if (result == null) { // complete collection, tenuring age reached, or survivor space full + result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + assert result != null : "promotion failure in old generation must have been handled"; + } + + /// from visitor code + if (result != original) { + // ... update the reference to point to the copy, making the reference black. +// counters.noteCopiedReferent(); + Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + } else { + Log.log().string("PP unmod ref").newline();/// +// counters.noteUnmodifiedReference(); + } + + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. +// RememberedSet.get().dirtyCardIfNecessary(holderObject, result); + + ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + return true; + } + + public void doPromoteParallel(Object original, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + /// CAREFUL: result -> original + RememberedSet.get().dirtyCardIfNecessary(holderObject, original); + } + private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); @@ -1140,7 +1197,6 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - ParallelGCImpl.waitForIdle(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 95205bc7dc98..cb4c7588d820 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -102,19 +102,22 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - Object copy = GCImpl.getGCImpl().promoteObject(obj, header); - if (copy != obj) { - // ... update the reference to point to the copy, making the reference black. - counters.noteCopiedReferent(); - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); - } else { - counters.noteUnmodifiedReference(); + boolean scheduled = GCImpl.getGCImpl().promoteParallel(obj, header, objRef, innerOffset, compressed, holderObject); + if (!scheduled) { + Object copy = GCImpl.getGCImpl().promoteObject(obj, header); + if (copy != obj) { + // ... update the reference to point to the copy, making the reference black. + counters.noteCopiedReferent(); + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + } else { + counters.noteUnmodifiedReference(); + } + + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } - - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index ebf45ded2431..bb5da2af38a8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -100,7 +100,7 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig void releaseSpaces(ChunkReleaser chunkReleaser) { // getFromSpace().setInd(1000); - getFromSpace().releaseChunksParallel(chunkReleaser); + getFromSpace().releaseChunks(chunkReleaser); } void prepareForPromotion() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 005cb7d0a26b..23527a6b70bc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -210,12 +210,6 @@ private int getInd() { } } - public void releaseChunksParallel(ChunkReleaser chunkReleaser) { -// Log.log().string(" put space ").object(this) -// .string(", chunkReleaser ").object(chunkReleaser).newline(); - ParallelGCImpl.QUEUE.put(this, chunkReleaser); - } - public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 4c515d4019f5..16a1a0f293fb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -137,10 +137,10 @@ private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { void releaseSpaces(ChunkReleaser chunkReleaser) { // getEden().setInd(333); - getEden().releaseChunksParallel(chunkReleaser); + getEden().releaseChunks(chunkReleaser); for (int i = 0; i < maxSurvivorSpaces; i++) { // getSurvivorFromSpaceAt(i).setInd(100+i); - getSurvivorFromSpaceAt(i).releaseChunksParallel(chunkReleaser); + getSurvivorFromSpaceAt(i).releaseChunks(chunkReleaser); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 5ff308229c9d..050e11f44352 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -2,22 +2,27 @@ import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.genscavenge.GCImpl; -import com.oracle.svm.core.genscavenge.Space; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; -import java.util.function.BiConsumer; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 2; - public static final ObjectQueue QUEUE = new ObjectQueue("pargc"); + public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); + + public static final TaskQueue.Consumer PROMOTE_TASK = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); + }; + + private static boolean enabled; @Override public void startWorkerThreads() { @@ -26,15 +31,13 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { final Log trace = Log.log(); - final BiConsumer releaseChunks = (s, cr) -> { - Space space = (Space) s; - GCImpl.ChunkReleaser chunkReleaser = (GCImpl.ChunkReleaser) cr; -// trace.string(" got space ").object(space) -// .string(", chunkReleaser ").object(chunkReleaser) -// .string(" on PGCWorker-").unsigned(n).newline(); - space.releaseChunks(chunkReleaser); -// space.setInd(n); - }; +// final TaskQueue.Consumer promoteTask = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { +// trace.string(">> promote on worker-").unsigned(n).newline();/// +// if (original == null || holderObject == null) { +// trace.string("PP orig=").object(original).string(", holder=").object(holderObject).newline(); +// } +// GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); +// }; Thread t = new Thread() { @Override public void run() { @@ -42,7 +45,7 @@ public void run() { VMThreads.SafepointBehavior.markThreadAsCrashed(); try { while (!stopped) { - QUEUE.consume(releaseChunks); + QUEUE.consume(PROMOTE_TASK); } } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); @@ -57,6 +60,14 @@ public void run() { public static void waitForIdle() { QUEUE.waitUntilIdle(WORKERS_COUNT); } + + public static boolean isEnabled() { + return enabled; + } + + public static void setEnabled(boolean enabled) { + ParallelGCImpl.enabled = enabled; + } } @AutomaticFeature diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 597a5d3276a9..562b4855d065 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -2,57 +2,92 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -///rm public class TaskQueue { - private static final UnsignedWord ZERO = WordFactory.zero(); - private final VMMutex mutex; private final VMCondition cond; - private UnsignedWord item; + private Object object, holderObject; + private Pointer objRef; + private boolean compressed; + private boolean empty; + private volatile int idleCount; public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); + empty = true; } - public UnsignedWord get() { +// private UnsignedWord get() { +// try { +// mutex.lock(); +// while (empty) { +// cond.block(); +// } +// return word0; +// } finally { +// empty = true; +// mutex.unlock(); +// cond.signal(); +// } +// } + + public void put(Object original, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); - while (item.equal(ZERO)) { + while (!empty) { cond.block(); } - return item; + this.object = original; + this.objRef = objRef; + this.compressed = compressed; + this.holderObject = holderObject; } finally { - item = ZERO; + empty = false; mutex.unlock(); - cond.signal(); + cond.broadcast(); } } - public void put(UnsignedWord obj) { + public void consume(Consumer consumer) { + Object obj, owner; + Pointer ref; + boolean comp; try { mutex.lock(); - while (item.notEqual(ZERO)) { + idleCount++; + while (empty) { cond.block(); } - item = obj; + obj = this.object; + ref = this.objRef; + comp = this.compressed; + owner = this.holderObject; } finally { + empty = true; + idleCount--; mutex.unlock(); - cond.signal(); + cond.broadcast(); } + consumer.accept(obj, ref, comp, owner); } - public void waitUntilEmpty() { + public void waitUntilIdle(int expectedIdleCount) { try { mutex.lock(); - while (item.notEqual(ZERO)) { + while (!empty) { cond.block(); } } finally { mutex.unlock(); } + while (idleCount < expectedIdleCount);///signal? + } + + public interface Consumer { + void accept(Object original, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From 54683ecc30198606927a5bbcf3e9ef44a16c13a9 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Jun 2022 12:55:16 +0300 Subject: [PATCH 009/125] Parallel objRef update --- .../oracle/svm/core/genscavenge/GCImpl.java | 24 +++++++++---------- .../genscavenge/parallel/ParallelGCImpl.java | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 9cf6a305b7fe..53b5c9484c4a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -30,7 +30,6 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.jfr.JfrTicks; @@ -1130,27 +1129,26 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in if (result != original) { // ... update the reference to point to the copy, making the reference black. // counters.noteCopiedReferent(); - Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } else { Log.log().string("PP unmod ref").newline();/// // counters.noteUnmodifiedReference(); + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, result); } - - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. -// RememberedSet.get().dirtyCardIfNecessary(holderObject, result); - - ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } - public void doPromoteParallel(Object original, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + /// from visitor code + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + // The reference will not be updated if a whole chunk is promoted. However, we still // might have to dirty the card. - /// CAREFUL: result -> original - RememberedSet.get().dirtyCardIfNecessary(holderObject, original); + RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } private static Header getChunk(Object obj, boolean isAligned) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 050e11f44352..85c4b6cd3882 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -15,7 +15,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 2; + public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { From 2bfc96ec0a90ff2045646c7e023e5bf11ef0920c Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 8 Jun 2022 11:49:20 +0300 Subject: [PATCH 010/125] Passing `original` object safely via Pointer --- .../oracle/svm/core/genscavenge/GCImpl.java | 68 ++++++++++++++++--- .../oracle/svm/core/genscavenge/Space.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 8 ++- .../core/genscavenge/parallel/TaskQueue.java | 22 +++--- 4 files changed, 75 insertions(+), 25 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 53b5c9484c4a..1cbd338f1774 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -26,12 +26,16 @@ import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jfr.JfrTicks; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; @@ -1121,31 +1125,75 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in } } if (result == null) { // complete collection, tenuring age reached, or survivor space full - result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); +// result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + assert originalSpace.isFromSpace(); + Space toSpace = heap.getOldGeneration().getToSpace(); +// result = toSpace.promoteAlignedObject(original, originalSpace); + assert ObjectHeaderImpl.isAlignedObject(original); + assert toSpace != originalSpace && originalSpace.isFromSpace(); +// result = toSpace.copyAlignedObject(original); + assert VMOperation.isGCInProgress(); + assert ObjectHeaderImpl.isAlignedObject(original); + + UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + Pointer copyMemory = toSpace.allocateMemory(size); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + result = null; + } + + /* + * This does a direct memory copy, without regard to whether the copied data contains object + * references. That's okay, because all references in the copy are visited and overwritten + * later on anyways (the card table is also updated at that point if necessary). + */ + Pointer originalMemory = Word.objectToUntrackedPointer(original); + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); + result = copyMemory.toObject(); assert result != null : "promotion failure in old generation must have been handled"; + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(result); + RememberedSet.get().enableRememberedSetForObject(copyChunk, result); + } + if (result != null) { + ObjectHeaderImpl.installForwardingPointer(original, result); + } +// doPromoteParallel(original, result, objRef, innerOffset, compressed, holderObject); +// ParallelGCImpl.PROMOTE_TASK.accept(result, objRef, compressed, holderObject); + ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + return true; } /// from visitor code if (result != original) { // ... update the reference to point to the copy, making the reference black. + Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); // counters.noteCopiedReferent(); - ParallelGCImpl.QUEUE.put(result, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } else { Log.log().string("PP unmod ref").newline();/// // counters.noteUnmodifiedReference(); - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, result); } + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, result); return true; } - public void doPromoteParallel(Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + Object original = originalPtr.toObject(); /// from visitor code - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); - + if (copy != original) { + // ... update the reference to point to the copy, making the reference black. + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); +// counters.noteCopiedReferent(); + } else { + Log.log().string("PP unmod ref").newline();/// +// counters.noteUnmodifiedReference(); + } // The reference will not be updated if a whole chunk is promoted. However, we still // might have to dirty the card. RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 23527a6b70bc..7fccdfb502a2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -163,7 +163,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") - private Pointer allocateMemory(UnsignedWord objectSize) { + Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 85c4b6cd3882..69b346fd41b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -18,9 +18,10 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); - }; + public static final TaskQueue.Consumer PROMOTE_TASK = + (Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(originalPtr, copy, objRef, 0, compressed, holderObject); + }; private static boolean enabled; @@ -31,6 +32,7 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { final Log trace = Log.log(); + trace.string("PP start worker-").unsigned(n).newline(); // final TaskQueue.Consumer promoteTask = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { // trace.string(">> promote on worker-").unsigned(n).newline();/// // if (original == null || holderObject == null) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 562b4855d065..e9cda952af0f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -3,14 +3,12 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private Object object, holderObject; - private Pointer objRef; + private Object copy, holderObject; + private Pointer original, objRef; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -35,13 +33,14 @@ public TaskQueue(String name) { // } // } - public void put(Object original, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer original, Object copy, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } - this.object = original; + this.original = original; + this.copy = copy; this.objRef = objRef; this.compressed = compressed; this.holderObject = holderObject; @@ -53,8 +52,8 @@ public void put(Object original, Pointer objRef, boolean compressed, Object hold } public void consume(Consumer consumer) { - Object obj, owner; - Pointer ref; + Object cp, owner; + Pointer orig, ref; boolean comp; try { mutex.lock(); @@ -62,7 +61,8 @@ public void consume(Consumer consumer) { while (empty) { cond.block(); } - obj = this.object; + orig = this.original; + cp = this.copy; ref = this.objRef; comp = this.compressed; owner = this.holderObject; @@ -72,7 +72,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(obj, ref, comp, owner); + consumer.accept(orig, cp, ref, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -88,6 +88,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Object original, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From 7d692484dadcdd043d9d2dcce6181659d56e534f Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 8 Jun 2022 17:05:25 +0300 Subject: [PATCH 011/125] `installFwPtr` works but not on workers --- substratevm/mx.substratevm/suite.py | 1 + .../oracle/svm/core/genscavenge/GCImpl.java | 16 +++++---- .../core/genscavenge/ObjectHeaderImpl.java | 36 ++++++++++++++++--- .../genscavenge/parallel/ParallelGCImpl.java | 2 +- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index f0eb593c9901..342adf6795aa 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -323,6 +323,7 @@ "requiresConcealed" : { "java.base": [ "sun.nio.ch", + "jdk.internal.misc", ], "java.management": [ "sun.management", diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 1cbd338f1774..ca65ecdeeab7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1156,13 +1156,8 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(result); RememberedSet.get().enableRememberedSetForObject(copyChunk, result); } - if (result != null) { - ObjectHeaderImpl.installForwardingPointer(original, result); - } -// doPromoteParallel(original, result, objRef, innerOffset, compressed, holderObject); -// ParallelGCImpl.PROMOTE_TASK.accept(result, objRef, compressed, holderObject); ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1183,7 +1178,14 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in } public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + Log trace = Log.log(); + trace.string("PP obj=").hex(originalPtr) +// .string(", ref=").hex(objRef) + .newline(); Object original = originalPtr.toObject(); + if (copy != null) { + ObjectHeaderImpl.installForwardingPointer(original, copy); + } /// from visitor code if (copy != original) { // ... update the reference to point to the copy, making the reference black. @@ -1191,7 +1193,7 @@ public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); // counters.noteCopiedReferent(); } else { - Log.log().string("PP unmod ref").newline();/// + trace.string("PP unmod ref").newline();/// // counters.noteUnmodifiedReference(); } // The reference will not be updated if a whole chunk is promoted. However, we still diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 2926cb06235f..8f9c69a9ef9c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.log.Log; +import jdk.internal.misc.Unsafe; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.CompressEncoding; @@ -325,17 +327,23 @@ static Object getForwardedObject(Pointer ptr, UnsignedWord header) { } } + private static final Unsafe UNSAFE = Unsafe.getUnsafe(); + /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") - static void installForwardingPointer(Object original, Object copy) { - assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + static Object installForwardingPointer(Object original, Object copy) { + Word originalPtr = Word.objectToUntrackedPointer(original); +// assert !isPointerToForwardedObject(originalPtr); + UnsignedWord oldHeader = readHeaderFromPointer(originalPtr); +// assert !isForwardedHeader(oldHeader); UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + assert false: "getCompressEncoding().hasShift()"; // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); + ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); ///non mt safe } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); } @@ -343,8 +351,26 @@ static void installForwardingPointer(Object original, Object copy) { forwardHeader = Word.objectToUntrackedPointer(copy); } assert ObjectHeaderImpl.getHeaderBitsFromHeader(forwardHeader).equal(0); - writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); - assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); +// writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); + UnsignedWord newHeader = forwardHeader.or(FORWARDED_BIT); + boolean installed; + if (getReferenceSize() == Integer.BYTES) { +// ObjectAccess.writeInt(original, getHubOffset(), (int) newHeader.rawValue()); + int expected = (int) oldHeader.rawValue(); + int witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), expected, (int) newHeader.rawValue()); + installed = witness == expected; + } else { +// ObjectAccess.writeWord(original, getHubOffset(), newHeader); + long expected = oldHeader.rawValue(); + long witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), expected, newHeader.rawValue()); + installed = witness == expected; + } + assert isPointerToForwardedObject(originalPtr); + if (!installed) { + Log.log().string("PP collision for obj").hex(originalPtr).newline(); + return getForwardedObject(originalPtr); + } + return copy; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 69b346fd41b5..268fc368ab8c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -15,7 +15,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 4; + public static final int WORKERS_COUNT = 0; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = From f0368540cab973d326a466ab71e338f8e56737e0 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Jun 2022 14:47:26 +0300 Subject: [PATCH 012/125] `installForwardingPointer` works on parallel threads --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 8 ++++---- .../svm/core/genscavenge/ObjectHeaderImpl.java | 14 +++++++++----- .../core/genscavenge/parallel/ParallelGCImpl.java | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ca65ecdeeab7..f2360a0f641a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1157,7 +1157,7 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in RememberedSet.get().enableRememberedSetForObject(copyChunk, result); } ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); +// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1179,12 +1179,12 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { Log trace = Log.log(); - trace.string("PP obj=").hex(originalPtr) -// .string(", ref=").hex(objRef) + trace.string("PP obj ").hex(originalPtr) +// .string(", ref ").hex(objRef) .newline(); Object original = originalPtr.toObject(); if (copy != null) { - ObjectHeaderImpl.installForwardingPointer(original, copy); + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy } /// from visitor code if (copy != original) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 8f9c69a9ef9c..ac5c138b903f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -332,10 +332,14 @@ static Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") static Object installForwardingPointer(Object original, Object copy) { + Log trace = Log.log(); Word originalPtr = Word.objectToUntrackedPointer(original); // assert !isPointerToForwardedObject(originalPtr); - UnsignedWord oldHeader = readHeaderFromPointer(originalPtr); -// assert !isForwardedHeader(oldHeader); + UnsignedWord curHeader = readHeaderFromPointer(originalPtr); + if (isForwardedHeader(curHeader)) { + trace.string("PP reinstall for obj ").hex(originalPtr).newline(); + return getForwardedObject(originalPtr, curHeader); + } UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { @@ -356,18 +360,18 @@ static Object installForwardingPointer(Object original, Object copy) { boolean installed; if (getReferenceSize() == Integer.BYTES) { // ObjectAccess.writeInt(original, getHubOffset(), (int) newHeader.rawValue()); - int expected = (int) oldHeader.rawValue(); + int expected = (int) curHeader.rawValue(); int witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), expected, (int) newHeader.rawValue()); installed = witness == expected; } else { // ObjectAccess.writeWord(original, getHubOffset(), newHeader); - long expected = oldHeader.rawValue(); + long expected = curHeader.rawValue(); long witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), expected, newHeader.rawValue()); installed = witness == expected; } assert isPointerToForwardedObject(originalPtr); if (!installed) { - Log.log().string("PP collision for obj").hex(originalPtr).newline(); + trace.string("PP collision for obj ").hex(originalPtr).newline(); return getForwardedObject(originalPtr); } return copy; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 268fc368ab8c..69b346fd41b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -15,7 +15,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 0; + public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = From b8dd7c8805d8678e57d41273b2921fa04791cfe3 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Jun 2022 15:47:53 +0300 Subject: [PATCH 013/125] `enableRememberedSetForObject` works in parallel --- .../oracle/svm/core/genscavenge/GCImpl.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index f2360a0f641a..dbfc03447a4b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1150,13 +1150,7 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); result = copyMemory.toObject(); assert result != null : "promotion failure in old generation must have been handled"; - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(result); - RememberedSet.get().enableRememberedSetForObject(copyChunk, result); - } - ParallelGCImpl.QUEUE.put(Word.objectToUntrackedPointer(original), result, objRef, compressed, holderObject); + ParallelGCImpl.QUEUE.put(originalMemory, result, objRef, compressed, holderObject); // ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1182,6 +1176,16 @@ public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, trace.string("PP obj ").hex(originalPtr) // .string(", ref ").hex(objRef) .newline(); + + HeapImpl heap = HeapImpl.getHeapImpl(); + Space toSpace = heap.getOldGeneration().getToSpace(); + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } + Object original = originalPtr.toObject(); if (copy != null) { copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy From ac780158ee97c1e5c811f7c796381deb186fcefe Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Jun 2022 17:31:44 +0300 Subject: [PATCH 014/125] Protect against promoting an object twice. Crashes frequently in FOT (:183,:256), so use `-H:-UseRememberedSet` --- .../oracle/svm/core/genscavenge/GCImpl.java | 40 +++++++++++-------- .../genscavenge/parallel/ParallelGCImpl.java | 4 +- .../core/genscavenge/parallel/TaskQueue.java | 12 +++--- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index dbfc03447a4b..7b800cc93050 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1141,16 +1141,14 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in result = null; } + Pointer originalMemory = Word.objectToUntrackedPointer(original); /* * This does a direct memory copy, without regard to whether the copied data contains object * references. That's okay, because all references in the copy are visited and overwritten * later on anyways (the card table is also updated at that point if necessary). */ - Pointer originalMemory = Word.objectToUntrackedPointer(original); UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); - result = copyMemory.toObject(); - assert result != null : "promotion failure in old generation must have been handled"; - ParallelGCImpl.QUEUE.put(originalMemory, result, objRef, compressed, holderObject); + ParallelGCImpl.QUEUE.put(originalMemory, copyMemory, objRef, compressed, holderObject); // ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); return true; } @@ -1171,22 +1169,32 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in return true; } - public void doPromoteParallel(Pointer originalPtr, Object copy, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Pointer originalPtr, Pointer copyPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { Log trace = Log.log(); - trace.string("PP obj ").hex(originalPtr) + Object original = originalPtr.toObject(); + Object copy; + if (ObjectHeaderImpl.isPointerToForwardedObject(originalPtr)) { + copy = ObjectHeaderImpl.getForwardedObject(originalPtr); + } else { + UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + if (size.belowOrEqual(0)) { + trace.string("PP warn obj ").hex(originalPtr) + .string(" size ").signed(size) // .string(", ref ").hex(objRef) - .newline(); + .newline(); + } + copy = copyPtr.toObject(); + assert copy != null : "promotion failure in old generation must have been handled"; - HeapImpl heap = HeapImpl.getHeapImpl(); - Space toSpace = heap.getOldGeneration().getToSpace(); - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + HeapImpl heap = HeapImpl.getHeapImpl(); + Space toSpace = heap.getOldGeneration().getToSpace(); + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } } - - Object original = originalPtr.toObject(); if (copy != null) { copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 69b346fd41b5..14edfae9843e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -19,8 +19,8 @@ public class ParallelGCImpl extends ParallelGC { public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(originalPtr, copy, objRef, 0, compressed, holderObject); + (Pointer originalPtr, Pointer copyPtr, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(originalPtr, copyPtr, objRef, 0, compressed, holderObject); }; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index e9cda952af0f..51e6c28476f0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -7,8 +7,8 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private Object copy, holderObject; - private Pointer original, objRef; + private Object holderObject; + private Pointer original, copy, objRef; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -33,7 +33,7 @@ public TaskQueue(String name) { // } // } - public void put(Pointer original, Object copy, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer original, Pointer copy, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { @@ -52,8 +52,8 @@ public void put(Pointer original, Object copy, Pointer objRef, boolean compresse } public void consume(Consumer consumer) { - Object cp, owner; - Pointer orig, ref; + Object owner; + Pointer orig, cp, ref; boolean comp; try { mutex.lock(); @@ -88,6 +88,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer originalPtr, Object copy, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer originalPtr, Pointer copy, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From d8b11d0f985f3dfa41edf312dbcb93dc9ea5e07b Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 21 Jun 2022 13:38:43 +0300 Subject: [PATCH 015/125] Uninlined several methods --- .../com/oracle/svm/core/genscavenge/AlignedHeapChunk.java | 3 ++- .../com/oracle/svm/core/genscavenge/GreyObjectsWalker.java | 4 ++-- .../svm/core/genscavenge/GreyToBlackObjRefVisitor.java | 6 +++++- .../svm/core/genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../src/com/oracle/svm/core/genscavenge/HeapChunk.java | 2 +- .../svm/core/genscavenge/ReferenceObjectProcessing.java | 4 +++- .../src/com/oracle/svm/core/hub/InteriorObjRefWalker.java | 2 +- 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 15a6eef96953..b010ca3b9965 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; @@ -138,7 +139,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFrom(that, getObjectsStart(that), visitor); } - @AlwaysInline("GC performance") + @NeverInline("GC performance") static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 3788ec2572df..d67cb5c9333e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -84,7 +84,7 @@ void walkGreyObjects() { } } - @AlwaysInline("GC performance") + @NeverInline("GC performance") private void walkAlignedGreyObjects() { AlignedHeapChunk.AlignedHeader aChunk; if (alignedHeapChunk.isNull() && alignedTop.isNull()) { @@ -112,7 +112,7 @@ private void walkAlignedGreyObjects() { } } - @AlwaysInline("GC performance") + @NeverInline("GC performance") private void walkUnalignedGreyObjects() { /* Visit the Objects in the UnalignedChunk after the snapshot UnalignedChunk. */ UnalignedHeapChunk.UnalignedHeader uChunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index cb4c7588d820..ec5623540c82 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,6 +24,10 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.annotate.NeverInline; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; +import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -64,7 +68,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override - @AlwaysInline("GC performance") + @NeverInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 88adc11c4a0b..b6a3e3fda292 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -54,7 +54,7 @@ public boolean visitObject(Object o) { } @Override - @AlwaysInline("GC performance") + @NeverInline("GC performance") public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 962423f5c1f2..7f7740d00329 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -279,7 +279,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi return walkObjectsFromInline(that, offset, visitor); } - @AlwaysInline("GC performance") + @NeverInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 3c0a614336fb..7923285d6e2a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -31,6 +31,7 @@ import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; +import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -77,7 +78,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { softReferencesAreWeak = enabled; } - @AlwaysInline("GC performance") + @NeverInline("GC performance") public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -86,6 +87,7 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } + @NeverInline("GC performance") private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 0d4ae77e5d2f..86c3f78f6406 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -60,7 +60,7 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor return walkObjectInline(obj, visitor); } - @AlwaysInline("Performance critical version") + @NeverInline("Performance critical version") public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final int layoutEncoding = objHub.getLayoutEncoding(); From 0820d8d7b8316af4b275abf2271e2acd21181657 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 23 Jun 2022 15:59:18 +0300 Subject: [PATCH 016/125] Parallel memory allocation in to-space. Simple test works on 4 threads, HyperAlloc on 1 thread --- .../oracle/svm/core/genscavenge/GCImpl.java | 61 ++++++++++--------- .../svm/core/genscavenge/HeapChunk.java | 3 + .../svm/core/genscavenge/OldGeneration.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 58 +++++++----------- .../svm/core/genscavenge/YoungGeneration.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 18 ++---- .../core/genscavenge/parallel/TaskQueue.java | 12 ++-- 7 files changed, 74 insertions(+), 86 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 7b800cc93050..d625f80dd89a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1057,7 +1057,6 @@ private void scanGreyObjects(boolean isIncremental) { scanGreyObjectsTimer.close(); } ParallelGCImpl.setEnabled(false); - ParallelGCImpl.waitForIdle(); } private static void scanGreyObjectsLoop() { @@ -1113,6 +1112,10 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); Header originalChunk = getChunk(original, isAligned); Space originalSpace = HeapChunk.getSpace(originalChunk); + if (originalSpace == null) { + Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) + .string(" in chunk ").zhex(originalChunk).newline(); + } if (!originalSpace.isFromSpace() || !isAligned || innerOffset != 0 || !ParallelGCImpl.isEnabled()) { return false; } @@ -1135,21 +1138,11 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); - Pointer copyMemory = toSpace.allocateMemory(size); - if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { - result = null; - } - Pointer originalMemory = Word.objectToUntrackedPointer(original); - /* - * This does a direct memory copy, without regard to whether the copied data contains object - * references. That's okay, because all references in the copy are visited and overwritten - * later on anyways (the card table is also updated at that point if necessary). - */ - UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, size); - ParallelGCImpl.QUEUE.put(originalMemory, copyMemory, objRef, compressed, holderObject); -// ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + ParallelGCImpl.QUEUE.put(originalMemory, objRef, compressed, holderObject); + if (ParallelGCImpl.WORKERS_COUNT <= 0) { + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + } return true; } @@ -1169,34 +1162,46 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in return true; } - public void doPromoteParallel(Pointer originalPtr, Pointer copyPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void doPromoteParallel(Pointer originalPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { Log trace = Log.log(); Object original = originalPtr.toObject(); Object copy; if (ObjectHeaderImpl.isPointerToForwardedObject(originalPtr)) { copy = ObjectHeaderImpl.getForwardedObject(originalPtr); } else { + HeapImpl heap = HeapImpl.getHeapImpl(); + Space toSpace = heap.getOldGeneration().getToSpace(); UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); if (size.belowOrEqual(0)) { - trace.string("PP warn obj ").hex(originalPtr) + trace.string("PP warn obj ").zhex(originalPtr) .string(" size ").signed(size) -// .string(", ref ").hex(objRef) +// .string(", ref ").zhex(objRef) .newline(); } - copy = copyPtr.toObject(); - assert copy != null : "promotion failure in old generation must have been handled"; - HeapImpl heap = HeapImpl.getHeapImpl(); - Space toSpace = heap.getOldGeneration().getToSpace(); - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + Pointer copyPtr = toSpace.allocateMemory(size); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyPtr.isNull())) { + copy = null; + } else { + /* + * This does a direct memory copy, without regard to whether the copied data contains object + * references. That's okay, because all references in the copy are visited and overwritten + * later on anyways (the card table is also updated at that point if necessary). + */ + UnmanagedMemoryUtil.copyLongsForward(originalPtr, copyPtr, size); + copy = copyPtr.toObject(); + assert copy != null : "promotion failure in old generation must have been handled"; + + if (toSpace.isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } } } if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); ///can lose duplicate copy + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); } /// from visitor code if (copy != original) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 7f7740d00329..f987cc619df7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,6 +26,8 @@ import java.util.function.IntUnaryOperator; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -283,6 +285,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read + ParallelGCImpl.waitForIdle(); Object obj = offset.toObject(); if (!visitor.visitObjectInline(obj)) { return false; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index bb5da2af38a8..6661624b9822 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -27,6 +27,7 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; @@ -99,7 +100,6 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig } void releaseSpaces(ChunkReleaser chunkReleaser) { -// getFromSpace().setInd(1000); getFromSpace().releaseChunks(chunkReleaser); } @@ -108,10 +108,14 @@ void prepareForPromotion() { } boolean scanGreyObjects() { + Log trace = Log.log().string("").newline(); return false; } toGreyObjectsWalker.walkGreyObjects(); + ParallelGCImpl.waitForIdle(); + trace.string(">").newline(); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 7fccdfb502a2..af3582f27e96 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -27,9 +27,12 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AlwaysInline; +import com.oracle.svm.core.annotate.NeverInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.thread.VMThreads; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -58,6 +61,7 @@ public final class Space { private final boolean isFromSpace; private final int age; private final ChunksAccounting accounting; + private final VMMutex mutex; /* Heads and tails of the HeapChunk lists. */ private AlignedHeapChunk.AlignedHeader firstAlignedHeapChunk; @@ -82,6 +86,7 @@ public final class Space { this.isFromSpace = isFromSpace; this.age = age; this.accounting = new ChunksAccounting(accounting); + this.mutex = new VMMutex(name + "-mutex"); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -162,19 +167,27 @@ public Log report(Log log, boolean traceHeapChunks) { /** * Allocate memory from an AlignedHeapChunk in this Space. */ - @AlwaysInline("GC performance") + @NeverInline("GC performance") Pointer allocateMemory(UnsignedWord objectSize) { - Pointer result = WordFactory.nullPointer(); - /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); - if (oldChunk.isNonNull()) { - result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); - } - if (result.isNonNull()) { + mutex.lock(); + try { + Pointer result = WordFactory.nullPointer(); + /* Fast-path: try allocating in the last chunk. */ + AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); + if (oldChunk.isNonNull()) { + result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); + } + if (result.isNonNull()) { + Log.log().string(" alloc fast ").zhex(result).newline(); + return result; + } + /* Slow-path: try allocating a new chunk for the requested memory. */ + result = allocateInNewChunk(objectSize); + Log.log().string(" alloc slow ").zhex(result).newline(); return result; + } finally { + mutex.unlock(); } - /* Slow-path: try allocating a new chunk for the requested memory. */ - return allocateInNewChunk(objectSize); } private Pointer allocateInNewChunk(UnsignedWord objectSize) { @@ -185,31 +198,6 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } - private int ind;///rm - - private final VMMutex mutex = new VMMutex("Space.ind.mutex"); - - private void setInd(int ind) { - try { - mutex.lock(); - if (ind < 10) { - Log.log().string("## ind ").unsigned(this.ind).string(" -> ").unsigned(ind).newline(); - } - this.ind = ind; - } finally { - mutex.unlock(); - } - } - - private int getInd() { - try { - mutex.lock(); - return ind; - } finally { - mutex.unlock(); - } - } - public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 16a1a0f293fb..a34fe38f9082 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -180,7 +180,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:"); + Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 14edfae9843e..de516ea01e02 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -19,8 +19,8 @@ public class ParallelGCImpl extends ParallelGC { public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer originalPtr, Pointer copyPtr, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(originalPtr, copyPtr, objRef, 0, compressed, holderObject); + (Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(originalPtr, objRef, 0, compressed, holderObject); }; private static boolean enabled; @@ -33,16 +33,7 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { final Log trace = Log.log(); trace.string("PP start worker-").unsigned(n).newline(); -// final TaskQueue.Consumer promoteTask = (Object original, Pointer objRef, boolean compressed, Object holderObject) -> { -// trace.string(">> promote on worker-").unsigned(n).newline();/// -// if (original == null || holderObject == null) { -// trace.string("PP orig=").object(original).string(", holder=").object(holderObject).newline(); -// } -// GCImpl.getGCImpl().doPromoteParallel(original, objRef, 0, compressed, holderObject); -// }; - Thread t = new Thread() { - @Override - public void run() { + Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); try { @@ -52,8 +43,7 @@ public void run() { } catch (Throwable e) { VMError.shouldNotReachHere(e.getClass().getName()); } - } - }; + }); t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 51e6c28476f0..96792c8058ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -8,7 +8,7 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; private Object holderObject; - private Pointer original, copy, objRef; + private Pointer original, objRef; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -33,14 +33,13 @@ public TaskQueue(String name) { // } // } - public void put(Pointer original, Pointer copy, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer original, Pointer objRef, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } this.original = original; - this.copy = copy; this.objRef = objRef; this.compressed = compressed; this.holderObject = holderObject; @@ -53,7 +52,7 @@ public void put(Pointer original, Pointer copy, Pointer objRef, boolean compress public void consume(Consumer consumer) { Object owner; - Pointer orig, cp, ref; + Pointer orig, ref; boolean comp; try { mutex.lock(); @@ -62,7 +61,6 @@ public void consume(Consumer consumer) { cond.block(); } orig = this.original; - cp = this.copy; ref = this.objRef; comp = this.compressed; owner = this.holderObject; @@ -72,7 +70,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(orig, cp, ref, comp, owner); + consumer.accept(orig, ref, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -88,6 +86,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer originalPtr, Pointer copy, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject); } } \ No newline at end of file From 980155b6d201e561e770bbad2ac09f4bd404e888 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 10:41:31 +0300 Subject: [PATCH 017/125] Better sync strategy in `HeapChunk.walkObjectsFromInline` --- .../svm/core/genscavenge/HeapChunk.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index f987cc619df7..e733d3cf9f70 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -284,14 +284,20 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi @NeverInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; - while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read - ParallelGCImpl.waitForIdle(); - Object obj = offset.toObject(); - if (!visitor.visitObjectInline(obj)) { - return false; + Pointer top = getTopPointer(that); + Pointer p; + do { + p = top; + while (offset.belowThan(top)) { + Object obj = offset.toObject(); + if (!visitor.visitObjectInline(obj)) { + return false; + } + offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); } - offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); - } + ParallelGCImpl.waitForIdle(); + top = getTopPointer(that); + } while (top.aboveThan(p)); return true; } From 66aa7fca1d9ecf51a5c8ac01beb0669efdf362bd Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 12:20:57 +0300 Subject: [PATCH 018/125] Simplified parallel invocation, assuming complete gc, innerOffset=0, and aligned chunk --- .../oracle/svm/core/genscavenge/GCImpl.java | 104 ++++++++---------- .../genscavenge/GreyToBlackObjRefVisitor.java | 35 +++--- .../genscavenge/parallel/ParallelGCImpl.java | 4 +- .../core/genscavenge/parallel/TaskQueue.java | 14 ++- 4 files changed, 78 insertions(+), 79 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index d625f80dd89a..b56b212f8cd9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1104,30 +1104,55 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } - @AlwaysInline("GC performance") - @SuppressWarnings("static-method") - boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - /// from promObj - HeapImpl heap = HeapImpl.getHeapImpl(); - boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); - Header originalChunk = getChunk(original, isAligned); - Space originalSpace = HeapChunk.getSpace(originalChunk); - if (originalSpace == null) { - Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) - .string(" in chunk ").zhex(originalChunk).newline(); + public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + Log trace = Log.log(); + + /// from visitor code + assert innerOffset >= 0; + assert !objRef.isNull(); +// counters.noteObjRef(); + + Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); + assert offsetP.isNonNull() || innerOffset == 0; + + Pointer originalPtr = offsetP.subtract(innerOffset); + if (originalPtr.isNull()) { +// counters.noteNullReferent(); + return; } - if (!originalSpace.isFromSpace() || !isAligned || innerOffset != 0 || !ParallelGCImpl.isEnabled()) { - return false; + + if (HeapImpl.getHeapImpl().isInImageHeap(originalPtr)) { +// counters.noteNonHeapReferent(); + return; } - Object result = null; - if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { - result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - if (result == null) { - accounting.onSurvivorOverflowed(); + // This is the most expensive check as it accesses the heap fairly randomly, which results + // in a lot of cache misses. + UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalPtr); + boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); + + /// temp assertions + assert completeCollection; + assert innerOffset == 0; + assert isAligned; + + Object original = originalPtr.toObject(); + Object copy; + if (ObjectHeaderImpl.isForwardedHeader(header)) { +// counters.noteForwardedReferent(); + copy = ObjectHeaderImpl.getForwardedObject(originalPtr, header); + } else { + assert innerOffset < LayoutEncoding.getSizeFromObject(original).rawValue(); + + /// from promObj + HeapImpl heap = HeapImpl.getHeapImpl(); + Header originalChunk = getChunk(original, isAligned); + Space originalSpace = HeapChunk.getSpace(originalChunk); + if (originalSpace == null) { + Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) + .string(" in chunk ").zhex(originalChunk).newline(); } - } - if (result == null) { // complete collection, tenuring age reached, or survivor space full + // result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); assert originalSpace.isFromSpace(); Space toSpace = heap.getOldGeneration().getToSpace(); @@ -1138,39 +1163,6 @@ boolean promoteParallel(Object original, UnsignedWord header, Pointer objRef, in assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - Pointer originalMemory = Word.objectToUntrackedPointer(original); - ParallelGCImpl.QUEUE.put(originalMemory, objRef, compressed, holderObject); - if (ParallelGCImpl.WORKERS_COUNT <= 0) { - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); - } - return true; - } - - /// from visitor code - if (result != original) { - // ... update the reference to point to the copy, making the reference black. - Object offsetCopy = (innerOffset == 0) ? result : Word.objectToUntrackedPointer(result).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); -// counters.noteCopiedReferent(); - } else { - Log.log().string("PP unmod ref").newline();/// -// counters.noteUnmodifiedReference(); - } - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, result); - return true; - } - - public void doPromoteParallel(Pointer originalPtr, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - Log trace = Log.log(); - Object original = originalPtr.toObject(); - Object copy; - if (ObjectHeaderImpl.isPointerToForwardedObject(originalPtr)) { - copy = ObjectHeaderImpl.getForwardedObject(originalPtr); - } else { - HeapImpl heap = HeapImpl.getHeapImpl(); - Space toSpace = heap.getOldGeneration().getToSpace(); UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); if (size.belowOrEqual(0)) { trace.string("PP warn obj ").zhex(originalPtr) @@ -1199,9 +1191,9 @@ public void doPromoteParallel(Pointer originalPtr, Pointer objRef, int innerOffs RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } } - } - if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); + if (copy != null) { + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); + } } /// from visitor code if (copy != original) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index ec5623540c82..b8db2127656a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -70,6 +70,14 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h @Override @NeverInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + if (ParallelGCImpl.isEnabled()) { + ParallelGCImpl.QUEUE.put(objRef, innerOffset, compressed, holderObject); + if (ParallelGCImpl.WORKERS_COUNT <= 0) { + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + } + return true; + } + assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); @@ -106,22 +114,19 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - boolean scheduled = GCImpl.getGCImpl().promoteParallel(obj, header, objRef, innerOffset, compressed, holderObject); - if (!scheduled) { - Object copy = GCImpl.getGCImpl().promoteObject(obj, header); - if (copy != obj) { - // ... update the reference to point to the copy, making the reference black. - counters.noteCopiedReferent(); - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); - } else { - counters.noteUnmodifiedReference(); - } - - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); + Object copy = GCImpl.getGCImpl().promoteObject(obj, header); + if (copy != obj) { + // ... update the reference to point to the copy, making the reference black. + counters.noteCopiedReferent(); + Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); + ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + } else { + counters.noteUnmodifiedReference(); } + + // The reference will not be updated if a whole chunk is promoted. However, we still + // might have to dirty the card. + RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index de516ea01e02..8feb331b55de 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -19,8 +19,8 @@ public class ParallelGCImpl extends ParallelGC { public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(originalPtr, objRef, 0, compressed, holderObject); + (Pointer objRef, int innerOffset, boolean compressed, Object holderObject) -> { + GCImpl.getGCImpl().doPromoteParallel(objRef, innerOffset, compressed, holderObject); }; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 96792c8058ed..c1882813f53e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -8,7 +8,8 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; private Object holderObject; - private Pointer original, objRef; + private Pointer objRef; + private int innerOffset; private boolean compressed; private boolean empty; private volatile int idleCount; @@ -33,13 +34,13 @@ public TaskQueue(String name) { // } // } - public void put(Pointer original, Pointer objRef, boolean compressed, Object holderObject) { + public void put(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } - this.original = original; + this.innerOffset = innerOffset; this.objRef = objRef; this.compressed = compressed; this.holderObject = holderObject; @@ -53,6 +54,7 @@ public void put(Pointer original, Pointer objRef, boolean compressed, Object hol public void consume(Consumer consumer) { Object owner; Pointer orig, ref; + int offset; boolean comp; try { mutex.lock(); @@ -60,8 +62,8 @@ public void consume(Consumer consumer) { while (empty) { cond.block(); } - orig = this.original; ref = this.objRef; + offset = this.innerOffset; comp = this.compressed; owner = this.holderObject; } finally { @@ -70,7 +72,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(orig, ref, comp, owner); + consumer.accept(ref, offset, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -86,6 +88,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer originalPtr, Pointer objRef, boolean compressed, Object holderObject); + void accept(Pointer objRef, int innerOffset, boolean compressed, Object holderObject); } } \ No newline at end of file From d52639380f3e8fe0db1e6c332b406b04f36a611c Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 14:57:07 +0300 Subject: [PATCH 019/125] Restored call to `Space.promoteAlignedObject()` --- .../oracle/svm/core/genscavenge/GCImpl.java | 40 +------------------ 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b56b212f8cd9..825475997fb5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1156,44 +1156,8 @@ public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compresse // result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); assert originalSpace.isFromSpace(); Space toSpace = heap.getOldGeneration().getToSpace(); -// result = toSpace.promoteAlignedObject(original, originalSpace); - assert ObjectHeaderImpl.isAlignedObject(original); - assert toSpace != originalSpace && originalSpace.isFromSpace(); -// result = toSpace.copyAlignedObject(original); - assert VMOperation.isGCInProgress(); - assert ObjectHeaderImpl.isAlignedObject(original); - - UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); - if (size.belowOrEqual(0)) { - trace.string("PP warn obj ").zhex(originalPtr) - .string(" size ").signed(size) -// .string(", ref ").zhex(objRef) - .newline(); - } - - Pointer copyPtr = toSpace.allocateMemory(size); - if (probability(VERY_SLOW_PATH_PROBABILITY, copyPtr.isNull())) { - copy = null; - } else { - /* - * This does a direct memory copy, without regard to whether the copied data contains object - * references. That's okay, because all references in the copy are visited and overwritten - * later on anyways (the card table is also updated at that point if necessary). - */ - UnmanagedMemoryUtil.copyLongsForward(originalPtr, copyPtr, size); - copy = copyPtr.toObject(); - assert copy != null : "promotion failure in old generation must have been handled"; - - if (toSpace.isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); - } - } - if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); - } + copy = toSpace.promoteAlignedObject(original, originalSpace); + assert copy != null : "promotion failure in old generation must have been handled"; } /// from visitor code if (copy != original) { From 3bf93c06217bf3ce0c2fd048a38f3e7807c51953 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 15:06:26 +0300 Subject: [PATCH 020/125] Restored call to `GCImpl.promoteObject()` --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 825475997fb5..2f8034ed422f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1143,21 +1143,7 @@ public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compresse copy = ObjectHeaderImpl.getForwardedObject(originalPtr, header); } else { assert innerOffset < LayoutEncoding.getSizeFromObject(original).rawValue(); - - /// from promObj - HeapImpl heap = HeapImpl.getHeapImpl(); - Header originalChunk = getChunk(original, isAligned); - Space originalSpace = HeapChunk.getSpace(originalChunk); - if (originalSpace == null) { - Log.log().string("PP warn space is null for object ").zhex(Word.objectToUntrackedPointer(original)) - .string(" in chunk ").zhex(originalChunk).newline(); - } - -// result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - assert originalSpace.isFromSpace(); - Space toSpace = heap.getOldGeneration().getToSpace(); - copy = toSpace.promoteAlignedObject(original, originalSpace); - assert copy != null : "promotion failure in old generation must have been handled"; + copy = promoteObject(original, header); } /// from visitor code if (copy != original) { From 1b15e0103f05da6032497d5ba957f6bb7a49d547 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 24 Jun 2022 15:38:27 +0300 Subject: [PATCH 021/125] Restored call to `GreyToBlackObjRefVisitor.visitObjectReferenceInline()` --- .../oracle/svm/core/genscavenge/GCImpl.java | 56 ------------------- .../genscavenge/GreyToBlackObjRefVisitor.java | 15 ++++- .../genscavenge/parallel/ParallelGCImpl.java | 6 +- .../core/genscavenge/parallel/TaskQueue.java | 19 ++++--- 4 files changed, 27 insertions(+), 69 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 2f8034ed422f..ca1bf1d73a84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1104,62 +1104,6 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } - public void doPromoteParallel(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - Log trace = Log.log(); - - /// from visitor code - assert innerOffset >= 0; - assert !objRef.isNull(); -// counters.noteObjRef(); - - Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); - assert offsetP.isNonNull() || innerOffset == 0; - - Pointer originalPtr = offsetP.subtract(innerOffset); - if (originalPtr.isNull()) { -// counters.noteNullReferent(); - return; - } - - if (HeapImpl.getHeapImpl().isInImageHeap(originalPtr)) { -// counters.noteNonHeapReferent(); - return; - } - - // This is the most expensive check as it accesses the heap fairly randomly, which results - // in a lot of cache misses. - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalPtr); - boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); - - /// temp assertions - assert completeCollection; - assert innerOffset == 0; - assert isAligned; - - Object original = originalPtr.toObject(); - Object copy; - if (ObjectHeaderImpl.isForwardedHeader(header)) { -// counters.noteForwardedReferent(); - copy = ObjectHeaderImpl.getForwardedObject(originalPtr, header); - } else { - assert innerOffset < LayoutEncoding.getSizeFromObject(original).rawValue(); - copy = promoteObject(original, header); - } - /// from visitor code - if (copy != original) { - // ... update the reference to point to the copy, making the reference black. - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); -// counters.noteCopiedReferent(); - } else { - trace.string("PP unmod ref").newline();/// -// counters.noteUnmodifiedReference(); - } - // The reference will not be updated if a whole chunk is promoted. However, we still - // might have to dirty the card. - RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); - } - private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index b8db2127656a..5c5254d01f98 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -50,7 +50,7 @@ * Since this visitor is used during collection, one instance of it is constructed during native * image generation. */ -final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { +public final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { private final Counters counters; @Platforms(Platform.HOSTED_ONLY.class) @@ -71,13 +71,24 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h @NeverInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.QUEUE.put(objRef, innerOffset, compressed, holderObject); + /// temp assertions + Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); + assert ObjectHeaderImpl.isAlignedHeader(offsetP); + assert GCImpl.getGCImpl().isCompleteCollection(); + assert innerOffset == 0; + + ParallelGCImpl.QUEUE.put(this, objRef, innerOffset, compressed, holderObject); if (ParallelGCImpl.WORKERS_COUNT <= 0) { ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } return true; + } else { + return doVisitObjectReference(objRef, innerOffset, compressed, holderObject); } + } + @NeverInline("GC performance") + public boolean doVisitObjectReference(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 8feb331b55de..1844fce618f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -2,6 +2,7 @@ import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.genscavenge.GCImpl; +import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; @@ -18,10 +19,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = - (Pointer objRef, int innerOffset, boolean compressed, Object holderObject) -> { - GCImpl.getGCImpl().doPromoteParallel(objRef, innerOffset, compressed, holderObject); - }; + public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjRefVisitor::doVisitObjectReference; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index c1882813f53e..e31a30eea674 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -1,5 +1,6 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import org.graalvm.word.Pointer; @@ -7,10 +8,11 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private Object holderObject; + private GreyToBlackObjRefVisitor visitor; private Pointer objRef; private int innerOffset; private boolean compressed; + private Object holderObject; private boolean empty; private volatile int idleCount; @@ -34,14 +36,15 @@ public TaskQueue(String name) { // } // } - public void put(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { try { mutex.lock(); while (!empty) { cond.block(); } - this.innerOffset = innerOffset; + this.visitor = visitor; this.objRef = objRef; + this.innerOffset = innerOffset; this.compressed = compressed; this.holderObject = holderObject; } finally { @@ -52,16 +55,18 @@ public void put(Pointer objRef, int innerOffset, boolean compressed, Object hold } public void consume(Consumer consumer) { - Object owner; - Pointer orig, ref; + GreyToBlackObjRefVisitor v; + Pointer ref; int offset; boolean comp; + Object owner; try { mutex.lock(); idleCount++; while (empty) { cond.block(); } + v = this.visitor; ref = this.objRef; offset = this.innerOffset; comp = this.compressed; @@ -72,7 +77,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(ref, offset, comp, owner); + consumer.accept(v, ref, offset, comp, owner); } public void waitUntilIdle(int expectedIdleCount) { @@ -88,6 +93,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(Pointer objRef, int innerOffset, boolean compressed, Object holderObject); + void accept(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject); } } \ No newline at end of file From 043da3831e4c51980f321603a42bdf259d862630 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 27 Jun 2022 14:43:01 +0300 Subject: [PATCH 022/125] Extended TaskQueue to 1024 task items --- .../core/genscavenge/parallel/TaskQueue.java | 80 +++++++++++-------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index e31a30eea674..bf31fd95714f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -5,50 +5,59 @@ import com.oracle.svm.core.locks.VMMutex; import org.graalvm.word.Pointer; +import java.util.stream.IntStream; +import java.util.stream.Stream; + public class TaskQueue { + private static final int SIZE = 1024; + + private static class TaskData { + private GreyToBlackObjRefVisitor visitor; + private Pointer objRef; + private int innerOffset; + private boolean compressed; + private Object holderObject; + } + private final VMMutex mutex; private final VMCondition cond; - private GreyToBlackObjRefVisitor visitor; - private Pointer objRef; - private int innerOffset; - private boolean compressed; - private Object holderObject; - private boolean empty; + private final TaskData[] data; + private int getIndex; + private int putIndex; private volatile int idleCount; public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); - empty = true; + data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); } -// private UnsignedWord get() { -// try { -// mutex.lock(); -// while (empty) { -// cond.block(); -// } -// return word0; -// } finally { -// empty = true; -// mutex.unlock(); -// cond.signal(); -// } -// } + private int next(int index) { + return (index + 1) % SIZE; + } + + private boolean canGet() { + return getIndex != putIndex; + } + + private boolean canPut() { + return next(putIndex) != getIndex; + } public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { try { mutex.lock(); - while (!empty) { + while (!canPut()) { cond.block(); } - this.visitor = visitor; - this.objRef = objRef; - this.innerOffset = innerOffset; - this.compressed = compressed; - this.holderObject = holderObject; + TaskData item = data[putIndex]; + item.visitor = visitor; + item.objRef = objRef; + item.innerOffset = innerOffset; + item.compressed = compressed; + item.holderObject = holderObject; } finally { - empty = false; + putIndex = next(putIndex); mutex.unlock(); cond.broadcast(); } @@ -63,16 +72,17 @@ public void consume(Consumer consumer) { try { mutex.lock(); idleCount++; - while (empty) { + while (!canGet()) { cond.block(); } - v = this.visitor; - ref = this.objRef; - offset = this.innerOffset; - comp = this.compressed; - owner = this.holderObject; + TaskData item = data[getIndex]; + v = item.visitor; + ref = item.objRef; + offset = item.innerOffset; + comp = item.compressed; + owner = item.holderObject; } finally { - empty = true; + getIndex = next(getIndex); idleCount--; mutex.unlock(); cond.broadcast(); @@ -83,7 +93,7 @@ public void consume(Consumer consumer) { public void waitUntilIdle(int expectedIdleCount) { try { mutex.lock(); - while (!empty) { + while (canGet()) { cond.block(); } } finally { From 0d936b0620c152ccfac66057e3298be16a2a21cd Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 27 Jun 2022 15:27:22 +0300 Subject: [PATCH 023/125] Put objects on the queue rather than refs --- .../genscavenge/GreyToBlackObjRefVisitor.java | 26 +------------ .../genscavenge/GreyToBlackObjectVisitor.java | 24 ++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 6 +-- .../core/genscavenge/parallel/TaskQueue.java | 39 +++++++------------ 4 files changed, 42 insertions(+), 53 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 5c5254d01f98..5707069b1776 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,9 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.annotate.NeverInline; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; @@ -50,7 +47,7 @@ * Since this visitor is used during collection, one instance of it is constructed during native * image generation. */ -public final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { +final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { private final Counters counters; @Platforms(Platform.HOSTED_ONLY.class) @@ -68,27 +65,8 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override - @NeverInline("GC performance") + @AlwaysInline("GC performance") public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - if (ParallelGCImpl.isEnabled()) { - /// temp assertions - Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); - assert ObjectHeaderImpl.isAlignedHeader(offsetP); - assert GCImpl.getGCImpl().isCompleteCollection(); - assert innerOffset == 0; - - ParallelGCImpl.QUEUE.put(this, objRef, innerOffset, compressed, holderObject); - if (ParallelGCImpl.WORKERS_COUNT <= 0) { - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); - } - return true; - } else { - return doVisitObjectReference(objRef, innerOffset, compressed, holderObject); - } - } - - @NeverInline("GC performance") - public boolean doVisitObjectReference(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index b6a3e3fda292..1f0db3f27681 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -24,6 +24,11 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ReferenceAccess; +import org.graalvm.compiler.nodes.java.ArrayLengthNode; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -56,6 +61,25 @@ public boolean visitObject(Object o) { @Override @NeverInline("GC performance") public boolean visitObjectInline(Object o) { + if (ParallelGCImpl.isEnabled()) { + /// temp assertions +// Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); +// assert ObjectHeaderImpl.isAlignedHeader(offsetP); + assert GCImpl.getGCImpl().isCompleteCollection(); +// assert innerOffset == 0; + + ParallelGCImpl.QUEUE.put(this, o); + if (ParallelGCImpl.WORKERS_COUNT <= 0) { + ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); + } + return true; + } else { + return doVisitObject(o); + } + } + + @NeverInline("GC performance") + public boolean doVisitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 1844fce618f3..3a4d59512212 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,15 +1,13 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; -import com.oracle.svm.core.genscavenge.GCImpl; -import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; +import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.word.Pointer; import java.util.stream.IntStream; @@ -19,7 +17,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjRefVisitor::doVisitObjectReference; + public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjectVisitor::doVisitObject; private static boolean enabled; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index bf31fd95714f..a9abe242b696 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -1,22 +1,18 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.genscavenge.GreyToBlackObjRefVisitor; +import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; -import org.graalvm.word.Pointer; +import com.oracle.svm.core.log.Log; import java.util.stream.IntStream; -import java.util.stream.Stream; public class TaskQueue { - private static final int SIZE = 1024; + private static final int SIZE = 1024; ///handle overflow private static class TaskData { - private GreyToBlackObjRefVisitor visitor; - private Pointer objRef; - private int innerOffset; - private boolean compressed; - private Object holderObject; + private GreyToBlackObjectVisitor visitor; + private Object object; } private final VMMutex mutex; @@ -44,18 +40,16 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + public void put(GreyToBlackObjectVisitor visitor, Object object) { try { mutex.lock(); while (!canPut()) { + Log.log().string("PP cannot put task\n"); cond.block(); } TaskData item = data[putIndex]; item.visitor = visitor; - item.objRef = objRef; - item.innerOffset = innerOffset; - item.compressed = compressed; - item.holderObject = holderObject; + item.object = object; } finally { putIndex = next(putIndex); mutex.unlock(); @@ -64,30 +58,25 @@ public void put(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffse } public void consume(Consumer consumer) { - GreyToBlackObjRefVisitor v; - Pointer ref; - int offset; - boolean comp; - Object owner; + GreyToBlackObjectVisitor v; + Object obj; try { mutex.lock(); idleCount++; while (!canGet()) { + Log.log().string("PP cannot get task\n"); cond.block(); } TaskData item = data[getIndex]; v = item.visitor; - ref = item.objRef; - offset = item.innerOffset; - comp = item.compressed; - owner = item.holderObject; + obj = item.object; } finally { getIndex = next(getIndex); idleCount--; mutex.unlock(); cond.broadcast(); } - consumer.accept(v, ref, offset, comp, owner); + consumer.accept(v, obj); } public void waitUntilIdle(int expectedIdleCount) { @@ -103,6 +92,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(GreyToBlackObjRefVisitor visitor, Pointer objRef, int innerOffset, boolean compressed, Object holderObject); + void accept(GreyToBlackObjectVisitor visitor, Object object); } } \ No newline at end of file From d47b6d29809d02ff3962b96cc0835b0c24eda1f2 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 11:52:52 +0300 Subject: [PATCH 024/125] Don't put G2BObjectVisitor instances on TaskQueue --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 2 +- .../svm/core/genscavenge/GreyObjectsWalker.java | 1 + .../core/genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../core/genscavenge/parallel/ParallelGCImpl.java | 8 +++++++- .../svm/core/genscavenge/parallel/TaskQueue.java | 13 ++++--------- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ca1bf1d73a84..cc7ac3a37d18 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1204,7 +1204,7 @@ public static boolean hasNeverCollectPolicy() { return getPolicy() instanceof NeverCollect; } - GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { + public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { return greyToBlackObjectVisitor; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index d67cb5c9333e..1e8a9478081c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -126,6 +126,7 @@ private void walkUnalignedGreyObjects() { UnalignedHeapChunk.UnalignedHeader lastChunk; do { lastChunk = uChunk; + Log.log().string("PP walk unaligned in chunk").zhex(uChunk).newline(); if (!UnalignedHeapChunk.walkObjectsInline(uChunk, visitor)) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 1f0db3f27681..02a5be92a5c6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -68,7 +68,7 @@ public boolean visitObjectInline(Object o) { assert GCImpl.getGCImpl().isCompleteCollection(); // assert innerOffset == 0; - ParallelGCImpl.QUEUE.put(this, o); + ParallelGCImpl.QUEUE.put(o); if (ParallelGCImpl.WORKERS_COUNT <= 0) { ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 3a4d59512212..9b7407477644 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; @@ -17,7 +18,8 @@ public class ParallelGCImpl extends ParallelGC { public static final int WORKERS_COUNT = 4; public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = GreyToBlackObjectVisitor::doVisitObject; + public static final TaskQueue.Consumer PROMOTE_TASK = + obj -> getVisitor().doVisitObject(obj); private static boolean enabled; @@ -45,6 +47,10 @@ public void startWorkerThread(int n) { t.start(); } + public static GreyToBlackObjectVisitor getVisitor() { + return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); + } + public static void waitForIdle() { QUEUE.waitUntilIdle(WORKERS_COUNT); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index a9abe242b696..0b4375bf7008 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -1,6 +1,5 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; @@ -11,8 +10,7 @@ public class TaskQueue { private static final int SIZE = 1024; ///handle overflow private static class TaskData { - private GreyToBlackObjectVisitor visitor; - private Object object; + private Object object;///no need to wrap } private final VMMutex mutex; @@ -40,7 +38,7 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(GreyToBlackObjectVisitor visitor, Object object) { + public void put(Object object) { try { mutex.lock(); while (!canPut()) { @@ -48,7 +46,6 @@ public void put(GreyToBlackObjectVisitor visitor, Object object) { cond.block(); } TaskData item = data[putIndex]; - item.visitor = visitor; item.object = object; } finally { putIndex = next(putIndex); @@ -58,7 +55,6 @@ public void put(GreyToBlackObjectVisitor visitor, Object object) { } public void consume(Consumer consumer) { - GreyToBlackObjectVisitor v; Object obj; try { mutex.lock(); @@ -68,7 +64,6 @@ public void consume(Consumer consumer) { cond.block(); } TaskData item = data[getIndex]; - v = item.visitor; obj = item.object; } finally { getIndex = next(getIndex); @@ -76,7 +71,7 @@ public void consume(Consumer consumer) { mutex.unlock(); cond.broadcast(); } - consumer.accept(v, obj); + consumer.accept(obj); } public void waitUntilIdle(int expectedIdleCount) { @@ -92,6 +87,6 @@ public void waitUntilIdle(int expectedIdleCount) { } public interface Consumer { - void accept(GreyToBlackObjectVisitor visitor, Object object); + void accept(Object object); ///j.u.f.Consumer } } \ No newline at end of file From 239c849458c0c27aed4fa2622c96455e5a266410 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 13:05:03 +0300 Subject: [PATCH 025/125] Added max queue size statistic --- .../oracle/svm/core/genscavenge/GCImpl.java | 3 +++ .../core/genscavenge/parallel/TaskQueue.java | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index cc7ac3a37d18..3da4b389410d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -212,6 +212,7 @@ private void collectOperation(CollectionVMOperationData data) { GCCause cause = GCCause.fromId(data.getCauseId()); printGCBefore(cause.getName()); + ParallelGCImpl.QUEUE.stats.reset(); boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); printGCAfter(cause.getName()); @@ -436,6 +437,8 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } + verboseGCLog.newline().string(" task queue max size: ") + .unsigned(ParallelGCImpl.QUEUE.stats.getMaxSize()).newline(); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 0b4375bf7008..76513cc8967e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -13,6 +13,8 @@ private static class TaskData { private Object object;///no need to wrap } + public final Stats stats; + private final VMMutex mutex; private final VMCondition cond; private final TaskData[] data; @@ -24,6 +26,7 @@ public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); + stats = new Stats(); } private int next(int index) { @@ -49,6 +52,7 @@ public void put(Object object) { item.object = object; } finally { putIndex = next(putIndex); + stats.noteSize(putIndex, getIndex); mutex.unlock(); cond.broadcast(); } @@ -67,6 +71,7 @@ public void consume(Consumer consumer) { obj = item.object; } finally { getIndex = next(getIndex); + stats.noteSize(putIndex, getIndex); idleCount--; mutex.unlock(); cond.broadcast(); @@ -89,4 +94,26 @@ public void waitUntilIdle(int expectedIdleCount) { public interface Consumer { void accept(Object object); ///j.u.f.Consumer } + + public static class Stats { + private int maxSize; + + public void noteSize(int putIndex, int getIndex) { + int size = putIndex - getIndex; + if (size < 0) { + size += SIZE; + } + if (size > maxSize) { + maxSize = size; + } + } + + public int getMaxSize() { + return maxSize; + } + + public void reset() { + maxSize = 0; + } + } } \ No newline at end of file From e04d49a5da06e9209f378b847a9eb081cea509c7 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 16:43:37 +0300 Subject: [PATCH 026/125] Made TaskQueue private in ParallelGCImpl --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 4 ++-- .../genscavenge/GreyToBlackObjectVisitor.java | 5 +---- .../core/genscavenge/parallel/ParallelGCImpl.java | 15 +++++++++++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 3da4b389410d..c099dbda75cc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -212,7 +212,7 @@ private void collectOperation(CollectionVMOperationData data) { GCCause cause = GCCause.fromId(data.getCauseId()); printGCBefore(cause.getName()); - ParallelGCImpl.QUEUE.stats.reset(); + ParallelGCImpl.getStats().reset(); boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); printGCAfter(cause.getName()); @@ -438,7 +438,7 @@ private void printGCAfter(String cause) { timers.logAfterCollection(verboseGCLog); } verboseGCLog.newline().string(" task queue max size: ") - .unsigned(ParallelGCImpl.QUEUE.stats.getMaxSize()).newline(); + .unsigned(ParallelGCImpl.getStats().getMaxSize()).newline(); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 02a5be92a5c6..8ac694e4bebc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -68,10 +68,7 @@ public boolean visitObjectInline(Object o) { assert GCImpl.getGCImpl().isCompleteCollection(); // assert innerOffset == 0; - ParallelGCImpl.QUEUE.put(o); - if (ParallelGCImpl.WORKERS_COUNT <= 0) { - ParallelGCImpl.QUEUE.consume(ParallelGCImpl.PROMOTE_TASK); - } + ParallelGCImpl.queue(o); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 9b7407477644..14c699191732 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -16,9 +16,9 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; - public static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); - public static final TaskQueue.Consumer PROMOTE_TASK = + private static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); + private static final TaskQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); private static boolean enabled; @@ -51,6 +51,13 @@ public static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } + public static void queue(Object obj) { + QUEUE.put(obj); + if (WORKERS_COUNT <= 0) { // execute synchronously + QUEUE.consume(PROMOTE_TASK); + } + } + public static void waitForIdle() { QUEUE.waitUntilIdle(WORKERS_COUNT); } @@ -62,6 +69,10 @@ public static boolean isEnabled() { public static void setEnabled(boolean enabled) { ParallelGCImpl.enabled = enabled; } + + public static TaskQueue.Stats getStats() { + return QUEUE.stats; + } } @AutomaticFeature From 6b692e2f968795c2bc1df684dbecb4851fc7ab00 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 28 Jun 2022 16:50:52 +0300 Subject: [PATCH 027/125] Queue grey objects instead of scanning chunks --- .../core/genscavenge/GreyObjectsWalker.java | 2 +- .../svm/core/genscavenge/HeapChunk.java | 23 ++++++++----------- .../oracle/svm/core/genscavenge/Space.java | 4 ++++ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 1e8a9478081c..f3c221915709 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -126,7 +126,7 @@ private void walkUnalignedGreyObjects() { UnalignedHeapChunk.UnalignedHeader lastChunk; do { lastChunk = uChunk; - Log.log().string("PP walk unaligned in chunk").zhex(uChunk).newline(); + Log.log().string("PP walk unaligned in chunk ").zhex(uChunk).newline(); if (!UnalignedHeapChunk.walkObjectsInline(uChunk, visitor)) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index e733d3cf9f70..74cc025ff5c3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,7 +26,6 @@ import java.util.function.IntUnaryOperator; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; @@ -281,23 +280,19 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi return walkObjectsFromInline(that, offset, visitor); } - @NeverInline("GC performance") + @AlwaysInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { + /// beware: this is called for both aligned and unaligned objects. Make sure both types are + /// put on the task queue. Currently only aligned ones are. Pointer offset = startOffset; Pointer top = getTopPointer(that); - Pointer p; - do { - p = top; - while (offset.belowThan(top)) { - Object obj = offset.toObject(); - if (!visitor.visitObjectInline(obj)) { - return false; - } - offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); + while (offset.belowThan(top)) { + Object obj = offset.toObject(); + if (!visitor.visitObjectInline(obj)) { + return false; } - ParallelGCImpl.waitForIdle(); - top = getTopPointer(that); - } while (top.aboveThan(p)); + offset = offset.add(LayoutEncoding.getSizeFromObjectInline(obj)); + } return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index af3582f27e96..f347742add55 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -397,11 +397,15 @@ private Object copyAlignedObject(Object originalObj) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } + if (ParallelGCImpl.isEnabled()) { + ParallelGCImpl.queue(copy); + } return copy; } /** Promote an AlignedHeapChunk by moving it to this space. */ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { + /// called from GCImpl.promoteChunksWithPinnedObjects() early in the GC routine assert this != originalSpace && originalSpace.isFromSpace(); originalSpace.extractAlignedHeapChunk(chunk); From 69bbd0dabd004acf1c60791ffd628c19b8bf31f4 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 30 Jun 2022 12:25:14 +0300 Subject: [PATCH 028/125] Queue unaligned objects for parallel scan --- .../svm/core/genscavenge/GreyObjectsWalker.java | 1 - .../com/oracle/svm/core/genscavenge/HeapChunk.java | 2 -- .../src/com/oracle/svm/core/genscavenge/Space.java | 12 ++++++++++-- .../svm/core/genscavenge/UnalignedHeapChunk.java | 2 ++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index f3c221915709..d67cb5c9333e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -126,7 +126,6 @@ private void walkUnalignedGreyObjects() { UnalignedHeapChunk.UnalignedHeader lastChunk; do { lastChunk = uChunk; - Log.log().string("PP walk unaligned in chunk ").zhex(uChunk).newline(); if (!UnalignedHeapChunk.walkObjectsInline(uChunk, visitor)) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 74cc025ff5c3..a81495658463 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -282,8 +282,6 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi @AlwaysInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { - /// beware: this is called for both aligned and unaligned objects. Make sure both types are - /// put on the task queue. Currently only aligned ones are. Pointer offset = startOffset; Pointer top = getTopPointer(that); while (offset.belowThan(top)) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index f347742add55..3d8c321cfbfc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -178,12 +179,12 @@ Pointer allocateMemory(UnsignedWord objectSize) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } if (result.isNonNull()) { - Log.log().string(" alloc fast ").zhex(result).newline(); +// Log.log().string(" alloc fast ").zhex(result).newline(); return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ result = allocateInNewChunk(objectSize); - Log.log().string(" alloc slow ").zhex(result).newline(); +// Log.log().string(" alloc slow ").zhex(result).newline(); return result; } finally { mutex.unlock(); @@ -425,9 +426,16 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); + Log.log().string("UA promote chunk ").zhex(chunk).newline(); originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); + if (ParallelGCImpl.isEnabled()) { + if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { + throw VMError.shouldNotReachHere(); + } + } + if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { RememberedSet.get().enableRememberedSetForChunk(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 299309cb970b..ad12df2e7cac 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.log.Log; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; @@ -146,6 +147,7 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { @AlwaysInline("GC performance") public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { + Log.log().string("UA walk unaligned in chunk ").zhex(that).newline(); return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } From 23b19acc36f2b6a2fe1fa1fb18370042806786b2 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 30 Jun 2022 13:50:01 +0300 Subject: [PATCH 029/125] TaskQueue fixes: `drain` and `idleCount` --- .../oracle/svm/core/genscavenge/GCImpl.java | 5 ++- .../genscavenge/parallel/ParallelGCImpl.java | 9 ++-- .../core/genscavenge/parallel/TaskQueue.java | 43 ++++++++++++++----- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c099dbda75cc..9431f5256f49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -437,8 +437,9 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } - verboseGCLog.newline().string(" task queue max size: ") - .unsigned(ParallelGCImpl.getStats().getMaxSize()).newline(); + TaskQueue.Stats stats = ParallelGCImpl.getStats(); + verboseGCLog.newline().string(" tasks: ").unsigned(stats.getCount()) + .string(", max size: ").unsigned(stats.getMaxSize()).newline(); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 14c699191732..280a967b5503 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -53,13 +53,14 @@ public static GreyToBlackObjectVisitor getVisitor() { public static void queue(Object obj) { QUEUE.put(obj); - if (WORKERS_COUNT <= 0) { // execute synchronously - QUEUE.consume(PROMOTE_TASK); - } } public static void waitForIdle() { - QUEUE.waitUntilIdle(WORKERS_COUNT); + if (WORKERS_COUNT > 0) { + QUEUE.waitUntilIdle(WORKERS_COUNT); + } else { + QUEUE.drain(PROMOTE_TASK); // execute synchronously + } } public static boolean isEnabled() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 76513cc8967e..52d6db141f0d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -4,6 +4,7 @@ import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; public class TaskQueue { @@ -13,19 +14,20 @@ private static class TaskData { private Object object;///no need to wrap } - public final Stats stats; + final Stats stats; private final VMMutex mutex; private final VMCondition cond; private final TaskData[] data; + private final AtomicInteger idleCount; private int getIndex; private int putIndex; - private volatile int idleCount; public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); + idleCount = new AtomicInteger(0); stats = new Stats(); } @@ -52,7 +54,7 @@ public void put(Object object) { item.object = object; } finally { putIndex = next(putIndex); - stats.noteSize(putIndex, getIndex); + stats.noteTask(putIndex, getIndex); mutex.unlock(); cond.broadcast(); } @@ -60,9 +62,9 @@ public void put(Object object) { public void consume(Consumer consumer) { Object obj; + idleCount.incrementAndGet(); + mutex.lock(); try { - mutex.lock(); - idleCount++; while (!canGet()) { Log.log().string("PP cannot get task\n"); cond.block(); @@ -71,15 +73,27 @@ public void consume(Consumer consumer) { obj = item.object; } finally { getIndex = next(getIndex); - stats.noteSize(putIndex, getIndex); - idleCount--; mutex.unlock(); + idleCount.decrementAndGet(); cond.broadcast(); } consumer.accept(obj); } + // Non MT safe. Only call when no workers are running + public void drain(Consumer consumer) { + idleCount.decrementAndGet(); + while (canGet()) { + TaskData item = data[getIndex]; + Object obj = item.object; + consumer.accept(obj); + getIndex = next(getIndex); + } + idleCount.incrementAndGet(); + } + public void waitUntilIdle(int expectedIdleCount) { + Log log = Log.log().string("PP waitForIdle ").unsigned(idleCount.get()).newline(); try { mutex.lock(); while (canGet()) { @@ -88,17 +102,19 @@ public void waitUntilIdle(int expectedIdleCount) { } finally { mutex.unlock(); } - while (idleCount < expectedIdleCount);///signal? + while (idleCount.get() < expectedIdleCount);///signal? + log.string("PP waitForIdle over").newline(); } public interface Consumer { void accept(Object object); ///j.u.f.Consumer } + // Non MT safe, needs locking public static class Stats { - private int maxSize; + private int count, maxSize; - public void noteSize(int putIndex, int getIndex) { + void noteTask(int putIndex, int getIndex) { int size = putIndex - getIndex; if (size < 0) { size += SIZE; @@ -106,6 +122,11 @@ public void noteSize(int putIndex, int getIndex) { if (size > maxSize) { maxSize = size; } + count++; + } + + public int getCount() { + return count; } public int getMaxSize() { @@ -113,7 +134,7 @@ public int getMaxSize() { } public void reset() { - maxSize = 0; + count = maxSize = 0; } } } \ No newline at end of file From 57e82756923c60acf869dc1c7e41382c31c39d4b Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 30 Jun 2022 15:00:35 +0300 Subject: [PATCH 030/125] Made TaskQueue reference-free --- .../genscavenge/GreyToBlackObjectVisitor.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 2 +- .../genscavenge/parallel/ObjectQueue.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 5 ++-- .../core/genscavenge/parallel/TaskQueue.java | 25 +++++++------------ 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 8ac694e4bebc..bda0a27f92e1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -68,7 +68,7 @@ public boolean visitObjectInline(Object o) { assert GCImpl.getGCImpl().isCompleteCollection(); // assert innerOffset == 0; - ParallelGCImpl.queue(o); + ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3d8c321cfbfc..403b92961fba 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -399,7 +399,7 @@ private Object copyAlignedObject(Object originalObj) { RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.queue(copy); + ParallelGCImpl.queue(copyMemory); } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java index 5b11cf0e19df..b256b6805437 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java @@ -8,7 +8,7 @@ import java.util.function.BiConsumer; -public class ObjectQueue { +public class ObjectQueue {///rm private final VMMutex mutex; private final VMCondition cond; private final Object[] item; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 280a967b5503..9dcf15e9aee0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -9,6 +9,7 @@ import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; import java.util.stream.IntStream; @@ -51,8 +52,8 @@ public static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - public static void queue(Object obj) { - QUEUE.put(obj); + public static void queue(Pointer ptr) { + QUEUE.put(ptr); } public static void waitForIdle() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 52d6db141f0d..64ce4e512c98 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -3,22 +3,18 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.IntStream; public class TaskQueue { private static final int SIZE = 1024; ///handle overflow - private static class TaskData { - private Object object;///no need to wrap - } - final Stats stats; private final VMMutex mutex; private final VMCondition cond; - private final TaskData[] data; + private final Pointer[] data; /// move out of heap? private final AtomicInteger idleCount; private int getIndex; private int putIndex; @@ -26,7 +22,7 @@ private static class TaskData { public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); - data = IntStream.range(0, SIZE).mapToObj(n -> new TaskData()).toArray(TaskData[]::new); + data = new Pointer[SIZE]; idleCount = new AtomicInteger(0); stats = new Stats(); } @@ -43,15 +39,14 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(Object object) { + public void put(Pointer ptr) { try { mutex.lock(); while (!canPut()) { Log.log().string("PP cannot put task\n"); cond.block(); } - TaskData item = data[putIndex]; - item.object = object; + data[putIndex] = ptr; } finally { putIndex = next(putIndex); stats.noteTask(putIndex, getIndex); @@ -61,7 +56,7 @@ public void put(Object object) { } public void consume(Consumer consumer) { - Object obj; + Pointer ptr; idleCount.incrementAndGet(); mutex.lock(); try { @@ -69,23 +64,21 @@ public void consume(Consumer consumer) { Log.log().string("PP cannot get task\n"); cond.block(); } - TaskData item = data[getIndex]; - obj = item.object; + ptr = data[getIndex]; } finally { getIndex = next(getIndex); mutex.unlock(); idleCount.decrementAndGet(); cond.broadcast(); } - consumer.accept(obj); + consumer.accept(ptr.toObject()); } // Non MT safe. Only call when no workers are running public void drain(Consumer consumer) { idleCount.decrementAndGet(); while (canGet()) { - TaskData item = data[getIndex]; - Object obj = item.object; + Object obj = data[getIndex].toObject(); consumer.accept(obj); getIndex = next(getIndex); } From c0129b412690f9f68622b088b77a1734f16b0b02 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 4 Jul 2022 12:03:19 +0300 Subject: [PATCH 031/125] Protect `GCImpl.promoteObject` with critical section --- .../oracle/svm/core/genscavenge/GCImpl.java | 49 ++++++++++++------- .../genscavenge/GreyToBlackObjRefVisitor.java | 2 +- .../core/genscavenge/ObjectHeaderImpl.java | 4 +- .../oracle/svm/core/genscavenge/Space.java | 10 +++- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 9431f5256f49..b110275a4d1d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1076,7 +1076,8 @@ private static void scanGreyObjectsLoop() { @AlwaysInline("GC performance") @SuppressWarnings("static-method") - Object promoteObject(Object original, UnsignedWord header) { + Object promoteObject(Pointer p, UnsignedWord header) { + Object original = p.toObject(); HeapImpl heap = HeapImpl.getHeapImpl(); boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); Header originalChunk = getChunk(original, isAligned); @@ -1085,27 +1086,39 @@ Object promoteObject(Object original, UnsignedWord header) { return original; } - Object result = null; - if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { - if (isAligned) { - result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + // enter critical section + originalSpace.lock(); + try { + // reread header as it might have changed + header = ObjectHeaderImpl.readHeaderFromObject(original); + if (ObjectHeaderImpl.isForwardedHeader(header)) { + return ObjectHeaderImpl.getForwardedObject(p, header); } - if (result == null) { - accounting.onSurvivorOverflowed(); + /// mutex here and Unsafe.cas in OHI.installFwPtr -- check for redundancy + Object result = null; + if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { + if (isAligned) { + result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + } + if (result == null) { + accounting.onSurvivorOverflowed(); + } } - } - if (result == null) { // complete collection, tenuring age reached, or survivor space full - if (isAligned) { - result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + if (result == null) { // complete collection, tenuring age reached, or survivor space full + if (isAligned) { + result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + } + assert result != null : "promotion failure in old generation must have been handled"; } - assert result != null : "promotion failure in old generation must have been handled"; - } - return result; + return result; + } finally { + originalSpace.unlock(); + } } private static Header getChunk(Object obj, boolean isAligned) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 5707069b1776..0ff70d74fb9d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -103,7 +103,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - Object copy = GCImpl.getGCImpl().promoteObject(obj, header); + Object copy = GCImpl.getGCImpl().promoteObject(p, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index ac5c138b903f..6009ef596b8a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -337,7 +337,7 @@ static Object installForwardingPointer(Object original, Object copy) { // assert !isPointerToForwardedObject(originalPtr); UnsignedWord curHeader = readHeaderFromPointer(originalPtr); if (isForwardedHeader(curHeader)) { - trace.string("PP reinstall for obj ").hex(originalPtr).newline(); + trace.string("PP reinstall for obj ").zhex(originalPtr).newline(); return getForwardedObject(originalPtr, curHeader); } UnsignedWord forwardHeader; @@ -371,7 +371,7 @@ static Object installForwardingPointer(Object original, Object copy) { } assert isPointerToForwardedObject(originalPtr); if (!installed) { - trace.string("PP collision for obj ").hex(originalPtr).newline(); + trace.string("PP collision for obj ").zhex(originalPtr).newline(); return getForwardedObject(originalPtr); } return copy; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 403b92961fba..5d4025dbc4f6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -165,6 +165,14 @@ public Log report(Log log, boolean traceHeapChunks) { return log; } + void lock() { + mutex.lock(); + } + + void unlock() { + mutex.unlock(); + } + /** * Allocate memory from an AlignedHeapChunk in this Space. */ @@ -367,7 +375,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { Object copy = copyAlignedObject(original); if (copy != null) { - ObjectHeaderImpl.installForwardingPointer(original, copy); + copy = ObjectHeaderImpl.installForwardingPointer(original, copy); } return copy; } From ff66f18375d0e9d07fd05f7cca49a52069474fa8 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 4 Jul 2022 14:52:23 +0300 Subject: [PATCH 032/125] Fixed `TaskQueue.waitUntilIdle` --- .../core/genscavenge/parallel/TaskQueue.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 64ce4e512c98..1304b32e9827 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -5,8 +5,6 @@ import com.oracle.svm.core.log.Log; import org.graalvm.word.Pointer; -import java.util.concurrent.atomic.AtomicInteger; - public class TaskQueue { private static final int SIZE = 1024; ///handle overflow @@ -15,7 +13,7 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; private final Pointer[] data; /// move out of heap? - private final AtomicInteger idleCount; + private int idleCount; private int getIndex; private int putIndex; @@ -23,7 +21,6 @@ public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = new Pointer[SIZE]; - idleCount = new AtomicInteger(0); stats = new Stats(); } @@ -57,18 +54,18 @@ public void put(Pointer ptr) { public void consume(Consumer consumer) { Pointer ptr; - idleCount.incrementAndGet(); mutex.lock(); try { while (!canGet()) { Log.log().string("PP cannot get task\n"); + idleCount++; cond.block(); + idleCount--; } ptr = data[getIndex]; } finally { getIndex = next(getIndex); mutex.unlock(); - idleCount.decrementAndGet(); cond.broadcast(); } consumer.accept(ptr.toObject()); @@ -76,27 +73,29 @@ public void consume(Consumer consumer) { // Non MT safe. Only call when no workers are running public void drain(Consumer consumer) { - idleCount.decrementAndGet(); while (canGet()) { Object obj = data[getIndex].toObject(); consumer.accept(obj); getIndex = next(getIndex); } - idleCount.incrementAndGet(); } public void waitUntilIdle(int expectedIdleCount) { - Log log = Log.log().string("PP waitForIdle ").unsigned(idleCount.get()).newline(); - try { - mutex.lock(); - while (canGet()) { - cond.block(); + Log log = Log.log().string("PP waitForIdle\n"); + while (true) { + try { + mutex.lock(); + while (canGet()) { + cond.block(); + } + if (idleCount >= expectedIdleCount) { + log.string("PP waitForIdle over\n"); + return; + } + } finally { + mutex.unlock(); } - } finally { - mutex.unlock(); } - while (idleCount.get() < expectedIdleCount);///signal? - log.string("PP waitForIdle over").newline(); } public interface Consumer { From af6f70b53a153fa48dd7c5bbe749966dd17068e0 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 5 Jul 2022 15:42:36 +0300 Subject: [PATCH 033/125] Removed locking on fromSpace --- .../oracle/svm/core/genscavenge/GCImpl.java | 49 ++++++--------- .../genscavenge/GreyToBlackObjRefVisitor.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 62 ++++++++++++++++--- .../svm/core/hub/InteriorObjRefWalker.java | 4 ++ 4 files changed, 76 insertions(+), 41 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b110275a4d1d..9431f5256f49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1076,8 +1076,7 @@ private static void scanGreyObjectsLoop() { @AlwaysInline("GC performance") @SuppressWarnings("static-method") - Object promoteObject(Pointer p, UnsignedWord header) { - Object original = p.toObject(); + Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); Header originalChunk = getChunk(original, isAligned); @@ -1086,39 +1085,27 @@ Object promoteObject(Pointer p, UnsignedWord header) { return original; } - // enter critical section - originalSpace.lock(); - try { - // reread header as it might have changed - header = ObjectHeaderImpl.readHeaderFromObject(original); - if (ObjectHeaderImpl.isForwardedHeader(header)) { - return ObjectHeaderImpl.getForwardedObject(p, header); + Object result = null; + if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { + if (isAligned) { + result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); } - /// mutex here and Unsafe.cas in OHI.installFwPtr -- check for redundancy - Object result = null; - if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { - if (isAligned) { - result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); - } - if (result == null) { - accounting.onSurvivorOverflowed(); - } + if (result == null) { + accounting.onSurvivorOverflowed(); } - if (result == null) { // complete collection, tenuring age reached, or survivor space full - if (isAligned) { - result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); - } else { - result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); - } - assert result != null : "promotion failure in old generation must have been handled"; + } + if (result == null) { // complete collection, tenuring age reached, or survivor space full + if (isAligned) { + result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + } else { + result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); } - - return result; - } finally { - originalSpace.unlock(); + assert result != null : "promotion failure in old generation must have been handled"; } + + return result; } private static Header getChunk(Object obj, boolean isAligned) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 0ff70d74fb9d..5707069b1776 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -103,7 +103,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); - Object copy = GCImpl.getGCImpl().promoteObject(p, header); + Object copy = GCImpl.getGCImpl().promoteObject(obj, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 5d4025dbc4f6..ca20abfd4f97 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -165,14 +165,6 @@ public Log report(Log log, boolean traceHeapChunks) { return log; } - void lock() { - mutex.lock(); - } - - void unlock() { - mutex.unlock(); - } - /** * Allocate memory from an AlignedHeapChunk in this Space. */ @@ -373,13 +365,65 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); + if (ParallelGCImpl.isEnabled()) { + return promoteAlignedObjectParallel(original, originalSpace); + } + Object copy = copyAlignedObject(original); if (copy != null) { - copy = ObjectHeaderImpl.installForwardingPointer(original, copy); + ObjectHeaderImpl.installForwardingPointer(original, copy); } return copy; } + @AlwaysInline("GC performance") + Object promoteAlignedObjectParallel(Object original, Space originalSpace) { + assert VMOperation.isGCInProgress(); + assert ObjectHeaderImpl.isAlignedObject(original); + + UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + Pointer copyMemory = allocateMemory(size); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + return null; + } + Object copy = copyMemory.toObject(); + if (copy == null) { + return null; + } + + // Read original header + Pointer originalMemory = Word.objectToUntrackedPointer(original); + assert ObjectHeaderImpl.getHubOffset() == 0; + UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); + // Install forwarding pointer into the original header + /// mv CAS code from OHI here + Object forward = ObjectHeaderImpl.installForwardingPointer(original, copy); + + if (forward == copy) { + // We have won the race, now we must copy the object bits. First install the original header + int headerSize = ObjectHeaderImpl.getReferenceSize(); + if (headerSize == Integer.BYTES) { + copyMemory.writeInt(0, (int) header.rawValue()); + } else { + copyMemory.writeWord(0, header); + } + // copy the rest of original object + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); + + if (isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } + ParallelGCImpl.queue(copyMemory); + return copy; + } else { + /// Now the allocated memory is lost. Retract in TLAB? + return forward; + } + } + @AlwaysInline("GC performance") private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 86c3f78f6406..87e13af29b94 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.hub; +import com.oracle.svm.core.log.Log; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.word.Word; import org.graalvm.word.Pointer; @@ -65,6 +66,9 @@ public static boolean walkObjectInline(final Object obj, final ObjectReferenceVi final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final int layoutEncoding = objHub.getLayoutEncoding(); final Pointer objPointer = Word.objectToUntrackedPointer(obj); + if (objHub.getName() == null) { + Log.log().string("PP warn NULL obj ").zhex(objPointer).newline(); + } // Visit each Object reference in the array part of the Object. if (LayoutEncoding.isArrayLikeWithObjectElements(layoutEncoding)) { From 972002c8b084611572cf12f987a40f6382c2fbb1 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 6 Jul 2022 10:04:09 +0300 Subject: [PATCH 034/125] `scanGreyObjects` cleanup --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 11 ++++++----- .../core/genscavenge/GreyToBlackObjectVisitor.java | 6 ------ .../oracle/svm/core/genscavenge/OldGeneration.java | 7 +++---- .../src/com/oracle/svm/core/genscavenge/Space.java | 3 --- .../oracle/svm/core/genscavenge/YoungGeneration.java | 6 ++++-- .../svm/core/genscavenge/parallel/ParallelGCImpl.java | 2 +- .../svm/core/genscavenge/parallel/TaskQueue.java | 8 ++++---- .../com/oracle/svm/core/hub/InteriorObjRefWalker.java | 2 +- 8 files changed, 19 insertions(+), 26 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 9431f5256f49..5748b7c636f6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1049,18 +1049,19 @@ private static void prepareForPromotion(boolean isIncremental) { } private void scanGreyObjects(boolean isIncremental) { - ParallelGCImpl.setEnabled(true); + HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); + ParallelGCImpl.setEnabled(true); try { if (isIncremental) { - scanGreyObjectsLoop(); - } else { - HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); + heap.getYoungGeneration().scanGreyObjects(); } + heap.getOldGeneration().scanGreyObjects(); } finally { + ParallelGCImpl.waitForIdle(); + ParallelGCImpl.setEnabled(false); scanGreyObjectsTimer.close(); } - ParallelGCImpl.setEnabled(false); } private static void scanGreyObjectsLoop() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index bda0a27f92e1..4ec0d030e795 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -62,12 +62,6 @@ public boolean visitObject(Object o) { @NeverInline("GC performance") public boolean visitObjectInline(Object o) { if (ParallelGCImpl.isEnabled()) { - /// temp assertions -// Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); -// assert ObjectHeaderImpl.isAlignedHeader(offsetP); - assert GCImpl.getGCImpl().isCompleteCollection(); -// assert innerOffset == 0; - ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); return true; } else { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 6661624b9822..f6f2c9d9f24a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -108,14 +108,13 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("").newline(); + trace.string("]").newline(); return false; } toGreyObjectsWalker.walkGreyObjects(); - ParallelGCImpl.waitForIdle(); - trace.string(">").newline(); + trace.string("]").newline(); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ca20abfd4f97..0c662cf0cb6f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -450,9 +450,6 @@ private Object copyAlignedObject(Object originalObj) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.queue(copyMemory); - } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index a34fe38f9082..e1215f39c62c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -180,7 +180,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); + Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:").newline(); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { @@ -189,11 +189,13 @@ boolean scanGreyObjects() { } } if (!needScan) { + trace.string("]").newline(); return false; } for (int i = 0; i < maxSurvivorSpaces; i++) { - trace.string("[Scanning survivor-").signed(i).string("]").newline(); + trace.string("[Scanning survivor-").signed(i).newline(); getSurvivorGreyObjectsWalker(i).walkGreyObjects(); + trace.string("]").newline(); } trace.string("]").newline(); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 9dcf15e9aee0..f9634ff9bb9e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -18,7 +18,7 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; - private static final TaskQueue QUEUE = new TaskQueue("pargc-queue"); + private static final TaskQueue QUEUE = new TaskQueue("pargc"); private static final TaskQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 1304b32e9827..2038c4c03351 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -40,7 +40,7 @@ public void put(Pointer ptr) { try { mutex.lock(); while (!canPut()) { - Log.log().string("PP cannot put task\n"); + Log.log().string("TQ cannot put task\n"); cond.block(); } data[putIndex] = ptr; @@ -57,7 +57,7 @@ public void consume(Consumer consumer) { mutex.lock(); try { while (!canGet()) { - Log.log().string("PP cannot get task\n"); + Log.log().string("TQ cannot get task\n"); idleCount++; cond.block(); idleCount--; @@ -81,7 +81,7 @@ public void drain(Consumer consumer) { } public void waitUntilIdle(int expectedIdleCount) { - Log log = Log.log().string("PP waitForIdle\n"); + Log log = Log.log().string("TQ waitForIdle\n"); while (true) { try { mutex.lock(); @@ -89,7 +89,7 @@ public void waitUntilIdle(int expectedIdleCount) { cond.block(); } if (idleCount >= expectedIdleCount) { - log.string("PP waitForIdle over\n"); + log.string("TQ waitForIdle over\n"); return; } } finally { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 87e13af29b94..bc5dde38d981 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -67,7 +67,7 @@ public static boolean walkObjectInline(final Object obj, final ObjectReferenceVi final int layoutEncoding = objHub.getLayoutEncoding(); final Pointer objPointer = Word.objectToUntrackedPointer(obj); if (objHub.getName() == null) { - Log.log().string("PP warn NULL obj ").zhex(objPointer).newline(); + Log.log().string("PP warn hub.name=null for obj ").zhex(objPointer).newline(); } // Visit each Object reference in the array part of the Object. From b6c5168b6c91d6ab7b0d9baaffb9bd3fadadb0de Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 8 Jul 2022 12:32:27 +0300 Subject: [PATCH 035/125] Added `ParallelGCImpl.checkThrowable` --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 5 +++++ .../core/genscavenge/parallel/ParallelGCImpl.java | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 5748b7c636f6..8fff237c339c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -171,6 +171,7 @@ private void collect(GCCause cause, boolean forceFullGC) { if (outOfMemory) { throw OutOfMemoryUtil.heapSizeExceeded(); } + ParallelGCImpl.checkThrowable(); } } @@ -375,6 +376,10 @@ private void printGCBefore(String cause) { verboseGCLog.string(" MinimumHeapSize: ").unsigned(getPolicy().getMinimumHeapSize()).newline(); verboseGCLog.string(" AlignedChunkSize: ").unsigned(HeapParameters.getAlignedHeapChunkSize()).newline(); verboseGCLog.string(" LargeArrayThreshold: ").unsigned(HeapParameters.getLargeArrayThreshold()).string("]").newline(); + boolean compressed = ReferenceAccess.singleton().haveCompressedReferences(); + boolean hasShift = ReferenceAccess.singleton().getCompressEncoding().hasShift(); + verboseGCLog.string(" has compressed refs: ").bool(compressed).string(", has shift: ").bool(hasShift).newline(); + verboseGCLog.string("]").newline(); if (SerialGCOptions.PrintHeapShape.getValue()) { HeapImpl.getHeapImpl().logImageHeapPartitionBoundaries(verboseGCLog); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index f9634ff9bb9e..0374e9248a73 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -6,7 +6,6 @@ import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; @@ -22,7 +21,8 @@ public class ParallelGCImpl extends ParallelGC { private static final TaskQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); - private static boolean enabled; + private static volatile boolean enabled; + private static volatile Throwable throwable; @Override public void startWorkerThreads() { @@ -40,7 +40,7 @@ public void startWorkerThread(int n) { QUEUE.consume(PROMOTE_TASK); } } catch (Throwable e) { - VMError.shouldNotReachHere(e.getClass().getName()); + throwable = e; } }); t.setName("ParallelGCWorker-" + n); @@ -75,6 +75,15 @@ public static void setEnabled(boolean enabled) { public static TaskQueue.Stats getStats() { return QUEUE.stats; } + + public static void checkThrowable() { + if (throwable != null) { + Log.log().string("PGC error : ").string(throwable.getClass().getName()) + .string(" : ").string(throwable.getMessage()).newline(); + throwable.printStackTrace(); + throw new Error(throwable); + } + } } @AutomaticFeature From 2295b159fa32b2c03fa2c0c8f10074f5098ec99e Mon Sep 17 00:00:00 2001 From: peterz Date: Sun, 10 Jul 2022 14:41:29 +0300 Subject: [PATCH 036/125] Increased queue size for HyperAlloc --- .../src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 2038c4c03351..519c784767bd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -6,7 +6,7 @@ import org.graalvm.word.Pointer; public class TaskQueue { - private static final int SIZE = 1024; ///handle overflow + private static final int SIZE = 1024 * 1024; ///handle overflow final Stats stats; From 81c02c74c4c0bfb3ab930fafa470699282d45d8b Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 8 Jul 2022 12:33:53 +0300 Subject: [PATCH 037/125] Forwarding pointers done right + VerifyHeap hack HyperAlloc survives 1 minute! --- .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../core/genscavenge/ObjectHeaderImpl.java | 66 ++++++++++++------- .../oracle/svm/core/genscavenge/Space.java | 58 ++++++++++++---- .../oracle/svm/core/hub/LayoutEncoding.java | 6 ++ 4 files changed, 94 insertions(+), 37 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 5707069b1776..b9df0d6832ea 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -102,7 +102,6 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // Promote the Object if necessary, making it at least grey, and ... Object obj = p.toObject(); - assert innerOffset < LayoutEncoding.getSizeFromObject(obj).rawValue(); Object copy = GCImpl.getGCImpl().promoteObject(obj, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 6009ef596b8a..eca5f1aa732b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -327,19 +327,33 @@ static Object getForwardedObject(Pointer ptr, UnsignedWord header) { } } - private static final Unsafe UNSAFE = Unsafe.getUnsafe(); - /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") - static Object installForwardingPointer(Object original, Object copy) { - Log trace = Log.log(); - Word originalPtr = Word.objectToUntrackedPointer(original); -// assert !isPointerToForwardedObject(originalPtr); - UnsignedWord curHeader = readHeaderFromPointer(originalPtr); - if (isForwardedHeader(curHeader)) { - trace.string("PP reinstall for obj ").zhex(originalPtr).newline(); - return getForwardedObject(originalPtr, curHeader); + static void installForwardingPointer(Object original, Object copy) { + assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + UnsignedWord forwardHeader; + if (ReferenceAccess.singleton().haveCompressedReferences()) { + if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // Compression with a shift uses all bits of a reference, so store the forwarding + // pointer in the location following the hub pointer. + forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); + ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); + } else { + forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); + } + } else { + forwardHeader = Word.objectToUntrackedPointer(copy); } + assert ObjectHeaderImpl.getHeaderBitsFromHeader(forwardHeader).equal(0); + writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); + assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + } + + private static final Unsafe UNSAFE = Unsafe.getUnsafe(); + + @AlwaysInline("GC performance") + static Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { + // create forwarding header UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { @@ -354,25 +368,27 @@ static Object installForwardingPointer(Object original, Object copy) { } else { forwardHeader = Word.objectToUntrackedPointer(copy); } - assert ObjectHeaderImpl.getHeaderBitsFromHeader(forwardHeader).equal(0); -// writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); - UnsignedWord newHeader = forwardHeader.or(FORWARDED_BIT); - boolean installed; + UnsignedWord bits = getHeaderBitsFromHeader(forwardHeader); + if (!bits.equal(0)) { + Log.log().string("OHI catch hdrbits in hdr ").unsigned(forwardHeader) + .string(" from obj ").object(copy).newline(); + } + forwardHeader = forwardHeader.or(FORWARDED_BIT); + + // try installing the new header + long witness; if (getReferenceSize() == Integer.BYTES) { -// ObjectAccess.writeInt(original, getHubOffset(), (int) newHeader.rawValue()); - int expected = (int) curHeader.rawValue(); - int witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), expected, (int) newHeader.rawValue()); - installed = witness == expected; + witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), + (int) originalHeader.rawValue(), (int) forwardHeader.rawValue()); } else { -// ObjectAccess.writeWord(original, getHubOffset(), newHeader); - long expected = curHeader.rawValue(); - long witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), expected, newHeader.rawValue()); - installed = witness == expected; + witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), + originalHeader.rawValue(), forwardHeader.rawValue()); } + Pointer originalPtr = Word.objectToUntrackedPointer(original); assert isPointerToForwardedObject(originalPtr); - if (!installed) { - trace.string("PP collision for obj ").zhex(originalPtr).newline(); - return getForwardedObject(originalPtr); + if (witness != originalHeader.rawValue()) { + Log.log().string("PP collision for obj ").zhex(originalPtr).newline(); + return getForwardedObject(originalPtr, WordFactory.unsigned(witness)); } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 0c662cf0cb6f..058d3770bef0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -31,9 +31,17 @@ import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.annotate.NeverInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.thread.Continuation; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; +import org.graalvm.compiler.nodes.java.ArrayLengthNode; +import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -376,36 +384,45 @@ Object promoteAlignedObject(Object original, Space originalSpace) { return copy; } - @AlwaysInline("GC performance") +/// @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - UnsignedWord size = LayoutEncoding.getSizeFromObjectInline(original); + Log trace = Log.log(); + ObjectHeaderImpl impl = ObjectHeaderImpl.getObjectHeaderImpl(); + Pointer originalMemory = Word.objectToUntrackedPointer(original); + UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); + if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { + trace.string("PP forward obj ").zhex(originalMemory).newline(); + return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); + } + + UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); + assert size.notEqual(0) : "zero obj size"; ///debug,rm + assert size.aboveThan(0) : "negative obj size"; Pointer copyMemory = allocateMemory(size); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + Log.log().string("SPC catch prob\n"); return null; } Object copy = copyMemory.toObject(); if (copy == null) { + Log.log().string("SPC catch null\n"); return null; } - // Read original header - Pointer originalMemory = Word.objectToUntrackedPointer(original); - assert ObjectHeaderImpl.getHubOffset() == 0; - UnsignedWord header = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); // Install forwarding pointer into the original header - /// mv CAS code from OHI here - Object forward = ObjectHeaderImpl.installForwardingPointer(original, copy); + assert ObjectHeaderImpl.getHubOffset() == 0; ///PGC prerequisite + Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original header + // We have won the race, now we must copy the object bits. First install the original originalHeader int headerSize = ObjectHeaderImpl.getReferenceSize(); if (headerSize == Integer.BYTES) { - copyMemory.writeInt(0, (int) header.rawValue()); + copyMemory.writeInt(0, (int) originalHeader.rawValue()); } else { - copyMemory.writeWord(0, header); + copyMemory.writeWord(0, originalHeader); } // copy the rest of original object UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); @@ -419,11 +436,30 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { ParallelGCImpl.queue(copyMemory); return copy; } else { + /// DEBUG: make a long array to make heap verifier happy + UnmanagedMemoryUtil.fillLongs(copyMemory, size, 0L); + DynamicHub fillClass = DynamicHub.fromClass(Object[].class); + Word fillHeader = ObjectHeaderImpl.encodeAsObjectHeader(fillClass, false, false); + int headerSize = ObjectHeaderImpl.getReferenceSize(); + copyMemory.writeWord(0, fillHeader); + copyMemory.writeWord(headerSize, size.unsignedDivide(headerSize).subtract(2).shiftLeft(4 * headerSize)); +// Log.log().string("PP fill ").zhex(copyMemory) +// .string(" len ").signed(size.rawValue() / headerSize - 2).newline(); +// Log.log().string(" hdr: ").zhex(copyMemory.readWord(0)) +// .string(" ").zhex(copyMemory.readWord(headerSize)) +// .newline(); + /// END DEBUG /// Now the allocated memory is lost. Retract in TLAB? return forward; } } + @AlwaysInline("GC performance") + public static UnsignedWord getSizeFromHeader(Object obj, UnsignedWord header, ObjectHeaderImpl impl) { + int encoding = impl.dynamicHubFromObjectHeader(header).getLayoutEncoding(); + return LayoutEncoding.getSizeFromEncoding(obj, encoding); + } + @AlwaysInline("GC performance") private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 1a73be6c8d1d..b9b97cebfeb6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -269,6 +269,12 @@ public static UnsignedWord getSizeFromObject(Object obj) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInline(Object obj) { int encoding = KnownIntrinsics.readHub(obj).getLayoutEncoding(); + return getSizeFromEncoding(obj, encoding); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getSizeFromEncoding(Object obj, int encoding) { if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj)); } else { From db7c057bdda8af604cb06b7cbdfe7d1b93e53879 Mon Sep 17 00:00:00 2001 From: peterz Date: Sun, 10 Jul 2022 14:37:09 +0300 Subject: [PATCH 038/125] Protected chunk promotion with mutex --- .../oracle/svm/core/genscavenge/Space.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 058d3770bef0..29490f6f037b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -494,8 +494,21 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina /// called from GCImpl.promoteChunksWithPinnedObjects() early in the GC routine assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); + originalSpace.mutex.lock(); + mutex.lock(); + try { + originalSpace.extractAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk); + } finally { + mutex.unlock(); + originalSpace.mutex.unlock(); + } + + if (ParallelGCImpl.isEnabled()) { + if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { + throw VMError.shouldNotReachHere(); + } + } if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { @@ -511,9 +524,15 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - Log.log().string("UA promote chunk ").zhex(chunk).newline(); - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); + originalSpace.mutex.lock(); + mutex.lock(); + try { + originalSpace.extractUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk); + } finally { + mutex.unlock(); + originalSpace.mutex.unlock(); + } if (ParallelGCImpl.isEnabled()) { if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { From 57d2372397648be3f518cb8f882dda7a9c790dd8 Mon Sep 17 00:00:00 2001 From: peterz Date: Sun, 10 Jul 2022 15:33:40 +0300 Subject: [PATCH 039/125] Cleanup --- .../core/genscavenge/AlignedHeapChunk.java | 3 +- .../core/genscavenge/GreyObjectsWalker.java | 2 +- .../genscavenge/GreyToBlackObjectVisitor.java | 5 +- .../core/genscavenge/ObjectHeaderImpl.java | 29 +++--- .../oracle/svm/core/genscavenge/Space.java | 17 +--- .../core/genscavenge/UnalignedHeapChunk.java | 1 - .../svm/core/genscavenge/YoungGeneration.java | 7 -- .../genscavenge/parallel/ObjectQueue.java | 97 ------------------- .../core/genscavenge/parallel/TaskQueue.java | 12 ++- 9 files changed, 26 insertions(+), 147 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index b010ca3b9965..15a6eef96953 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; @@ -139,7 +138,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFrom(that, getObjectsStart(that), visitor); } - @NeverInline("GC performance") + @AlwaysInline("GC performance") static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index d67cb5c9333e..366c20d0c9be 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -84,7 +84,7 @@ void walkGreyObjects() { } } - @NeverInline("GC performance") + @AlwaysInline("GC performance") private void walkAlignedGreyObjects() { AlignedHeapChunk.AlignedHeader aChunk; if (alignedHeapChunk.isNull() && alignedTop.isNull()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 4ec0d030e795..28f8e495d526 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -25,7 +25,6 @@ package com.oracle.svm.core.genscavenge; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ReferenceAccess; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; @@ -59,7 +58,7 @@ public boolean visitObject(Object o) { } @Override - @NeverInline("GC performance") + @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { if (ParallelGCImpl.isEnabled()) { ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); @@ -69,7 +68,7 @@ public boolean visitObjectInline(Object o) { } } - @NeverInline("GC performance") + @AlwaysInline("GC performance") public boolean doVisitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index eca5f1aa732b..5d5673fa41f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -349,46 +349,41 @@ static void installForwardingPointer(Object original, Object copy) { assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); } - private static final Unsafe UNSAFE = Unsafe.getUnsafe(); - @AlwaysInline("GC performance") static Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { // create forwarding header UnsignedWord forwardHeader; + boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - assert false: "getCompressEncoding().hasShift()"; - // Compression with a shift uses all bits of a reference, so store the forwarding - // pointer in the location following the hub pointer. forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); ///non mt safe + hasShift = true; } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); } } else { forwardHeader = Word.objectToUntrackedPointer(copy); } - UnsignedWord bits = getHeaderBitsFromHeader(forwardHeader); - if (!bits.equal(0)) { - Log.log().string("OHI catch hdrbits in hdr ").unsigned(forwardHeader) - .string(" from obj ").object(copy).newline(); - } + assert getHeaderBitsFromHeader(forwardHeader).equal(0); forwardHeader = forwardHeader.or(FORWARDED_BIT); // try installing the new header + Pointer originalPtr = Word.objectToUntrackedPointer(original); long witness; if (getReferenceSize() == Integer.BYTES) { - witness = UNSAFE.compareAndExchangeInt(original, getHubOffset(), - (int) originalHeader.rawValue(), (int) forwardHeader.rawValue()); + witness = originalPtr.compareAndSwapInt(getHubOffset(), + (int) originalHeader.rawValue(), (int) forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); } else { - witness = UNSAFE.compareAndExchangeLong(original, getHubOffset(), - originalHeader.rawValue(), forwardHeader.rawValue()); + witness = originalPtr.compareAndSwapLong(getHubOffset(), + originalHeader.rawValue(), forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); } - Pointer originalPtr = Word.objectToUntrackedPointer(original); assert isPointerToForwardedObject(originalPtr); if (witness != originalHeader.rawValue()) { - Log.log().string("PP collision for obj ").zhex(originalPtr).newline(); return getForwardedObject(originalPtr, WordFactory.unsigned(witness)); + } else if (hasShift) { + // Compression with a shift uses all bits of a reference, so store the forwarding + // pointer in the location following the hub pointer. + ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); } return copy; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 29490f6f037b..8e74f4401cfb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -187,12 +187,10 @@ Pointer allocateMemory(UnsignedWord objectSize) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } if (result.isNonNull()) { -// Log.log().string(" alloc fast ").zhex(result).newline(); return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ result = allocateInNewChunk(objectSize); -// Log.log().string(" alloc slow ").zhex(result).newline(); return result; } finally { mutex.unlock(); @@ -384,31 +382,26 @@ Object promoteAlignedObject(Object original, Space originalSpace) { return copy; } -/// @AlwaysInline("GC performance") + @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - Log trace = Log.log(); ObjectHeaderImpl impl = ObjectHeaderImpl.getObjectHeaderImpl(); Pointer originalMemory = Word.objectToUntrackedPointer(original); UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { - trace.string("PP forward obj ").zhex(originalMemory).newline(); return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); } UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); - assert size.notEqual(0) : "zero obj size"; ///debug,rm - assert size.aboveThan(0) : "negative obj size"; + assert size.aboveThan(0); Pointer copyMemory = allocateMemory(size); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { - Log.log().string("SPC catch prob\n"); return null; } Object copy = copyMemory.toObject(); if (copy == null) { - Log.log().string("SPC catch null\n"); return null; } @@ -443,11 +436,6 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { int headerSize = ObjectHeaderImpl.getReferenceSize(); copyMemory.writeWord(0, fillHeader); copyMemory.writeWord(headerSize, size.unsignedDivide(headerSize).subtract(2).shiftLeft(4 * headerSize)); -// Log.log().string("PP fill ").zhex(copyMemory) -// .string(" len ").signed(size.rawValue() / headerSize - 2).newline(); -// Log.log().string(" hdr: ").zhex(copyMemory.readWord(0)) -// .string(" ").zhex(copyMemory.readWord(headerSize)) -// .newline(); /// END DEBUG /// Now the allocated memory is lost. Retract in TLAB? return forward; @@ -491,7 +479,6 @@ private Object copyAlignedObject(Object originalObj) { /** Promote an AlignedHeapChunk by moving it to this space. */ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { - /// called from GCImpl.promoteChunksWithPinnedObjects() early in the GC routine assert this != originalSpace && originalSpace.isFromSpace(); originalSpace.mutex.lock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index ad12df2e7cac..56843ba30cef 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -147,7 +147,6 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { @AlwaysInline("GC performance") public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { - Log.log().string("UA walk unaligned in chunk ").zhex(that).newline(); return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index e1215f39c62c..10f8d1dbb3e8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -145,13 +145,6 @@ void releaseSpaces(ChunkReleaser chunkReleaser) { } void swapSpaces() { -// try (Log trace = Log.log()) { -// trace.string(">>> eden ind=").unsigned(getEden().getInd()).newline(); -// for (int i = 0; i < maxSurvivorSpaces; i++) { -// trace.string(">>> ss").unsigned(i).string(" ind=") -// .unsigned(getSurvivorFromSpaceAt(i).getInd()).newline(); -// } -// } for (int i = 0; i < maxSurvivorSpaces; i++) { assert getSurvivorFromSpaceAt(i).isEmpty() : "Survivor fromSpace should be empty."; assert getSurvivorFromSpaceAt(i).getChunkBytes().equal(0) : "Chunk bytes must be 0"; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java deleted file mode 100644 index b256b6805437..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ObjectQueue.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; - -import java.util.function.BiConsumer; - -public class ObjectQueue {///rm - private final VMMutex mutex; - private final VMCondition cond; - private final Object[] item; - private boolean empty; - private volatile int idleCount; - - public ObjectQueue(String name) { - mutex = new VMMutex(name + "-queue"); - cond = new VMCondition(mutex); - item = new Object[2]; - empty = true; - } - - private Object[] get() { -// Log.log().string(" >>> OQ.get() called").newline(); - try { - mutex.lock(); - idleCount++; - while (empty) { -// Log.log().string(" >>> OQ.get() WAIT").newline(); - cond.block(); - } -// Log.log().string(" >>> OQ.get() returns").newline(); - return item; - } finally { - empty = true; - idleCount--; - mutex.unlock(); - cond.broadcast(); - } - } - - public void put(Object value0, Object value1) { -// Log.log().string(" >>> OQ.put() called").newline(); - try { - mutex.lock(); - while (!empty) { -// Log.log().string(" >>> OQ.put() WAIT").newline(); - cond.block(); - } - item[0] = value0; - item[1] = value1; -// Log.log().string(" >>> OQ.put() returns").newline(); - } finally { - empty = false; - mutex.unlock(); - cond.broadcast(); - } - } - - public void consume(BiConsumer consumer) { - Object val0, val1; -// Log.log().string(" >>> OQ.consume() called").newline(); - try { - mutex.lock(); - idleCount++; - while (empty) { -// Log.log().string(" >>> OQ.consume() WAIT").newline(); - cond.block(); - } -// Log.log().string(" >>> OQ.consume() unblocks").newline(); - val0 = item[0]; - val1 = item[1]; - } finally { - empty = true; - idleCount--; - mutex.unlock(); - cond.broadcast(); - } - consumer.accept(val0, val1); - } - - public void waitUntilIdle(int expectedIdleCount) { - try { - mutex.lock(); -// Log.log().string(">>> OQ.wait() empty=").bool(empty) -// .string(", idle=").signed(idleCount).newline(); - while (!empty) { - cond.block(); - } - } finally { - mutex.unlock(); - } - while (idleCount < expectedIdleCount); - } -} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 519c784767bd..67f03a1326a3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -24,6 +24,10 @@ public TaskQueue(String name) { stats = new Stats(); } + private Log log() { + return Log.noopLog(); + } + private int next(int index) { return (index + 1) % SIZE; } @@ -40,7 +44,7 @@ public void put(Pointer ptr) { try { mutex.lock(); while (!canPut()) { - Log.log().string("TQ cannot put task\n"); + log().string("TQ cannot put task\n"); cond.block(); } data[putIndex] = ptr; @@ -57,7 +61,7 @@ public void consume(Consumer consumer) { mutex.lock(); try { while (!canGet()) { - Log.log().string("TQ cannot get task\n"); + log().string("TQ cannot get task\n"); idleCount++; cond.block(); idleCount--; @@ -81,7 +85,7 @@ public void drain(Consumer consumer) { } public void waitUntilIdle(int expectedIdleCount) { - Log log = Log.log().string("TQ waitForIdle\n"); + log().string("TQ waitForIdle\n"); while (true) { try { mutex.lock(); @@ -89,7 +93,7 @@ public void waitUntilIdle(int expectedIdleCount) { cond.block(); } if (idleCount >= expectedIdleCount) { - log.string("TQ waitForIdle over\n"); + log().string("TQ waitForIdle over\n"); return; } } finally { From e376e13d4d416c41b7dac10e71cf930715b3b0fb Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 11 Jul 2022 10:58:30 +0300 Subject: [PATCH 040/125] Handle queue overflow by executing tasks synchronously --- .../genscavenge/parallel/ParallelGCImpl.java | 4 +++- .../core/genscavenge/parallel/TaskQueue.java | 17 +++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 0374e9248a73..e3f224c7ed43 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -53,7 +53,9 @@ public static GreyToBlackObjectVisitor getVisitor() { } public static void queue(Pointer ptr) { - QUEUE.put(ptr); + if (!QUEUE.put(ptr)) { + PROMOTE_TASK.accept(ptr.toObject()); + } } public static void waitForIdle() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 67f03a1326a3..71aeaf0f5871 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -6,7 +6,7 @@ import org.graalvm.word.Pointer; public class TaskQueue { - private static final int SIZE = 1024 * 1024; ///handle overflow + private static final int SIZE = 128 * 1024; final Stats stats; @@ -40,20 +40,21 @@ private boolean canPut() { return next(putIndex) != getIndex; } - public void put(Pointer ptr) { + public boolean put(Pointer ptr) { try { mutex.lock(); - while (!canPut()) { + if (!canPut()) { log().string("TQ cannot put task\n"); - cond.block(); + return false; } data[putIndex] = ptr; - } finally { putIndex = next(putIndex); stats.noteTask(putIndex, getIndex); + } finally { mutex.unlock(); - cond.broadcast(); } + cond.broadcast(); + return true; } public void consume(Consumer consumer) { @@ -67,11 +68,11 @@ public void consume(Consumer consumer) { idleCount--; } ptr = data[getIndex]; - } finally { getIndex = next(getIndex); + } finally { mutex.unlock(); - cond.broadcast(); } + cond.broadcast(); consumer.accept(ptr.toObject()); } From 1c891d1023565dd7d8ef7560b0bbdbf936ccb689 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 11 Jul 2022 11:53:46 +0300 Subject: [PATCH 041/125] Cleanup --- substratevm/mx.substratevm/suite.py | 1 - .../com/oracle/svm/core/genscavenge/GCImpl.java | 8 ++++---- .../svm/core/genscavenge/GreyObjectsWalker.java | 2 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 1 - .../svm/core/genscavenge/ObjectHeaderImpl.java | 2 -- .../svm/core/genscavenge/OldGeneration.java | 4 ---- .../genscavenge/ReferenceObjectProcessing.java | 4 +--- .../com/oracle/svm/core/genscavenge/Timers.java | 2 -- .../core/genscavenge/UnalignedHeapChunk.java | 1 - .../svm/core/genscavenge/YoungGeneration.java | 8 ++------ .../genscavenge/parallel/ParallelGCImpl.java | 2 -- .../core/genscavenge/parallel/TaskQueue.java | 6 +++--- .../oracle/svm/core/SubstrateDiagnostics.java | 7 ------- .../svm/core/hub/InteriorObjRefWalker.java | 6 +----- .../oracle/svm/core/stack/JavaStackWalker.java | 17 ----------------- .../com/oracle/svm/core/thread/VMThreads.java | 2 +- 16 files changed, 13 insertions(+), 60 deletions(-) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 342adf6795aa..f0eb593c9901 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -323,7 +323,6 @@ "requiresConcealed" : { "java.base": [ "sun.nio.ch", - "jdk.internal.misc", ], "java.management": [ "sun.management", diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 8fff237c339c..f6fd46b9745b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -866,7 +866,7 @@ private void blackenStackRoots() { JavaStackWalk walk = StackValue.get(JavaStackWalk.class); JavaStackWalker.initWalk(walk, sp, ip); - walkStack(walk, WordFactory.nullPointer()); + walkStack(walk); if (SubstrateOptions.MultiThreaded.getValue()) { /* @@ -884,7 +884,7 @@ private void blackenStackRoots() { continue; } if (JavaStackWalker.initWalk(walk, vmThread)) { - walkStack(walk, vmThread); + walkStack(walk); } } } @@ -900,7 +900,7 @@ private void blackenStackRoots() { * calls to a stack frame visitor. */ @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) - private void walkStack(JavaStackWalk walk, IsolateThread thread) { + private void walkStack(JavaStackWalk walk) { assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether"; while (true) { @@ -913,7 +913,7 @@ private void walkStack(JavaStackWalk walk, IsolateThread thread) { DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp); if (deoptFrame == null) { if (codeInfo.isNull()) { - throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame, thread, walk.getAnchor(), walk.getIPCodeInfo(), codeInfo); + throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame); } CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 366c20d0c9be..3788ec2572df 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -112,7 +112,7 @@ private void walkAlignedGreyObjects() { } } - @NeverInline("GC performance") + @AlwaysInline("GC performance") private void walkUnalignedGreyObjects() { /* Visit the Objects in the UnalignedChunk after the snapshot UnalignedChunk. */ UnalignedHeapChunk.UnalignedHeader uChunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 342b077c949e..d9d2b1ab8b31 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -106,7 +106,6 @@ public final class HeapImpl extends Heap { private final RuntimeCodeInfoGCSupportImpl runtimeCodeInfoGcSupport; private final ImageHeapInfo imageHeapInfo = new ImageHeapInfo(); private final HeapAccounting accounting = new HeapAccounting(); - public final Timer timer = new Timer("promoteUnalignedHeapChunk()"); /** Head of the linked list of currently pending (ready to be enqueued) {@link Reference}s. */ private Reference refPendingList; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 5d5673fa41f3..345a32a91f6b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.log.Log; -import jdk.internal.misc.Unsafe; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.CompressEncoding; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index f6f2c9d9f24a..8b3e042e34f1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -27,7 +27,6 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; @@ -108,13 +107,10 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("[OldGeneration.scanGreyObjects:").newline(); if (!toGreyObjectsWalker.haveGreyObjects()) { - trace.string("]").newline(); return false; } toGreyObjectsWalker.walkGreyObjects(); - trace.string("]").newline(); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 7923285d6e2a..3c0a614336fb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -31,7 +31,6 @@ import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; -import com.oracle.svm.core.annotate.NeverInline; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -78,7 +77,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { softReferencesAreWeak = enabled; } - @NeverInline("GC performance") + @AlwaysInline("GC performance") public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -87,7 +86,6 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } - @NeverInline("GC performance") private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index e00701d54eaf..bd5b1cd255fd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -150,7 +150,6 @@ void resetAllExceptMutator() { referenceObjects.reset(); releaseSpaces.reset(); verifyAfter.reset(); - HeapImpl.getHeapImpl().timer.reset(); /* The mutator timer is *not* reset here. */ trace.string("]").newline(); } @@ -172,7 +171,6 @@ void logAfterCollection(Log log) { logOneTimer(log, " ", blackenImageHeapRoots); logOneTimer(log, " ", blackenDirtyCardRoots); logOneTimer(log, " ", scanGreyObjects); - logOneTimer(log, " ", HeapImpl.getHeapImpl().timer); logOneTimer(log, " ", cleanCodeCache); logOneTimer(log, " ", referenceObjects); logOneTimer(log, " ", releaseSpaces); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 56843ba30cef..299309cb970b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 10f8d1dbb3e8..209fbad2731d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -136,10 +136,8 @@ private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { } void releaseSpaces(ChunkReleaser chunkReleaser) { -// getEden().setInd(333); getEden().releaseChunks(chunkReleaser); for (int i = 0; i < maxSurvivorSpaces; i++) { -// getSurvivorFromSpaceAt(i).setInd(100+i); getSurvivorFromSpaceAt(i).releaseChunks(chunkReleaser); } } @@ -173,7 +171,7 @@ void prepareForPromotion() { } boolean scanGreyObjects() { - Log trace = Log.log().string("[YoungGeneration.scanGreyObjects:").newline(); + Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { @@ -182,13 +180,11 @@ boolean scanGreyObjects() { } } if (!needScan) { - trace.string("]").newline(); return false; } for (int i = 0; i < maxSurvivorSpaces; i++) { - trace.string("[Scanning survivor-").signed(i).newline(); + trace.string("[Scanning survivor-").signed(i).string("]").newline(); getSurvivorGreyObjectsWalker(i).walkGreyObjects(); - trace.string("]").newline(); } trace.string("]").newline(); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e3f224c7ed43..cca0a7acae96 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -30,8 +30,6 @@ public void startWorkerThreads() { } public void startWorkerThread(int n) { - final Log trace = Log.log(); - trace.string("PP start worker-").unsigned(n).newline(); Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 71aeaf0f5871..89a3ab4a3170 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -12,7 +12,7 @@ public class TaskQueue { private final VMMutex mutex; private final VMCondition cond; - private final Pointer[] data; /// move out of heap? + private final Pointer[] data; private int idleCount; private int getIndex; private int putIndex; @@ -103,8 +103,8 @@ public void waitUntilIdle(int expectedIdleCount) { } } - public interface Consumer { - void accept(Object object); ///j.u.f.Consumer + interface Consumer { + void accept(Object object); } // Non MT safe, needs locking diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index 1c9dd8d108d4..d0b83fa3ace9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -712,13 +712,6 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } log.string(", stack(").zhex(VMThreads.StackEnd.get(thread)).string(",").zhex(VMThreads.StackBase.get(thread)).string(")"); - JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(thread); - if (anchor.isNonNull()) { - log.newline().spaces(21) - .string("anchor=").zhex(anchor) - .string(", sp=").zhex(anchor.getLastJavaSP()) - .string(", ip=").zhex(anchor.getLastJavaIP()); - } log.newline(); printed++; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index bc5dde38d981..0d4ae77e5d2f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.hub; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.word.Word; import org.graalvm.word.Pointer; @@ -61,14 +60,11 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor return walkObjectInline(obj, visitor); } - @NeverInline("Performance critical version") + @AlwaysInline("Performance critical version") public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final int layoutEncoding = objHub.getLayoutEncoding(); final Pointer objPointer = Word.objectToUntrackedPointer(obj); - if (objHub.getName() == null) { - Log.log().string("PP warn hub.name=null for obj ").zhex(objPointer).newline(); - } // Visit each Object reference in the array part of the Object. if (LayoutEncoding.isArrayLikeWithObjectElements(layoutEncoding)) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java index c9bc20c9e0b6..96bfa38324f8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java @@ -245,23 +245,6 @@ public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePoi } - @Uninterruptible(reason = "Not really uninterruptible, but we are about to fatally fail.", calleeMustBe = false) - public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePointer ip, DeoptimizedFrame deoptFrame, - IsolateThread thread, JavaFrameAnchor anchor, UntetheredCodeInfo uCodeInfo, CodeInfo codeInfo) - { - Log log = Log.log().string("Stack walk must walk only frames of known code:"); - log.string(" anchor=").hex(anchor).string(" sp=").hex(sp).string(" ip=").hex(ip).newline() - .string(" thread=").hex(thread).string(" codeInfo=").hex(codeInfo) - .string(" untethered=").hex(uCodeInfo); - - if (DeoptimizationSupport.enabled()) { - log.string(" deoptFrame=").object(deoptFrame); - } - log.newline(); - throw VMError.shouldNotReachHere("Stack walk must walk only frames of known code"); - - } - public static boolean walkCurrentThread(Pointer startSP, StackFrameVisitor visitor) { return walkCurrentThread(startSP, visitor, null); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index f10e1ae9aa94..49fcd616faaa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -826,7 +826,7 @@ public static int getNewThreadStatus(CFunctionOptions.Transition transition) { } } - public static class ParallelGCSupport { + public static class ParallelGCSupport {///unused private static final FastThreadLocalInt isParallelGCThreadTL = FastThreadLocalFactory.createInt("ParallelGCSupport.isParallelGCThreadTL"); private static final int REGULAR_THREAD = 0; private static final int PARALLEL_GC_THREAD = 1; From 7b880b63404025bde7acf3583ef2992d972a4b3a Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 19 Jul 2022 18:40:01 +0300 Subject: [PATCH 042/125] Added thread local Stats --- .../oracle/svm/core/genscavenge/GCImpl.java | 5 +- .../genscavenge/parallel/ParallelGCImpl.java | 13 +- .../svm/core/genscavenge/parallel/Stats.java | 204 ++++++++++++++++++ .../core/genscavenge/parallel/TaskQueue.java | 32 --- 4 files changed, 215 insertions(+), 39 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index f6fd46b9745b..64334259017b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -32,6 +32,7 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.Stats; import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.VMOperationInfos; @@ -442,9 +443,7 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } - TaskQueue.Stats stats = ParallelGCImpl.getStats(); - verboseGCLog.newline().string(" tasks: ").unsigned(stats.getCount()) - .string(", max size: ").unsigned(stats.getMaxSize()).newline(); + ParallelGCImpl.getStats().print(Log.log()); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index cca0a7acae96..e6aa63d03c51 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -33,6 +33,7 @@ public void startWorkerThread(int n) { Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); + getStats().install(); try { while (!stopped) { QUEUE.consume(PROMOTE_TASK); @@ -72,10 +73,6 @@ public static void setEnabled(boolean enabled) { ParallelGCImpl.enabled = enabled; } - public static TaskQueue.Stats getStats() { - return QUEUE.stats; - } - public static void checkThrowable() { if (throwable != null) { Log.log().string("PGC error : ").string(throwable.getClass().getName()) @@ -84,6 +81,14 @@ public static void checkThrowable() { throw new Error(throwable); } } + + public static Stats getStats() { + return Stats.stats(); + } + + static Log log() { + return Log.noopLog(); + } } @AutomaticFeature diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java new file mode 100644 index 000000000000..a08cbd0c1ae7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java @@ -0,0 +1,204 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; +import org.graalvm.nativeimage.IsolateThread; + +public abstract class Stats { + private static final Stats realStats = new RealStats(); + private static final Stats noopStats = new NoopStats(); + + static Stats stats() { + return realStats; + } + + static Stats noopStats() { + return noopStats; + } + + public void reset() {} + public void print(Log log) {} + void install() {} + public void noteAllocTime(long t) {} + public void noteForwardInstallTime(long t) {} + public void noteObjectVisitTime(long t) {} + public void noteQueueCopyTime(long t) {} + public void noteQueueScanTime(long t) {} + public void noteLostObject() {} + void notePut(int putIndex, int getIndex, int capacity) {} + void noteGet() {} +} + +class NoopStats extends Stats { +} + +class RealStats extends Stats { + private static final FastThreadLocalObject statsTL = + FastThreadLocalFactory.createObject(StatsImpl.class, "ParGC.Stats"); + + private final StatsImpl defaultStats = new StatsImpl(); + private final StatsImpl totalStats = new StatsImpl(); + + private StatsImpl impl() { + StatsImpl stats = statsTL.get(); + if (stats == null) { + stats = defaultStats; + statsTL.set(stats); + } + return stats; + } + + @Override + void install() { + statsTL.set(new StatsImpl()); + } + + @Override + public void reset() { + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + StatsImpl stats = statsTL.get(vmThread); + if (stats != null) { + stats.reset(); + } + } + } + + @Override + public void print(Log log) { + totalStats.reset(); + log.string("PGC stats:").newline(); + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + StatsImpl stats = statsTL.get(vmThread); + if (stats != null) { + Thread jt = PlatformThreads.fromVMThread(vmThread); + log.string(" ").string(jt.getName()).string(":").newline(); + totalStats.objectVisitTime += stats.objectVisitTime; + totalStats.allocTime += stats.allocTime; + totalStats.forwardInstallTime += stats.forwardInstallTime; + totalStats.queueScanTime += stats.queueScanTime; + totalStats.queueCopyTime += stats.queueCopyTime; + totalStats.lostObjectCount += stats.lostObjectCount; + totalStats.putCount += stats.putCount; + totalStats.getCount += stats.getCount; + totalStats.maxQueueSize = Math.max(totalStats.maxQueueSize, stats.maxQueueSize); + stats.print(log); + } + } + log.string(" total:").newline(); + totalStats.print(log); + } + + @Override + public void noteAllocTime(long t) { + impl().noteAllocTime(t); + } + + @Override + public void noteForwardInstallTime(long t) { + impl().noteForwardInstallTime(t); + } + + @Override + public void noteObjectVisitTime(long t) { + impl().noteObjectVisitTime(t); + } + + @Override + public void noteQueueCopyTime(long t) { + impl().noteQueueCopyTime(t); + } + + @Override + public void noteQueueScanTime(long t) { + impl().noteQueueScanTime(t); + } + + @Override + public void noteLostObject() { + impl().noteLostObject(); + } + + @Override + void notePut(int putIndex, int getIndex, int capacity) { + impl().notePut(putIndex, getIndex, capacity); + } + + @Override + public void noteGet() { + impl().noteGet(); + } +} + +class StatsImpl extends Stats { + int allocTime, forwardInstallTime, objectVisitTime, queueCopyTime, queueScanTime; + int putCount, getCount, lostObjectCount, maxQueueSize; + + @Override + public void reset() { + allocTime = forwardInstallTime = objectVisitTime = queueCopyTime = queueScanTime = 0; + putCount = getCount = lostObjectCount = maxQueueSize = 0; + } + + @Override + public void print(Log log) { + log.string(" visit ").unsigned(objectVisitTime) + .string(" alloc ").unsigned(allocTime) + .string(" fwptr ").unsigned(forwardInstallTime) + .string(" qscan ").unsigned(queueScanTime) + .string(" qcopy ").unsigned(queueCopyTime) + .string(" lost ").unsigned(lostObjectCount).newline(); + log.string(" put ").unsigned(putCount) + .string(" get ").unsigned(getCount) + .string(" max ").unsigned(maxQueueSize).newline(); + } + + @Override + public void noteAllocTime(long t) { + allocTime += t; + } + + @Override + public void noteForwardInstallTime(long t) { + forwardInstallTime += t; + } + + @Override + public void noteObjectVisitTime(long t) { + objectVisitTime += t; + } + + @Override + public void noteQueueCopyTime(long t) { + queueCopyTime += t; + } + + @Override + public void noteQueueScanTime(long t) { + queueScanTime += t; + } + + @Override + public void noteLostObject() { + lostObjectCount++; + } + + @Override + void notePut(int putIndex, int getIndex, int capacity) { + int size = putIndex - getIndex; + if (size < 0) { + size += capacity; + } + if (size > maxQueueSize) { + maxQueueSize = size; + } + putCount++; + } + + @Override + void noteGet() { + getCount++; + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java index 89a3ab4a3170..aaafd18fdea6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java @@ -8,8 +8,6 @@ public class TaskQueue { private static final int SIZE = 128 * 1024; - final Stats stats; - private final VMMutex mutex; private final VMCondition cond; private final Pointer[] data; @@ -21,7 +19,6 @@ public TaskQueue(String name) { mutex = new VMMutex(name + "-queue"); cond = new VMCondition(mutex); data = new Pointer[SIZE]; - stats = new Stats(); } private Log log() { @@ -49,7 +46,6 @@ public boolean put(Pointer ptr) { } data[putIndex] = ptr; putIndex = next(putIndex); - stats.noteTask(putIndex, getIndex); } finally { mutex.unlock(); } @@ -106,32 +102,4 @@ public void waitUntilIdle(int expectedIdleCount) { interface Consumer { void accept(Object object); } - - // Non MT safe, needs locking - public static class Stats { - private int count, maxSize; - - void noteTask(int putIndex, int getIndex) { - int size = putIndex - getIndex; - if (size < 0) { - size += SIZE; - } - if (size > maxSize) { - maxSize = size; - } - count++; - } - - public int getCount() { - return count; - } - - public int getMaxSize() { - return maxSize; - } - - public void reset() { - count = maxSize = 0; - } - } } \ No newline at end of file From cdf48155055b1327e4e548803b8e4aad41631218 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 18 Jul 2022 11:30:47 +0300 Subject: [PATCH 043/125] (Incomplete) CAS based queue with minimal synchronization. --- .../core/genscavenge/parallel/CasQueue.java | 85 +++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 38 +++++++-- 2 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java new file mode 100644 index 000000000000..107fedc3afbd --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java @@ -0,0 +1,85 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; + +import java.util.concurrent.atomic.AtomicInteger; + +/// Lame, test only. Doesn't handle overflows +public class CasQueue { + private static final int SIZE = 1024 * 1024; + + final Stats stats = new Stats(); + + private final Pointer[] data = new Pointer[SIZE]; + private final AtomicInteger getIndex = new AtomicInteger(0); + private final AtomicInteger putIndex = new AtomicInteger(0); + + private Log log() { + return ParallelGCImpl.log(); + } + + public boolean put(Pointer ptr) { + int cur = putIndex.get(); + int witness; + while ((witness = putIndex.compareAndExchange(cur, cur + 1)) != cur) { + cur = witness; + } + data[cur] = ptr; + return true; + } + + public boolean consume(Consumer consumer) { + int cur = getIndex.get(); + int witness; + while (cur < putIndex.get() && (witness = getIndex.compareAndExchange(cur, cur + 1)) != cur) { + cur = witness; + } + if (cur >= putIndex.get()) { + return false; + } + Object obj = data[cur].toObject(); + consumer.accept(obj); + return true; + } + + public void drain(Consumer consumer) { + while (consume(consumer)); + getIndex.set(0); + putIndex.set(0); + } + + interface Consumer { + void accept(Object object); + } + + // Non MT safe, needs locking + public static class Stats { + private int count, maxSize; + + void noteTask(int putIndex, int getIndex) { + int size = putIndex - getIndex; + if (size < 0) { + size += SIZE; + } + if (size > maxSize) { + maxSize = size; + } + count++; + } + + public int getCount() { + return count; + } + + public int getMaxSize() { + return maxSize; + } + + public void reset() { + count = maxSize = 0; + } + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e6aa63d03c51..6f7812a32243 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -4,12 +4,15 @@ import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -17,8 +20,12 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; - private static final TaskQueue QUEUE = new TaskQueue("pargc"); - private static final TaskQueue.Consumer PROMOTE_TASK = + private static final VMMutex mutex = new VMMutex("pargc"); + private static final VMCondition cond = new VMCondition(mutex); + private static AtomicInteger busy = new AtomicInteger(0); + + private static final CasQueue QUEUE = new CasQueue(); + private static final CasQueue.Consumer PROMOTE_TASK = obj -> getVisitor().doVisitObject(obj); private static volatile boolean enabled; @@ -33,10 +40,22 @@ public void startWorkerThread(int n) { Thread t = new Thread(() -> { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); + log().string("WW start ").unsigned(n).newline(); getStats().install(); try { while (!stopped) { - QUEUE.consume(PROMOTE_TASK); + mutex.lock(); + while (busy.get() < WORKERS_COUNT) { + log().string("WW block ").unsigned(n).newline(); + cond.block(); + } + mutex.unlock(); + + QUEUE.drain(PROMOTE_TASK); + if (busy.decrementAndGet() <= 0) { + cond.broadcast(); + } + log().string("WW idle ").unsigned(n).newline(); } } catch (Throwable e) { throwable = e; @@ -58,11 +77,16 @@ public static void queue(Pointer ptr) { } public static void waitForIdle() { - if (WORKERS_COUNT > 0) { - QUEUE.waitUntilIdle(WORKERS_COUNT); - } else { - QUEUE.drain(PROMOTE_TASK); // execute synchronously + log().string("PP start workers\n"); + busy.set(WORKERS_COUNT); + cond.broadcast(); // let worker threads run + + mutex.lock(); + while (busy.get() > 0) { + log().string("PP wait busy=").unsigned(busy.get()).newline(); + cond.block(); // wait for them to become idle } + mutex.unlock(); } public static boolean isEnabled() { From 71c515a851ed9f0b4c221045450bf1db5b40930a Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 19 Jul 2022 12:20:05 +0300 Subject: [PATCH 044/125] Thread local memory allocation --- .../oracle/svm/core/genscavenge/GCImpl.java | 1 + .../oracle/svm/core/genscavenge/Space.java | 21 ++++++++++--------- .../genscavenge/parallel/ParallelGCImpl.java | 9 ++++++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 64334259017b..6761a8f28b84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1157,6 +1157,7 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } + ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8e74f4401cfb..df7cd6ce9a16 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -178,18 +178,18 @@ public Log report(Log log, boolean traceHeapChunks) { */ @NeverInline("GC performance") Pointer allocateMemory(UnsignedWord objectSize) { + Pointer result = WordFactory.nullPointer(); + /* Fast-path: try allocating in the last chunk. */ + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + if (oldChunk.isNonNull()) { + result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); + } + if (result.isNonNull()) { + return result; + } + /* Slow-path: try allocating a new chunk for the requested memory. */ mutex.lock(); try { - Pointer result = WordFactory.nullPointer(); - /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); - if (oldChunk.isNonNull()) { - result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); - } - if (result.isNonNull()) { - return result; - } - /* Slow-path: try allocating a new chunk for the requested memory. */ result = allocateInNewChunk(objectSize); return result; } finally { @@ -199,6 +199,7 @@ Pointer allocateMemory(UnsignedWord objectSize) { private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + ParallelGCImpl.TLAB.set(newChunk); if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 6f7812a32243..8a7afa9a68af 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; @@ -8,9 +9,12 @@ import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; @@ -20,6 +24,10 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons public static final int WORKERS_COUNT = 4; + /// tlab of the current to-space + public static final FastThreadLocalWord TLAB = + FastThreadLocalFactory.createWord("ParallelGCImpl.TLAB"); + private static final VMMutex mutex = new VMMutex("pargc"); private static final VMCondition cond = new VMCondition(mutex); private static AtomicInteger busy = new AtomicInteger(0); @@ -52,6 +60,7 @@ public void startWorkerThread(int n) { mutex.unlock(); QUEUE.drain(PROMOTE_TASK); + TLAB.set(WordFactory.nullPointer()); if (busy.decrementAndGet() <= 0) { cond.broadcast(); } From 1d2f791087cb192514708b0e59e657cb8d8fa503 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 8 Aug 2022 18:08:07 +0300 Subject: [PATCH 045/125] Retract speculatively allocated memory --- .../core/genscavenge/AlignedHeapChunk.java | 8 +++++ .../oracle/svm/core/genscavenge/Space.java | 29 +++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 15a6eef96953..8d81707a7293 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -114,6 +114,14 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { return result; } + /** Retract last allocation made. Used by parallel collector. */ + static Pointer freeMemory(AlignedHeader that, UnsignedWord size) { + Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); + assert newTop.aboveOrEqual(HeapChunk.asPointer(that)); + HeapChunk.setTopPointer(that, newTop); + return newTop; + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static UnsignedWord getCommittedObjectMemory(AlignedHeader that) { return HeapChunk.getEndOffset(that).subtract(getObjectsStartOffset()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index df7cd6ce9a16..afa6519f0bb9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -176,7 +176,7 @@ public Log report(Log log, boolean traceHeapChunks) { /** * Allocate memory from an AlignedHeapChunk in this Space. */ - @NeverInline("GC performance") + @AlwaysInline("GC performance") Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ @@ -197,6 +197,16 @@ Pointer allocateMemory(UnsignedWord objectSize) { } } + /** + * Retract last allocation made. Used by parallel collector. + */ + @AlwaysInline("GC performance") + Pointer freeMemory(UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + assert oldChunk.isNonNull(); + return AlignedHeapChunk.freeMemory(oldChunk, objectSize); + } + private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); ParallelGCImpl.TLAB.set(newChunk); @@ -395,6 +405,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); } + // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. + // If another thread copies the object first, we retract the allocation later. UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); assert size.aboveThan(0); Pointer copyMemory = allocateMemory(size); @@ -411,14 +423,14 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original originalHeader + // We have won the race, now we must copy the object bits. First install the original header int headerSize = ObjectHeaderImpl.getReferenceSize(); if (headerSize == Integer.BYTES) { copyMemory.writeInt(0, (int) originalHeader.rawValue()); } else { copyMemory.writeWord(0, originalHeader); } - // copy the rest of original object + // Copy the rest of original object UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); if (isOldSpace()) { @@ -430,15 +442,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { ParallelGCImpl.queue(copyMemory); return copy; } else { - /// DEBUG: make a long array to make heap verifier happy - UnmanagedMemoryUtil.fillLongs(copyMemory, size, 0L); - DynamicHub fillClass = DynamicHub.fromClass(Object[].class); - Word fillHeader = ObjectHeaderImpl.encodeAsObjectHeader(fillClass, false, false); - int headerSize = ObjectHeaderImpl.getReferenceSize(); - copyMemory.writeWord(0, fillHeader); - copyMemory.writeWord(headerSize, size.unsignedDivide(headerSize).subtract(2).shiftLeft(4 * headerSize)); - /// END DEBUG - /// Now the allocated memory is lost. Retract in TLAB? + // Retract speculatively allocated memory + freeMemory(size); return forward; } } From 17c695bbe59466ed3f7bbee745104586dda07879 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 10 Aug 2022 12:33:49 +0300 Subject: [PATCH 046/125] Thread local task stacks --- .../oracle/svm/core/genscavenge/GCImpl.java | 18 ++++----- .../genscavenge/GreyToBlackObjectVisitor.java | 4 +- .../oracle/svm/core/genscavenge/Space.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 32 ++++++++++++---- .../parallel/ThreadLocalTaskStack.java | 37 +++++++++++++++++++ 5 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 6761a8f28b84..1d41c97d9063 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1055,17 +1055,15 @@ private static void prepareForPromotion(boolean isIncremental) { private void scanGreyObjects(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); - ParallelGCImpl.setEnabled(true); - try { - if (isIncremental) { - heap.getYoungGeneration().scanGreyObjects(); - } - heap.getOldGeneration().scanGreyObjects(); - } finally { - ParallelGCImpl.waitForIdle(); - ParallelGCImpl.setEnabled(false); - scanGreyObjectsTimer.close(); + if (isIncremental) { + heap.getYoungGeneration().scanGreyObjects(); } + heap.getOldGeneration().scanGreyObjects(); + + ParallelGCImpl.setEnabled(true); + ParallelGCImpl.waitForIdle(); + ParallelGCImpl.setEnabled(false); + scanGreyObjectsTimer.close(); } private static void scanGreyObjectsLoop() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 28f8e495d526..b32f1ee828c0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -60,8 +60,8 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { - if (ParallelGCImpl.isEnabled()) { - ParallelGCImpl.queue(Word.objectToUntrackedPointer(o)); + if (!ParallelGCImpl.isEnabled()) { + ParallelGCImpl.push(Word.objectToUntrackedPointer(o)); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index afa6519f0bb9..8d4b7f8c9674 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -439,7 +439,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.queue(copyMemory); + ParallelGCImpl.localPush(copyMemory); return copy; } else { // Retract speculatively allocated memory diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 8a7afa9a68af..859ca460775a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -32,9 +32,10 @@ public class ParallelGCImpl extends ParallelGC { private static final VMCondition cond = new VMCondition(mutex); private static AtomicInteger busy = new AtomicInteger(0); - private static final CasQueue QUEUE = new CasQueue(); - private static final CasQueue.Consumer PROMOTE_TASK = - obj -> getVisitor().doVisitObject(obj); + private static final ThreadLocalTaskStack[] STACKS = + IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); + private static final ThreadLocal localStack = new ThreadLocal<>(); + private static int currentStack; private static volatile boolean enabled; private static volatile Throwable throwable; @@ -49,6 +50,9 @@ public void startWorkerThread(int n) { // VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); + + final ThreadLocalTaskStack stack = STACKS[n]; + localStack.set(STACKS[n]); getStats().install(); try { while (!stopped) { @@ -59,7 +63,11 @@ public void startWorkerThread(int n) { } mutex.unlock(); - QUEUE.drain(PROMOTE_TASK); + log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); + Object obj; + while ((obj = stack.pop()) != null) { + getVisitor().doVisitObject(obj); + } TLAB.set(WordFactory.nullPointer()); if (busy.decrementAndGet() <= 0) { cond.broadcast(); @@ -79,9 +87,19 @@ public static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - public static void queue(Pointer ptr) { - if (!QUEUE.put(ptr)) { - PROMOTE_TASK.accept(ptr.toObject()); + /// name, explain + public static void push(Pointer ptr) { + push(ptr, STACKS[currentStack]); + currentStack = (currentStack + 1) % WORKERS_COUNT; + } + + public static void localPush(Pointer ptr) { + push(ptr, localStack.get()); + } + + private static void push(Pointer ptr, ThreadLocalTaskStack stack) { + if (!stack.push(ptr)) { + getVisitor().doVisitObject(ptr.toObject()); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java new file mode 100644 index 000000000000..09645ebcc404 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java @@ -0,0 +1,37 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; + +/// thread local, non MT safe +public class ThreadLocalTaskStack { + private static final int SIZE = 128 * 1024; + + private final Pointer[] data = new Pointer[SIZE]; + private int top = 0; + + private Log log() { + return ParallelGCImpl.log(); + } + + boolean push(Pointer ptr) { + if (top >= SIZE) { + log().string("TT cannot put task\n"); + return false; + } + data[top++] = ptr; + return true; + } + + Object pop() { + if (top > 0) { + return data[--top].toObjectNonNull(); + } else { + return null; + } + } + + int size() { + return top; + } +} \ No newline at end of file From d9a47b7075b9a6471567a24e2838ee3034dc2cfc Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 12 Aug 2022 10:25:20 +0300 Subject: [PATCH 047/125] Post merge cleanup --- .../oracle/svm/core/genscavenge/GCImpl.java | 14 ++------ .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../genscavenge/GreyToBlackObjectVisitor.java | 2 -- .../svm/core/genscavenge/HeapChunk.java | 1 - .../oracle/svm/core/genscavenge/Space.java | 12 +------ .../genscavenge/parallel/ParallelGCImpl.java | 1 - .../com/oracle/svm/core/heap/ParallelGC.java | 33 ------------------- .../com/oracle/svm/core/thread/VMThreads.java | 14 -------- 8 files changed, 4 insertions(+), 74 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 1d41c97d9063..a8cb29389656 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -26,20 +26,12 @@ import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.genscavenge.parallel.Stats; -import com.oracle.svm.core.genscavenge.parallel.TaskQueue; import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.heap.VMOperationInfos; -import com.oracle.svm.core.hub.LayoutEncoding; -import com.oracle.svm.core.jfr.JfrTicks; import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -377,10 +369,9 @@ private void printGCBefore(String cause) { verboseGCLog.string(" MinimumHeapSize: ").unsigned(getPolicy().getMinimumHeapSize()).newline(); verboseGCLog.string(" AlignedChunkSize: ").unsigned(HeapParameters.getAlignedHeapChunkSize()).newline(); verboseGCLog.string(" LargeArrayThreshold: ").unsigned(HeapParameters.getLargeArrayThreshold()).string("]").newline(); - boolean compressed = ReferenceAccess.singleton().haveCompressedReferences(); + boolean compressed = ReferenceAccess.singleton().haveCompressedReferences();///rm boolean hasShift = ReferenceAccess.singleton().getCompressEncoding().hasShift(); verboseGCLog.string(" has compressed refs: ").bool(compressed).string(", has shift: ").bool(hasShift).newline(); - verboseGCLog.string("]").newline(); if (SerialGCOptions.PrintHeapShape.getValue()) { HeapImpl.getHeapImpl().logImageHeapPartitionBoundaries(verboseGCLog); } @@ -1052,6 +1043,7 @@ private static void prepareForPromotion(boolean isIncremental) { } } + /// queue objects as they are copied? private void scanGreyObjects(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); @@ -1155,7 +1147,7 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); + ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); ///rm? } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index b9df0d6832ea..34637ae2b532 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index b32f1ee828c0..ccce076078a1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -25,8 +25,6 @@ package com.oracle.svm.core.genscavenge; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import org.graalvm.compiler.nodes.java.ArrayLengthNode; -import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index a81495658463..f56f97261d9c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,7 +26,6 @@ import java.util.function.IntUnaryOperator; -import org.graalvm.compiler.options.Option; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8d4b7f8c9674..3d5b61d00ce9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -27,21 +27,10 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AlwaysInline; -import com.oracle.svm.core.annotate.NeverInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.heap.StoredContinuation; -import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.snippets.KnownIntrinsics; -import com.oracle.svm.core.thread.Continuation; -import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; -import org.graalvm.compiler.nodes.java.ArrayLengthNode; -import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -234,6 +223,7 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ ///assert owns mutex || pargc thread ? + ///use thread mutex to protect appendNewChunk // if (SubstrateOptions.MultiThreaded.getValue()) { // VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); // } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 859ca460775a..c46d32d6ecff 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -47,7 +47,6 @@ public void startWorkerThreads() { public void startWorkerThread(int n) { Thread t = new Thread(() -> { -// VMThreads.ParallelGCSupport.setParallelGCThread(); VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 0275910defab..c62182edba9f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,40 +1,7 @@ package com.oracle.svm.core.heap; -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; - public abstract class ParallelGC { - protected final VMMutex mutex = new VMMutex("pargc"); - protected final VMCondition cond = new VMCondition(mutex); - - protected volatile boolean workerThreadActive; protected volatile boolean stopped; public abstract void startWorkerThreads(); - - public void waitForNotification(boolean onWorkerThread) { - Log trace = Log.log(); - mutex.lock(); - try { - while (workerThreadActive != onWorkerThread) { - trace.string(" blocking on ").string(Thread.currentThread().getName()).newline(); - cond.block(); - trace.string(" unblocking ").string(Thread.currentThread().getName()).newline(); - } - } finally { - mutex.unlock(); - } - } - - public void signal(boolean onWorkerThread) { - mutex.lock(); - try { - Log.log().string("signaling on ").string(Thread.currentThread().getName()).newline(); - workerThreadActive = !onWorkerThread; - cond.broadcast(); - } finally { - mutex.unlock(); - } - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 49fcd616faaa..9a4b14251f57 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -826,20 +826,6 @@ public static int getNewThreadStatus(CFunctionOptions.Transition transition) { } } - public static class ParallelGCSupport {///unused - private static final FastThreadLocalInt isParallelGCThreadTL = FastThreadLocalFactory.createInt("ParallelGCSupport.isParallelGCThreadTL"); - private static final int REGULAR_THREAD = 0; - private static final int PARALLEL_GC_THREAD = 1; - - public static boolean isParallelGCThread(IsolateThread vmThread) { - return isParallelGCThreadTL.getVolatile(vmThread) == PARALLEL_GC_THREAD; - } - - public static void setParallelGCThread() { - isParallelGCThreadTL.setVolatile(PARALLEL_GC_THREAD); - } - } - public static class SafepointBehavior { /** Determines how this thread interacts with the safepoint handling. */ private static final FastThreadLocalInt safepointBehaviorTL = FastThreadLocalFactory.createInt("StatusSupport.safepointBehaviorTL"); From ef9c35f7eca5c2fa450c35a020b30837051f122f Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 15 Aug 2022 17:04:43 +0300 Subject: [PATCH 048/125] Introduced `UseParallelGC` option --- .../core/genscavenge/AlignedHeapChunk.java | 6 +-- .../oracle/svm/core/genscavenge/GCImpl.java | 19 ++++---- .../svm/core/genscavenge/HeapChunk.java | 3 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 2 +- .../core/genscavenge/PinnedObjectImpl.java | 2 +- .../oracle/svm/core/genscavenge/Space.java | 29 +++++------- .../core/genscavenge/UnalignedHeapChunk.java | 2 +- ...rialOrEpsilonGC.java => UseGraalCeGC.java} | 4 +- .../graal/GenScavengeGCFeature.java | 2 +- .../genscavenge/parallel/ParallelGCImpl.java | 45 ++++++++++++------- .../com/oracle/svm/core/SubstrateOptions.java | 26 +++++++++++ .../graal/snippets/CEntryPointSnippets.java | 2 +- .../com/oracle/svm/core/heap/ParallelGC.java | 26 ++++++++++- .../oracle/svm/hosted/heap/PodSupport.java | 2 +- 14 files changed, 114 insertions(+), 56 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/{UseSerialOrEpsilonGC.java => UseGraalCeGC.java} (90%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 8d81707a7293..25f25d0a653d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -114,10 +114,10 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { return result; } - /** Retract last allocation made. Used by parallel collector. */ + /** Retract the latest allocation. Used by parallel collector. */ static Pointer freeMemory(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); - assert newTop.aboveOrEqual(HeapChunk.asPointer(that)); + assert newTop.aboveThan(HeapChunk.asPointer(that)); HeapChunk.setTopPointer(that, newTop); return newTop; } @@ -184,7 +184,7 @@ public UnsignedWord getAllocationStart(AlignedHeapChunk.AlignedHeader heapChunk) class AlignedHeapChunkMemoryWalkerAccessFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index a8cb29389656..ee38df73b794 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -30,6 +30,7 @@ import java.lang.ref.Reference; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ReferenceAccess; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; @@ -1045,17 +1046,17 @@ private static void prepareForPromotion(boolean isIncremental) { /// queue objects as they are copied? private void scanGreyObjects(boolean isIncremental) { - HeapImpl heap = HeapImpl.getHeapImpl(); Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); - if (isIncremental) { - heap.getYoungGeneration().scanGreyObjects(); + try { + if (isIncremental) { + scanGreyObjectsLoop(); + } else { + HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); + ParallelGCImpl.waitForIdle(); + } + } finally { + scanGreyObjectsTimer.close(); } - heap.getOldGeneration().scanGreyObjects(); - - ParallelGCImpl.setEnabled(true); - ParallelGCImpl.waitForIdle(); - ParallelGCImpl.setEnabled(false); - scanGreyObjectsTimer.close(); } private static void scanGreyObjectsLoop() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index f56f97261d9c..962423f5c1f2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -282,8 +282,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi @AlwaysInline("GC performance") public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; - Pointer top = getTopPointer(that); - while (offset.belowThan(top)) { + while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read Object obj = offset.toObject(); if (!visitor.visitObjectInline(obj)) { return false; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index d9d2b1ab8b31..57d9da3f677a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -887,7 +887,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } -@TargetClass(value = java.lang.Runtime.class, onlyWith = UseSerialOrEpsilonGC.class) +@TargetClass(value = java.lang.Runtime.class, onlyWith = UseGraalCeGC.class) @SuppressWarnings("static-method") final class Target_java_lang_Runtime { @Substitute diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index 0622b3561bd3..1047bf0ff49d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -69,7 +69,7 @@ public boolean isPinned(Object object) { static class PinnedObjectFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3d5b61d00ce9..bebbe655e344 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -39,6 +39,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; @@ -47,6 +48,7 @@ import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; /** * A Space is a collection of HeapChunks. @@ -166,7 +168,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") - Pointer allocateMemory(UnsignedWord objectSize) { + private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); @@ -187,10 +189,10 @@ Pointer allocateMemory(UnsignedWord objectSize) { } /** - * Retract last allocation made. Used by parallel collector. + * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") - Pointer freeMemory(UnsignedWord objectSize) { + private Pointer freeMemory(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); assert oldChunk.isNonNull(); return AlignedHeapChunk.freeMemory(oldChunk, objectSize); @@ -222,11 +224,9 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - ///assert owns mutex || pargc thread ? - ///use thread mutex to protect appendNewChunk -// if (SubstrateOptions.MultiThreaded.getValue()) { -// VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); -// } + if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); + } appendAlignedHeapChunkUninterruptibly(aChunk); accounting.noteAlignedHeapChunk(); } @@ -277,10 +277,9 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - ///assert owns mutex || pargc thread ? -// if (SubstrateOptions.MultiThreaded.getValue()) { -// VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); -// } + if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); + } appendUnalignedHeapChunkUninterruptibly(uChunk); accounting.noteUnalignedHeapChunk(uChunk); } @@ -409,7 +408,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { } // Install forwarding pointer into the original header - assert ObjectHeaderImpl.getHubOffset() == 0; ///PGC prerequisite + assert ObjectHeaderImpl.getHubOffset() == 0; Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { @@ -477,14 +476,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.mutex.lock(); mutex.lock(); try { originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); } finally { mutex.unlock(); - originalSpace.mutex.unlock(); } if (ParallelGCImpl.isEnabled()) { @@ -507,14 +504,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - originalSpace.mutex.lock(); mutex.lock(); try { originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); } finally { mutex.unlock(); - originalSpace.mutex.unlock(); } if (ParallelGCImpl.isEnabled()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 299309cb970b..30014b36b163 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -186,7 +186,7 @@ public UnsignedWord getAllocationStart(UnalignedHeapChunk.UnalignedHeader heapCh class UnalignedHeapChunkMemoryWalkerAccessFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java similarity index 90% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java index cce0bb85bf71..47edd9b1ee6d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java @@ -32,9 +32,9 @@ import com.oracle.svm.core.SubstrateOptions; @Platforms(Platform.HOSTED_ONLY.class) -class UseSerialOrEpsilonGC implements BooleanSupplier { +class UseGraalCeGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index 61569d837a40..7fe017b4a49d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -63,7 +63,7 @@ class GenScavengeGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index c46d32d6ecff..c89638509b14 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,8 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -11,6 +13,7 @@ import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; +import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; @@ -42,6 +45,10 @@ public class ParallelGCImpl extends ParallelGC { @Override public void startWorkerThreads() { + /// determine worker count + int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); + VMError.guarantee(hubOffset == 0, "hub offset must be 0"); + IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); } @@ -54,7 +61,7 @@ public void startWorkerThread(int n) { localStack.set(STACKS[n]); getStats().install(); try { - while (!stopped) { + while (true) { mutex.lock(); while (busy.get() < WORKERS_COUNT) { log().string("WW block ").unsigned(n).newline(); @@ -103,23 +110,29 @@ private static void push(Pointer ptr, ThreadLocalTaskStack stack) { } public static void waitForIdle() { - log().string("PP start workers\n"); - busy.set(WORKERS_COUNT); - cond.broadcast(); // let worker threads run - - mutex.lock(); - while (busy.get() > 0) { - log().string("PP wait busy=").unsigned(busy.get()).newline(); - cond.block(); // wait for them to become idle + if (isSupported()) { + setEnabled(true); + + log().string("PP start workers\n"); + busy.set(WORKERS_COUNT); + cond.broadcast(); // let worker threads run + + mutex.lock(); + while (busy.get() > 0) { + log().string("PP wait busy=").unsigned(busy.get()).newline(); + cond.block(); // wait for them to become idle + } + mutex.unlock(); + + setEnabled(false); } - mutex.unlock(); } public static boolean isEnabled() { return enabled; } - public static void setEnabled(boolean enabled) { + private static void setEnabled(boolean enabled) { ParallelGCImpl.enabled = enabled; } @@ -143,11 +156,11 @@ static Log log() { @AutomaticFeature class ParallelGCFeature implements Feature { -/// -// @Override -// public boolean isInConfiguration(IsInConfigurationAccess access) { -// return SubstrateOptions.UseSerialGC.getValue(); -// } + + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return SubstrateOptions.UseParallelGC.getValue(); + } @Override public void afterRegistration(AfterRegistrationAccess access) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index d125b205cf10..94ff01a0fd98 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -39,6 +39,8 @@ import java.util.List; import java.util.function.Predicate; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.VMError; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -300,6 +302,7 @@ public String helpText() { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { + SubstrateOptions.UseParallelGC.update(values, false); SubstrateOptions.UseEpsilonGC.update(values, false); } } @@ -312,10 +315,33 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { SubstrateOptions.UseSerialGC.update(values, false); + SubstrateOptions.UseParallelGC.update(values, false); } } }; + @APIOption(name = "parallel", group = GCGroup.class, customHelp = "Parallel garbage collector")// + @Option(help = "Use a parallel GC")// + public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false) { + @Override + protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { + if (newValue) { + SubstrateOptions.UseSerialGC.update(values, false); + SubstrateOptions.UseEpsilonGC.update(values, false); + ConcealedOptions.UseRememberedSet.update(values, false); + SubstrateOptions.MultiThreaded.update(values, true); + } + } + }; + + @Fold + public static boolean useGraalCeGC() { + return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); + } + + @Option(help = "Number of parallel GC worker threads.", type = OptionType.User)// + public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0L); + @Option(help = "The size of each thread stack at run-time, in bytes.", type = OptionType.User)// public static final RuntimeOptionKey StackSize = new RuntimeOptionKey<>(0L); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 05f89811bce0..f081e00bca5d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -292,7 +292,7 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); + ParallelGC.start(); /* * After starting all the necessary threads, we can finally execute complex JDK code or code diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index c62182edba9f..f161bef7c9d1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,7 +1,31 @@ package com.oracle.svm.core.heap; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.log.Log; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; + public abstract class ParallelGC { - protected volatile boolean stopped; + + @Fold + public static ParallelGC singleton() { + return ImageSingletons.lookup(ParallelGC.class); + } + + @Fold + protected static boolean isSupported() { + return SubstrateOptions.UseParallelGC.getValue(); + } + + public static void start() { + Log log = Log.log().string("ParGC ");/// + if (isSupported()) { + log.string("supported").newline(); + singleton().startWorkerThreads(); + } else { + log.string("missing").newline(); + } + } public abstract void startWorkerThreads(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java index 23bb8ea222b0..7c92b153de27 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java @@ -118,7 +118,7 @@ final class PodFeature implements PodSupport, Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() || SubstrateOptions.UseEpsilonGC.getValue(); + return SubstrateOptions.useGraalCeGC(); } @Override From 4c77aa3e1fae306860b1d93add67fe32b9dbc6c5 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 16 Aug 2022 13:11:47 +0300 Subject: [PATCH 049/125] Removed some static state from ParallelGCImpl --- .../oracle/svm/core/genscavenge/GCImpl.java | 2 - .../genscavenge/GreyToBlackObjectVisitor.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 45 ++++-- .../genscavenge/parallel/ParallelGCImpl.java | 132 ++++++++++-------- .../com/oracle/svm/core/heap/ParallelGC.java | 9 +- 5 files changed, 107 insertions(+), 87 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ee38df73b794..ce972b44606f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -165,7 +165,6 @@ private void collect(GCCause cause, boolean forceFullGC) { if (outOfMemory) { throw OutOfMemoryUtil.heapSizeExceeded(); } - ParallelGCImpl.checkThrowable(); } } @@ -1148,7 +1147,6 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - ParallelGCImpl.TLAB.set(WordFactory.nullPointer()); ///rm? } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index ccce076078a1..ac996cd2c6b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -25,6 +25,8 @@ package com.oracle.svm.core.genscavenge; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.log.Log; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -58,8 +60,8 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { - if (!ParallelGCImpl.isEnabled()) { - ParallelGCImpl.push(Word.objectToUntrackedPointer(o)); + if (ParallelGC.isSupported() && !ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(o)); return true; } else { return doVisitObject(o); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index bebbe655e344..5fec82ce51cb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.word.Word; @@ -171,7 +172,12 @@ public Log report(Log log, boolean traceHeapChunks) { private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + AlignedHeapChunk.AlignedHeader oldChunk; + if (ParallelGC.isSupported()) { + oldChunk = ParallelGCImpl.getThreadLocalChunk(); + } else { + oldChunk = getLastAlignedHeapChunk(); + } if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -179,13 +185,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ - mutex.lock(); - try { - result = allocateInNewChunk(objectSize); - return result; - } finally { - mutex.unlock(); - } + return allocateInNewChunk(objectSize); } /** @@ -193,14 +193,25 @@ private Pointer allocateMemory(UnsignedWord objectSize) { */ @AlwaysInline("GC performance") private Pointer freeMemory(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.TLAB.get(); + assert ParallelGCImpl.isInParallelPhase(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.freeMemory(oldChunk, objectSize); } private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); - ParallelGCImpl.TLAB.set(newChunk); + AlignedHeapChunk.AlignedHeader newChunk; + if (ParallelGC.isSupported()) { + mutex.lock(); + try { + newChunk = requestAlignedHeapChunk(); + } finally { + mutex.unlock(); + } + ParallelGCImpl.setThreadLocalChunk(newChunk); + } else { + newChunk = requestAlignedHeapChunk(); + } if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } @@ -215,6 +226,10 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { lastAlignedHeapChunk = WordFactory.nullPointer(); firstUnalignedHeapChunk = WordFactory.nullPointer(); lastUnalignedHeapChunk = WordFactory.nullPointer(); + + if (ParallelGC.isSupported()) { + ParallelGCImpl.killThreadLocalChunk(); + } accounting.reset(); } @@ -371,7 +386,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -428,7 +443,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.localPush(copyMemory); + ParallelGCImpl.singleton().pushLocally(copyMemory); return copy; } else { // Retract speculatively allocated memory @@ -484,7 +499,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina mutex.unlock(); } - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isInParallelPhase()) { if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } @@ -512,7 +527,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o mutex.unlock(); } - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isInParallelPhase()) { if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index c89638509b14..d854952312ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,5 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; @@ -14,6 +13,7 @@ import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; @@ -25,23 +25,22 @@ public class ParallelGCImpl extends ParallelGC { /// static -> ImageSingletons - public static final int WORKERS_COUNT = 4; + private static final int WORKERS_COUNT = 4; /// tlab of the current to-space - public static final FastThreadLocalWord TLAB = - FastThreadLocalFactory.createWord("ParallelGCImpl.TLAB"); + private static final FastThreadLocalWord chunkTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); private static final VMMutex mutex = new VMMutex("pargc"); private static final VMCondition cond = new VMCondition(mutex); - private static AtomicInteger busy = new AtomicInteger(0); + private final AtomicInteger busy = new AtomicInteger(0); private static final ThreadLocalTaskStack[] STACKS = IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); private static final ThreadLocal localStack = new ThreadLocal<>(); - private static int currentStack; + private int currentStack; - private static volatile boolean enabled; - private static volatile Throwable throwable; + private volatile boolean inParallelPhase; @Override public void startWorkerThreads() { @@ -52,36 +51,32 @@ public void startWorkerThreads() { IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); } - public void startWorkerThread(int n) { + private void startWorkerThread(int n) { Thread t = new Thread(() -> { VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); final ThreadLocalTaskStack stack = STACKS[n]; - localStack.set(STACKS[n]); + localStack.set(stack); getStats().install(); - try { - while (true) { - mutex.lock(); - while (busy.get() < WORKERS_COUNT) { - log().string("WW block ").unsigned(n).newline(); - cond.block(); - } - mutex.unlock(); - - log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); - Object obj; - while ((obj = stack.pop()) != null) { - getVisitor().doVisitObject(obj); - } - TLAB.set(WordFactory.nullPointer()); - if (busy.decrementAndGet() <= 0) { - cond.broadcast(); - } - log().string("WW idle ").unsigned(n).newline(); + while (true) { + mutex.lock(); + while (busy.get() < WORKERS_COUNT) { + log().string("WW block ").unsigned(n).newline(); + cond.block(); } - } catch (Throwable e) { - throwable = e; + mutex.unlock(); + + log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); + Object obj; + while ((obj = stack.pop()) != null) { + getVisitor().doVisitObject(obj); + } + killThreadLocalChunk(); + if (busy.decrementAndGet() <= 0) { + cond.broadcast(); + } + log().string("WW idle ").unsigned(n).newline(); } }); t.setName("ParallelGCWorker-" + n); @@ -89,60 +84,75 @@ public void startWorkerThread(int n) { t.start(); } - public static GreyToBlackObjectVisitor getVisitor() { + private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } /// name, explain - public static void push(Pointer ptr) { + public void push(Pointer ptr) { + assert !isInParallelPhase(); push(ptr, STACKS[currentStack]); currentStack = (currentStack + 1) % WORKERS_COUNT; } - public static void localPush(Pointer ptr) { + public void pushLocally(Pointer ptr) { + assert isInParallelPhase(); push(ptr, localStack.get()); } - private static void push(Pointer ptr, ThreadLocalTaskStack stack) { + private void push(Pointer ptr, ThreadLocalTaskStack stack) { if (!stack.push(ptr)) { getVisitor().doVisitObject(ptr.toObject()); } } - public static void waitForIdle() { - if (isSupported()) { - setEnabled(true); - - log().string("PP start workers\n"); - busy.set(WORKERS_COUNT); - cond.broadcast(); // let worker threads run + @Fold + public static ParallelGCImpl singleton() { + return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + } - mutex.lock(); - while (busy.get() > 0) { - log().string("PP wait busy=").unsigned(busy.get()).newline(); - cond.block(); // wait for them to become idle - } - mutex.unlock(); + public static boolean isInParallelPhase() { + return isSupported() && singleton().inParallelPhase; + } - setEnabled(false); + public static void waitForIdle() { + if (isSupported()) { + singleton().waitForIdleImpl(); } } - public static boolean isEnabled() { - return enabled; + public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + return chunkTL.get(); + } + + public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + chunkTL.set(chunk); } - private static void setEnabled(boolean enabled) { - ParallelGCImpl.enabled = enabled; + public static void killThreadLocalChunk() { + chunkTL.set(WordFactory.nullPointer()); } - public static void checkThrowable() { - if (throwable != null) { - Log.log().string("PGC error : ").string(throwable.getClass().getName()) - .string(" : ").string(throwable.getMessage()).newline(); - throwable.printStackTrace(); - throw new Error(throwable); + private void waitForIdleImpl() { + setInParallelPhase(true); + + log().string("PP start workers\n"); + busy.set(WORKERS_COUNT); + cond.broadcast(); // let worker threads run + + mutex.lock(); + while (busy.get() > 0) { + log().string("PP wait busy=").unsigned(busy.get()).newline(); + cond.block(); // wait for them to become idle } + mutex.unlock(); + + setInParallelPhase(false); + } + + private void setInParallelPhase(boolean inParallelPhase) { + assert isSupported(); + singleton().inParallelPhase = inParallelPhase; } public static Stats getStats() { @@ -150,7 +160,7 @@ public static Stats getStats() { } static Log log() { - return Log.noopLog(); + return Log.log();/// } } @@ -159,7 +169,7 @@ class ParallelGCFeature implements Feature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseParallelGC.getValue(); + return ParallelGC.isSupported(); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index f161bef7c9d1..cc498ea348ca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -8,12 +8,7 @@ public abstract class ParallelGC { @Fold - public static ParallelGC singleton() { - return ImageSingletons.lookup(ParallelGC.class); - } - - @Fold - protected static boolean isSupported() { + public static boolean isSupported() { return SubstrateOptions.UseParallelGC.getValue(); } @@ -21,7 +16,7 @@ public static void start() { Log log = Log.log().string("ParGC ");/// if (isSupported()) { log.string("supported").newline(); - singleton().startWorkerThreads(); + ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); } else { log.string("missing").newline(); } From 3d6b797cb8e5595559c9de9bb20a85cfe66f7cda Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 16 Aug 2022 15:26:19 +0300 Subject: [PATCH 050/125] Replaced `Space.mutex` with `ParallelGCImpl.mutex` --- .../oracle/svm/core/genscavenge/Space.java | 46 ++++++++----------- .../genscavenge/parallel/ParallelGCImpl.java | 4 +- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 5fec82ce51cb..a77e18189428 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -62,7 +62,6 @@ public final class Space { private final boolean isFromSpace; private final int age; private final ChunksAccounting accounting; - private final VMMutex mutex; /* Heads and tails of the HeapChunk lists. */ private AlignedHeapChunk.AlignedHeader firstAlignedHeapChunk; @@ -87,7 +86,6 @@ public final class Space { this.isFromSpace = isFromSpace; this.age = age; this.accounting = new ChunksAccounting(accounting); - this.mutex = new VMMutex(name + "-mutex"); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -200,17 +198,13 @@ private Pointer freeMemory(UnsignedWord objectSize) { } private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk; if (ParallelGC.isSupported()) { - mutex.lock(); - try { - newChunk = requestAlignedHeapChunk(); - } finally { - mutex.unlock(); - } + ParallelGCImpl.mutex.lock(); + } + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + if (ParallelGC.isSupported()) { + ParallelGCImpl.mutex.unlock(); ParallelGCImpl.setThreadLocalChunk(newChunk); - } else { - newChunk = requestAlignedHeapChunk(); } if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); @@ -491,15 +485,14 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - mutex.lock(); - try { - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); - } finally { - mutex.unlock(); + boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + if (parallel) { + ParallelGCImpl.mutex.lock(); } - - if (ParallelGCImpl.isInParallelPhase()) { + originalSpace.extractAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk); + if (parallel) { + ParallelGCImpl.mutex.unlock(); if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } @@ -519,15 +512,14 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - mutex.lock(); - try { - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); - } finally { - mutex.unlock(); + boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + if (parallel) { + ParallelGCImpl.mutex.lock(); } - - if (ParallelGCImpl.isInParallelPhase()) { + originalSpace.extractUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk); + if (parallel) { + ParallelGCImpl.mutex.unlock(); if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index d854952312ed..3c386e843f1b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -31,8 +31,8 @@ public class ParallelGCImpl extends ParallelGC { private static final FastThreadLocalWord chunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); - private static final VMMutex mutex = new VMMutex("pargc"); - private static final VMCondition cond = new VMCondition(mutex); + public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); + private final VMCondition cond = new VMCondition(mutex); private final AtomicInteger busy = new AtomicInteger(0); private static final ThreadLocalTaskStack[] STACKS = From 4a3c01c616e68087f16ec4fd4f62a624ea15de2c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 16 Aug 2022 16:53:15 +0300 Subject: [PATCH 051/125] Cleanup: removed Stats printout --- .../com/oracle/svm/core/genscavenge/GCImpl.java | 8 +++++--- .../src/com/oracle/svm/core/genscavenge/Space.java | 2 +- .../core/genscavenge/parallel/ParallelGCImpl.java | 14 +++++++------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index ce972b44606f..4c04753fdb73 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -129,6 +129,8 @@ public final class GCImpl implements GC { public String getName() { if (SubstrateOptions.UseEpsilonGC.getValue()) { return "Epsilon GC"; + } else if (ParallelGC.isSupported()) { + return "Parallel GC"; } else { return "Serial GC"; } @@ -206,7 +208,6 @@ private void collectOperation(CollectionVMOperationData data) { GCCause cause = GCCause.fromId(data.getCauseId()); printGCBefore(cause.getName()); - ParallelGCImpl.getStats().reset(); boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC()); printGCAfter(cause.getName()); @@ -434,7 +435,6 @@ private void printGCAfter(String cause) { } else { timers.logAfterCollection(verboseGCLog); } - ParallelGCImpl.getStats().print(Log.log()); verboseGCLog.string("]"); verboseGCLog.string("]").newline(); } @@ -1051,7 +1051,9 @@ private void scanGreyObjects(boolean isIncremental) { scanGreyObjectsLoop(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); - ParallelGCImpl.waitForIdle(); + if (ParallelGC.isSupported()) { + ParallelGCImpl.waitForIdle(); + } } } finally { scanGreyObjectsTimer.close(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index a77e18189428..1b353f2bed7b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -380,7 +380,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGC.isSupported() && ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 3c386e843f1b..0ad3f33a0c08 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -24,10 +24,12 @@ public class ParallelGCImpl extends ParallelGC { - /// static -> ImageSingletons + /// determine at runtime? private static final int WORKERS_COUNT = 4; - /// tlab of the current to-space + /** + * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. + */ private static final FastThreadLocalWord chunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); @@ -44,7 +46,6 @@ public class ParallelGCImpl extends ParallelGC { @Override public void startWorkerThreads() { - /// determine worker count int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); @@ -112,13 +113,12 @@ public static ParallelGCImpl singleton() { } public static boolean isInParallelPhase() { - return isSupported() && singleton().inParallelPhase; + assert isSupported(); + return singleton().inParallelPhase; } public static void waitForIdle() { - if (isSupported()) { - singleton().waitForIdleImpl(); - } + singleton().waitForIdleImpl(); } public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { From d99ca0f666144e7796034f1d08ad5a63d6de05a2 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 10:53:05 +0300 Subject: [PATCH 052/125] Enqueue objects as they are copied `scanGreyObjects` is a noop --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 8 +++----- .../src/com/oracle/svm/core/genscavenge/Space.java | 3 +++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 4c04753fdb73..4ab75b898420 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1043,17 +1043,15 @@ private static void prepareForPromotion(boolean isIncremental) { } } - /// queue objects as they are copied? private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { - if (isIncremental) { + if (ParallelGC.isSupported()) { + ParallelGCImpl.waitForIdle(); + } else if (isIncremental) { scanGreyObjectsLoop(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); - if (ParallelGC.isSupported()) { - ParallelGCImpl.waitForIdle(); - } } } finally { scanGreyObjectsTimer.close(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 1b353f2bed7b..182ca48f9a62 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -387,6 +387,9 @@ Object promoteAlignedObject(Object original, Space originalSpace) { Object copy = copyAlignedObject(original); if (copy != null) { ObjectHeaderImpl.installForwardingPointer(original, copy); + if (ParallelGC.isSupported()) { + ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(copy)); + } } return copy; } From e29e81debb622fdf663c35ac0df2ad5a13169c6e Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 11:13:28 +0300 Subject: [PATCH 053/125] Cleanup --- .../oracle/svm/core/genscavenge/GCImpl.java | 8 +++---- .../genscavenge/GreyToBlackObjectVisitor.java | 5 ++--- .../oracle/svm/core/genscavenge/Space.java | 21 ++++++++----------- .../genscavenge/parallel/ParallelGCImpl.java | 15 ++++++++----- .../com/oracle/svm/core/SubstrateOptions.java | 5 ----- .../graal/snippets/CEntryPointSnippets.java | 4 ++-- .../com/oracle/svm/core/heap/ParallelGC.java | 11 +++------- 7 files changed, 30 insertions(+), 39 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 4ab75b898420..b66b85a79047 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -29,9 +29,6 @@ import java.lang.ref.Reference; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.heap.ReferenceAccess; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; @@ -69,12 +66,15 @@ import com.oracle.svm.core.genscavenge.BasicCollectionPolicies.NeverCollect; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.CodeReferenceMapDecoder; import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.NoAllocationVerifier; import com.oracle.svm.core.heap.OutOfMemoryUtil; +import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceMapIndex; import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner; @@ -129,7 +129,7 @@ public final class GCImpl implements GC { public String getName() { if (SubstrateOptions.UseEpsilonGC.getValue()) { return "Epsilon GC"; - } else if (ParallelGC.isSupported()) { + } else if (SubstrateOptions.UseParallelGC.getValue()) { return "Parallel GC"; } else { return "Serial GC"; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index ac996cd2c6b5..0f725cc1cb92 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -24,15 +24,14 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.annotate.NeverInline; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 182ca48f9a62..f8f34b2427ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,10 +28,6 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import com.oracle.svm.core.annotate.AlwaysInline; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -44,12 +40,15 @@ import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; /** * A Space is a collection of HeapChunks. @@ -380,7 +379,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGC.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -440,7 +439,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.singleton().pushLocally(copyMemory); + ParallelGCImpl.singleton().pushToLocalStack(copyMemory); return copy; } else { // Retract speculatively allocated memory @@ -488,13 +487,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); @@ -515,13 +513,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - boolean parallel = ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (parallel) { + if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { throw VMError.shouldNotReachHere(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 0ad3f33a0c08..8c95df1f9cef 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -45,7 +45,7 @@ public class ParallelGCImpl extends ParallelGC { private volatile boolean inParallelPhase; @Override - public void startWorkerThreads() { + public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); @@ -89,14 +89,20 @@ private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - /// name, explain + /** + * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' stacks + * in round-robin fashion, attempting to balance load between workers. + */ public void push(Pointer ptr) { assert !isInParallelPhase(); push(ptr, STACKS[currentStack]); currentStack = (currentStack + 1) % WORKERS_COUNT; } - public void pushLocally(Pointer ptr) { + /** + * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local stack. + */ + public void pushToLocalStack(Pointer ptr) { assert isInParallelPhase(); push(ptr, localStack.get()); } @@ -113,8 +119,7 @@ public static ParallelGCImpl singleton() { } public static boolean isInParallelPhase() { - assert isSupported(); - return singleton().inParallelPhase; + return isSupported() && singleton().inParallelPhase; } public static void waitForIdle() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 94ff01a0fd98..5fa114886186 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -39,8 +39,6 @@ import java.util.List; import java.util.function.Predicate; -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.util.VMError; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -339,9 +337,6 @@ public static boolean useGraalCeGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); } - @Option(help = "Number of parallel GC worker threads.", type = OptionType.User)// - public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0L); - @Option(help = "The size of each thread stack at run-time, in bytes.", type = OptionType.User)// public static final RuntimeOptionKey StackSize = new RuntimeOptionKey<>(0L); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index f081e00bca5d..14f67dd1a2c7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -33,7 +33,6 @@ import java.util.Map; -import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; @@ -87,6 +86,7 @@ import com.oracle.svm.core.graal.nodes.CEntryPointEnterNode; import com.oracle.svm.core.graal.nodes.CEntryPointLeaveNode; import com.oracle.svm.core.graal.nodes.CEntryPointUtilityNode; +import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.jdk.PlatformNativeLibrarySupport; @@ -292,7 +292,7 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ParallelGC.start(); + ParallelGC.startWorkerThreads(); /* * After starting all the necessary threads, we can finally execute complex JDK code or code diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index cc498ea348ca..3bd3cf42b34a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,7 +1,6 @@ package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.log.Log; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; @@ -12,15 +11,11 @@ public static boolean isSupported() { return SubstrateOptions.UseParallelGC.getValue(); } - public static void start() { - Log log = Log.log().string("ParGC ");/// + public static void startWorkerThreads() { if (isSupported()) { - log.string("supported").newline(); - ImageSingletons.lookup(ParallelGC.class).startWorkerThreads(); - } else { - log.string("missing").newline(); + ImageSingletons.lookup(ParallelGC.class).startWorkerThreadsImpl(); } } - public abstract void startWorkerThreads(); + public abstract void startWorkerThreadsImpl(); } From e068f43258dc74012ecfe2e5caa9d49fb7370a42 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 16:10:00 +0300 Subject: [PATCH 054/125] Fixed hangup; better worker thread management --- .../genscavenge/parallel/ParallelGCImpl.java | 57 ++++++++++--------- .../parallel/ThreadLocalTaskStack.java | 2 +- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 8c95df1f9cef..e1b8129bfbf4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -24,8 +24,10 @@ public class ParallelGCImpl extends ParallelGC { - /// determine at runtime? + /// determine at runtime, max 32 (from busyWorkers) private static final int WORKERS_COUNT = 4; + private static final int WORKERS_ALL = 0b1111; + private static final int WORKERS_NONE = 0; /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -34,8 +36,9 @@ public class ParallelGCImpl extends ParallelGC { FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); - private final VMCondition cond = new VMCondition(mutex); - private final AtomicInteger busy = new AtomicInteger(0); + private final VMCondition seqPhase = new VMCondition(mutex); + private final VMCondition parPhase = new VMCondition(mutex); + private final AtomicInteger busyWorkers = new AtomicInteger(0); private static final ThreadLocalTaskStack[] STACKS = IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); @@ -61,12 +64,12 @@ private void startWorkerThread(int n) { localStack.set(stack); getStats().install(); while (true) { - mutex.lock(); - while (busy.get() < WORKERS_COUNT) { + do { log().string("WW block ").unsigned(n).newline(); - cond.block(); - } - mutex.unlock(); + mutex.lock(); + parPhase.block(); + mutex.unlock(); + } while ((busyWorkers.get() & (1 << n)) == 0); log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); Object obj; @@ -74,8 +77,15 @@ private void startWorkerThread(int n) { getVisitor().doVisitObject(obj); } killThreadLocalChunk(); - if (busy.decrementAndGet() <= 0) { - cond.broadcast(); + + int witness = busyWorkers.get(); + int expected; + do { + expected = witness; + witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); + } while (witness != expected); + if (busyWorkers.get() == WORKERS_NONE) { + seqPhase.signal(); } log().string("WW idle ").unsigned(n).newline(); } @@ -139,25 +149,20 @@ public static void killThreadLocalChunk() { } private void waitForIdleImpl() { - setInParallelPhase(true); + inParallelPhase = true; log().string("PP start workers\n"); - busy.set(WORKERS_COUNT); - cond.broadcast(); // let worker threads run - - mutex.lock(); - while (busy.get() > 0) { - log().string("PP wait busy=").unsigned(busy.get()).newline(); - cond.block(); // wait for them to become idle + busyWorkers.set(WORKERS_ALL); + parPhase.broadcast(); // let worker threads run + + while (busyWorkers.get() != WORKERS_NONE) { + mutex.lock(); + log().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); + seqPhase.block(); // wait for them to become idle + mutex.unlock(); } - mutex.unlock(); - - setInParallelPhase(false); - } - private void setInParallelPhase(boolean inParallelPhase) { - assert isSupported(); - singleton().inParallelPhase = inParallelPhase; + inParallelPhase = false; } public static Stats getStats() { @@ -165,7 +170,7 @@ public static Stats getStats() { } static Log log() { - return Log.log();/// + return Log.noopLog();/// } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java index 09645ebcc404..eb1201dccf89 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java @@ -33,5 +33,5 @@ Object pop() { int size() { return top; - } + } ///rm? } \ No newline at end of file From e67370c9724364fa38f6f8852f1d4c2b26c75e74 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 17 Aug 2022 17:10:54 +0300 Subject: [PATCH 055/125] Reuse GC thread as one of the workers --- .../oracle/svm/core/genscavenge/Space.java | 6 +- .../core/genscavenge/parallel/CasQueue.java | 85 ----------- .../genscavenge/parallel/ParallelGCImpl.java | 140 +++++++++--------- .../core/genscavenge/parallel/TaskQueue.java | 105 ------------- ...lTaskStack.java => ThreadLocalBuffer.java} | 6 +- 5 files changed, 76 insertions(+), 266 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/{ThreadLocalTaskStack.java => ThreadLocalBuffer.java} (85%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index f8f34b2427ed..8b8ca33cb467 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -219,10 +219,6 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { lastAlignedHeapChunk = WordFactory.nullPointer(); firstUnalignedHeapChunk = WordFactory.nullPointer(); lastUnalignedHeapChunk = WordFactory.nullPointer(); - - if (ParallelGC.isSupported()) { - ParallelGCImpl.killThreadLocalChunk(); - } accounting.reset(); } @@ -439,7 +435,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.singleton().pushToLocalStack(copyMemory); + ParallelGCImpl.singleton().pushToLocalBuffer(copyMemory); return copy; } else { // Retract speculatively allocated memory diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java deleted file mode 100644 index 107fedc3afbd..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/CasQueue.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; -import org.graalvm.word.Pointer; - -import java.util.concurrent.atomic.AtomicInteger; - -/// Lame, test only. Doesn't handle overflows -public class CasQueue { - private static final int SIZE = 1024 * 1024; - - final Stats stats = new Stats(); - - private final Pointer[] data = new Pointer[SIZE]; - private final AtomicInteger getIndex = new AtomicInteger(0); - private final AtomicInteger putIndex = new AtomicInteger(0); - - private Log log() { - return ParallelGCImpl.log(); - } - - public boolean put(Pointer ptr) { - int cur = putIndex.get(); - int witness; - while ((witness = putIndex.compareAndExchange(cur, cur + 1)) != cur) { - cur = witness; - } - data[cur] = ptr; - return true; - } - - public boolean consume(Consumer consumer) { - int cur = getIndex.get(); - int witness; - while (cur < putIndex.get() && (witness = getIndex.compareAndExchange(cur, cur + 1)) != cur) { - cur = witness; - } - if (cur >= putIndex.get()) { - return false; - } - Object obj = data[cur].toObject(); - consumer.accept(obj); - return true; - } - - public void drain(Consumer consumer) { - while (consume(consumer)); - getIndex.set(0); - putIndex.set(0); - } - - interface Consumer { - void accept(Object object); - } - - // Non MT safe, needs locking - public static class Stats { - private int count, maxSize; - - void noteTask(int putIndex, int getIndex) { - int size = putIndex - getIndex; - if (size < 0) { - size += SIZE; - } - if (size > maxSize) { - maxSize = size; - } - count++; - } - - public int getCount() { - return count; - } - - public int getMaxSize() { - return maxSize; - } - - public void reset() { - count = maxSize = 0; - } - } -} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e1b8129bfbf4..79ad941a9c9a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -11,6 +11,7 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.api.replacements.Fold; @@ -26,9 +27,15 @@ public class ParallelGCImpl extends ParallelGC { /// determine at runtime, max 32 (from busyWorkers) private static final int WORKERS_COUNT = 4; - private static final int WORKERS_ALL = 0b1111; + private static final int WORKERS_ALL = 0b1110; private static final int WORKERS_NONE = 0; + private static final ThreadLocalBuffer[] BUFFERS = + IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); + private static final FastThreadLocalObject localBuffer = + FastThreadLocalFactory.createObject(ThreadLocalBuffer.class, "ParallelGCImpl.bufferTL"); + private int currentBuffer; + /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ @@ -40,19 +47,59 @@ public class ParallelGCImpl extends ParallelGC { private final VMCondition parPhase = new VMCondition(mutex); private final AtomicInteger busyWorkers = new AtomicInteger(0); - private static final ThreadLocalTaskStack[] STACKS = - IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalTaskStack()).toArray(ThreadLocalTaskStack[]::new); - private static final ThreadLocal localStack = new ThreadLocal<>(); - private int currentStack; - private volatile boolean inParallelPhase; + @Fold + public static ParallelGCImpl singleton() { + return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + } + + public static boolean isInParallelPhase() { + return isSupported() && singleton().inParallelPhase; + } + + public static void waitForIdle() { + singleton().waitForIdleImpl(); + } + + public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + return chunkTL.get(); + } + + public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + chunkTL.set(chunk); + } + + /** + * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' buffers + * in round-robin fashion, attempting to balance load between workers. + */ + public void push(Pointer ptr) { + assert !isInParallelPhase(); + push(ptr, BUFFERS[currentBuffer]); + currentBuffer = (currentBuffer + 1) % WORKERS_COUNT; + } + + /** + * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local buffer. + */ + public void pushToLocalBuffer(Pointer ptr) { + assert isInParallelPhase(); + push(ptr, localBuffer.get()); + } + + private void push(Pointer ptr, ThreadLocalBuffer buffer) { + if (!buffer.push(ptr)) { + getVisitor().doVisitObject(ptr.toObject()); + } + } + @Override public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - IntStream.range(0, WORKERS_COUNT).forEach(this::startWorkerThread); + IntStream.range(1, WORKERS_COUNT).forEach(this::startWorkerThread); } private void startWorkerThread(int n) { @@ -60,8 +107,8 @@ private void startWorkerThread(int n) { VMThreads.SafepointBehavior.markThreadAsCrashed(); log().string("WW start ").unsigned(n).newline(); - final ThreadLocalTaskStack stack = STACKS[n]; - localStack.set(stack); + final ThreadLocalBuffer buffer = BUFFERS[n]; + localBuffer.set(buffer); getStats().install(); while (true) { do { @@ -71,13 +118,6 @@ private void startWorkerThread(int n) { mutex.unlock(); } while ((busyWorkers.get() & (1 << n)) == 0); - log().string("WW run ").unsigned(n).string(", count=").unsigned(stack.size()).newline(); - Object obj; - while ((obj = stack.pop()) != null) { - getVisitor().doVisitObject(obj); - } - killThreadLocalChunk(); - int witness = busyWorkers.get(); int expected; do { @@ -95,59 +135,6 @@ private void startWorkerThread(int n) { t.start(); } - private GreyToBlackObjectVisitor getVisitor() { - return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); - } - - /** - * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' stacks - * in round-robin fashion, attempting to balance load between workers. - */ - public void push(Pointer ptr) { - assert !isInParallelPhase(); - push(ptr, STACKS[currentStack]); - currentStack = (currentStack + 1) % WORKERS_COUNT; - } - - /** - * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local stack. - */ - public void pushToLocalStack(Pointer ptr) { - assert isInParallelPhase(); - push(ptr, localStack.get()); - } - - private void push(Pointer ptr, ThreadLocalTaskStack stack) { - if (!stack.push(ptr)) { - getVisitor().doVisitObject(ptr.toObject()); - } - } - - @Fold - public static ParallelGCImpl singleton() { - return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); - } - - public static boolean isInParallelPhase() { - return isSupported() && singleton().inParallelPhase; - } - - public static void waitForIdle() { - singleton().waitForIdleImpl(); - } - - public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { - return chunkTL.get(); - } - - public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { - chunkTL.set(chunk); - } - - public static void killThreadLocalChunk() { - chunkTL.set(WordFactory.nullPointer()); - } - private void waitForIdleImpl() { inParallelPhase = true; @@ -155,6 +142,9 @@ private void waitForIdleImpl() { busyWorkers.set(WORKERS_ALL); parPhase.broadcast(); // let worker threads run + localBuffer.set(BUFFERS[0]); + drain(BUFFERS[0]); + while (busyWorkers.get() != WORKERS_NONE) { mutex.lock(); log().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); @@ -165,6 +155,18 @@ private void waitForIdleImpl() { inParallelPhase = false; } + private void drain(ThreadLocalBuffer buffer) { + Object obj; + while ((obj = buffer.pop()) != null) { + getVisitor().doVisitObject(obj); + } + chunkTL.set(WordFactory.nullPointer()); + } + + private GreyToBlackObjectVisitor getVisitor() { + return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); + } + public static Stats getStats() { return Stats.stats(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java deleted file mode 100644 index aaafd18fdea6..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/TaskQueue.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.locks.VMCondition; -import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; -import org.graalvm.word.Pointer; - -public class TaskQueue { - private static final int SIZE = 128 * 1024; - - private final VMMutex mutex; - private final VMCondition cond; - private final Pointer[] data; - private int idleCount; - private int getIndex; - private int putIndex; - - public TaskQueue(String name) { - mutex = new VMMutex(name + "-queue"); - cond = new VMCondition(mutex); - data = new Pointer[SIZE]; - } - - private Log log() { - return Log.noopLog(); - } - - private int next(int index) { - return (index + 1) % SIZE; - } - - private boolean canGet() { - return getIndex != putIndex; - } - - private boolean canPut() { - return next(putIndex) != getIndex; - } - - public boolean put(Pointer ptr) { - try { - mutex.lock(); - if (!canPut()) { - log().string("TQ cannot put task\n"); - return false; - } - data[putIndex] = ptr; - putIndex = next(putIndex); - } finally { - mutex.unlock(); - } - cond.broadcast(); - return true; - } - - public void consume(Consumer consumer) { - Pointer ptr; - mutex.lock(); - try { - while (!canGet()) { - log().string("TQ cannot get task\n"); - idleCount++; - cond.block(); - idleCount--; - } - ptr = data[getIndex]; - getIndex = next(getIndex); - } finally { - mutex.unlock(); - } - cond.broadcast(); - consumer.accept(ptr.toObject()); - } - - // Non MT safe. Only call when no workers are running - public void drain(Consumer consumer) { - while (canGet()) { - Object obj = data[getIndex].toObject(); - consumer.accept(obj); - getIndex = next(getIndex); - } - } - - public void waitUntilIdle(int expectedIdleCount) { - log().string("TQ waitForIdle\n"); - while (true) { - try { - mutex.lock(); - while (canGet()) { - cond.block(); - } - if (idleCount >= expectedIdleCount) { - log().string("TQ waitForIdle over\n"); - return; - } - } finally { - mutex.unlock(); - } - } - } - - interface Consumer { - void accept(Object object); - } -} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java similarity index 85% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java index eb1201dccf89..c34af0a589df 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalTaskStack.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java @@ -3,8 +3,10 @@ import com.oracle.svm.core.log.Log; import org.graalvm.word.Pointer; -/// thread local, non MT safe -public class ThreadLocalTaskStack { +/** + * Thread local (non MT safe) buffer where pointers to grey objects are stored. + */ +public class ThreadLocalBuffer { private static final int SIZE = 128 * 1024; private final Pointer[] data = new Pointer[SIZE]; From 8e33c3d0e752ffe7aa25ae7b404876852481a8b0 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 26 Aug 2022 10:06:59 +0300 Subject: [PATCH 056/125] Determine number of workers at runtime --- .../oracle/svm/core/genscavenge/Space.java | 7 +- .../genscavenge/parallel/ParallelGCImpl.java | 111 ++++++++++++------ .../parallel/ParallelGCOptions.java | 11 ++ .../parallel/ThreadLocalBuffer.java | 54 ++++++--- .../graal/snippets/CEntryPointSnippets.java | 5 +- 5 files changed, 127 insertions(+), 61 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8b8ca33cb467..d81a25b28bc5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -394,7 +394,6 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(original); - ObjectHeaderImpl impl = ObjectHeaderImpl.getObjectHeaderImpl(); Pointer originalMemory = Word.objectToUntrackedPointer(original); UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { @@ -403,7 +402,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. // If another thread copies the object first, we retract the allocation later. - UnsignedWord size = getSizeFromHeader(original, originalHeader, impl); + UnsignedWord size = getSizeFromHeader(original, originalHeader); assert size.aboveThan(0); Pointer copyMemory = allocateMemory(size); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { @@ -445,8 +444,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { } @AlwaysInline("GC performance") - public static UnsignedWord getSizeFromHeader(Object obj, UnsignedWord header, ObjectHeaderImpl impl) { - int encoding = impl.dynamicHubFromObjectHeader(header).getLayoutEncoding(); + public static UnsignedWord getSizeFromHeader(Object obj, UnsignedWord header) { + int encoding = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(header).getLayoutEncoding(); return LayoutEncoding.getSizeFromEncoding(obj, encoding); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 79ad941a9c9a..d99f6b454d38 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,11 +1,13 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; @@ -25,13 +27,19 @@ public class ParallelGCImpl extends ParallelGC { - /// determine at runtime, max 32 (from busyWorkers) - private static final int WORKERS_COUNT = 4; - private static final int WORKERS_ALL = 0b1110; - private static final int WORKERS_NONE = 0; + private static final int MAX_WORKERS_COUNT = 31; + /** + * Number of parallel GC workers. Default is 1 so that if GC happens before worker threads are started, + * it occurs on a single thread just like serial GC. + */ + private int workerCount = 1; + private int allWorkersBusy; + private final AtomicInteger busyWorkers = new AtomicInteger(0); + + /// create only as many as needed private static final ThreadLocalBuffer[] BUFFERS = - IntStream.range(0, WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); + IntStream.range(0, MAX_WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); private static final FastThreadLocalObject localBuffer = FastThreadLocalFactory.createObject(ThreadLocalBuffer.class, "ParallelGCImpl.bufferTL"); private int currentBuffer; @@ -45,7 +53,6 @@ public class ParallelGCImpl extends ParallelGC { public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); - private final AtomicInteger busyWorkers = new AtomicInteger(0); private volatile boolean inParallelPhase; @@ -76,8 +83,8 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { */ public void push(Pointer ptr) { assert !isInParallelPhase(); - push(ptr, BUFFERS[currentBuffer]); - currentBuffer = (currentBuffer + 1) % WORKERS_COUNT; + BUFFERS[currentBuffer].push(ptr); + currentBuffer = (currentBuffer + 1) % workerCount; } /** @@ -85,13 +92,7 @@ public void push(Pointer ptr) { */ public void pushToLocalBuffer(Pointer ptr) { assert isInParallelPhase(); - push(ptr, localBuffer.get()); - } - - private void push(Pointer ptr, ThreadLocalBuffer buffer) { - if (!buffer.push(ptr)) { - getVisitor().doVisitObject(ptr.toObject()); - } + localBuffer.get().push(ptr); } @Override @@ -99,35 +100,63 @@ public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - IntStream.range(1, WORKERS_COUNT).forEach(this::startWorkerThread); + workerCount = getWorkerCount(); ///cmp serialGC vs pargc with 1 worker + allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); + for (int i = 0; i < workerCount; i++) { + BUFFERS[i].runtimeInit(); + if (i > 0) { + // We reuse the gc thread for one of the workers, so number of worker threads is `workerCount - 1` + startWorkerThread(i); + } + } + } + + private int getWorkerCount() { + int setting = ParallelGCOptions.ParallelGCWorkers.getValue(); + int workers = setting > 0 ? setting : getDefaultWorkerCount(); + workers = Math.min(workers, MAX_WORKERS_COUNT); + verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); + return workers; + } + + private int getDefaultWorkerCount() { + // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() + int cpus = Jvm.JVM_ActiveProcessorCount(); + return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; } private void startWorkerThread(int n) { Thread t = new Thread(() -> { VMThreads.SafepointBehavior.markThreadAsCrashed(); - log().string("WW start ").unsigned(n).newline(); + debugLog().string("WW start ").unsigned(n).newline(); final ThreadLocalBuffer buffer = BUFFERS[n]; localBuffer.set(buffer); getStats().install(); while (true) { - do { - log().string("WW block ").unsigned(n).newline(); - mutex.lock(); - parPhase.block(); - mutex.unlock(); - } while ((busyWorkers.get() & (1 << n)) == 0); - - int witness = busyWorkers.get(); - int expected; - do { - expected = witness; - witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); - } while (witness != expected); - if (busyWorkers.get() == WORKERS_NONE) { - seqPhase.signal(); + try { + do { + debugLog().string("WW block ").unsigned(n).newline(); + mutex.lock(); + parPhase.block(); + mutex.unlock(); + } while ((busyWorkers.get() & (1 << n)) == 0); + + debugLog().string("WW run ").unsigned(n).newline(); + drain(buffer); + int witness = busyWorkers.get(); + int expected; + do { + expected = witness; + witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); + } while (witness != expected); + if (busyWorkers.get() == 0) { + seqPhase.signal(); + } + debugLog().string("WW idle ").unsigned(n).newline(); + } catch (Throwable ex) { + VMError.shouldNotReachHere(ex); } - log().string("WW idle ").unsigned(n).newline(); } }); t.setName("ParallelGCWorker-" + n); @@ -138,16 +167,16 @@ private void startWorkerThread(int n) { private void waitForIdleImpl() { inParallelPhase = true; - log().string("PP start workers\n"); - busyWorkers.set(WORKERS_ALL); + debugLog().string("PP start workers\n"); + busyWorkers.set(allWorkersBusy); parPhase.broadcast(); // let worker threads run localBuffer.set(BUFFERS[0]); drain(BUFFERS[0]); - while (busyWorkers.get() != WORKERS_NONE) { + while (busyWorkers.get() != 0) { mutex.lock(); - log().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); + debugLog().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); seqPhase.block(); // wait for them to become idle mutex.unlock(); } @@ -171,8 +200,12 @@ public static Stats getStats() { return Stats.stats(); } - static Log log() { - return Log.noopLog();/// + private static Log verboseGCLog() { + return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); + } + + static Log debugLog() {///rm + return Log.noopLog(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java new file mode 100644 index 000000000000..208f06fc633e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -0,0 +1,11 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.option.RuntimeOptionKey; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.options.OptionType; + +public class ParallelGCOptions { + + @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User) + public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0); +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java index c34af0a589df..1dc025567def 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java @@ -1,39 +1,61 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.log.Log; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; /** * Thread local (non MT safe) buffer where pointers to grey objects are stored. */ public class ThreadLocalBuffer { - private static final int SIZE = 128 * 1024; + private static final int INITIAL_SIZE = 128 * 1024 * 8; // 128k words + private static final OutOfMemoryError OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate a ThreadLocalBuffer"); - private final Pointer[] data = new Pointer[SIZE]; - private int top = 0; + private Pointer buffer = WordFactory.nullPointer(); + private UnsignedWord size = WordFactory.unsigned(INITIAL_SIZE); + private UnsignedWord top = WordFactory.zero(); private Log log() { - return ParallelGCImpl.log(); + return Log.log(); } - boolean push(Pointer ptr) { - if (top >= SIZE) { - log().string("TT cannot put task\n"); - return false; + void runtimeInit() { + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + ensureBuffer(); + } + + private void ensureCapacity() { + if (top.aboveOrEqual(size)) { + size = size.multiply(2); + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, size); + ensureBuffer(); + log().string("TLB grow to ").unsigned(size).newline(); + } + } + + private void ensureBuffer() { + if (buffer.isNull()) { + throw OutOfMemoryUtil.reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } - data[top++] = ptr; - return true; + } + + void push(Pointer ptr) { + ensureCapacity(); + buffer.writeWord(top, ptr); + top = top.add(8); } Object pop() { - if (top > 0) { - return data[--top].toObjectNonNull(); + if (top.aboveThan(0)) { + top = top.subtract(8); + Pointer ptr = buffer.readWord(top); + return ptr.toObjectNonNull(); } else { return null; } } - - int size() { - return top; - } ///rm? } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 14f67dd1a2c7..6be823f70e86 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -292,8 +292,6 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete ReferenceHandlerThread.start(); } - ParallelGC.startWorkerThreads(); - /* * After starting all the necessary threads, we can finally execute complex JDK code or code * that allocates a significant amount of memory. @@ -325,6 +323,9 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } + /* Number of workers can be affected by a runtime option, so start threads after options are parsed */ + ParallelGC.startWorkerThreads(); + boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); if (firstIsolate) { // let other isolates (if any) initialize now state = success ? FirstIsolateInitStates.SUCCESSFUL : FirstIsolateInitStates.FAILED; From 73177bd86ed7cf9ae4713ebf983d73a19f4a6139 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 15 Sep 2022 15:18:52 +0300 Subject: [PATCH 057/125] Shared synchronized buffer, so far holds object pointers --- .../genscavenge/parallel/ChunkBuffer.java | 37 ++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 38 +++++++------------ 2 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java new file mode 100644 index 000000000000..1da361a3609e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -0,0 +1,37 @@ +package com.oracle.svm.core.genscavenge.parallel; + +import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; + +/** + * Synchronized buffer where chunks to be scanned are stored. + */ +public class ChunkBuffer { + private static final int SIZE = 1024 * 1024; ///handle overflow + reduce for chunks + private static final Pointer[] buffer = new Pointer[SIZE]; + private int top; + + private Log debugLog() { + return ParallelGCImpl.debugLog(); + } + + void push(Pointer ptr) { + ParallelGCImpl.mutex.lock(); + assert top < SIZE; + buffer[top++] = ptr; + ParallelGCImpl.mutex.unlock(); + } + + Object pop() { + ParallelGCImpl.mutex.lock(); + try { + if (top > 0) { + return buffer[--top].toObjectNonNull(); + } else { + return null; + } + } finally { + ParallelGCImpl.mutex.unlock(); + } + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index d99f6b454d38..1d3148505bb4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -13,7 +13,6 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; -import com.oracle.svm.core.threadlocal.FastThreadLocalObject; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.api.replacements.Fold; @@ -23,7 +22,6 @@ import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -37,12 +35,7 @@ public class ParallelGCImpl extends ParallelGC { private int allWorkersBusy; private final AtomicInteger busyWorkers = new AtomicInteger(0); - /// create only as many as needed - private static final ThreadLocalBuffer[] BUFFERS = - IntStream.range(0, MAX_WORKERS_COUNT).mapToObj(i -> new ThreadLocalBuffer()).toArray(ThreadLocalBuffer[]::new); - private static final FastThreadLocalObject localBuffer = - FastThreadLocalFactory.createObject(ThreadLocalBuffer.class, "ParallelGCImpl.bufferTL"); - private int currentBuffer; + private static final ChunkBuffer buffer = new ChunkBuffer(); /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -83,8 +76,7 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { */ public void push(Pointer ptr) { assert !isInParallelPhase(); - BUFFERS[currentBuffer].push(ptr); - currentBuffer = (currentBuffer + 1) % workerCount; + buffer.push(ptr); } /** @@ -92,7 +84,7 @@ public void push(Pointer ptr) { */ public void pushToLocalBuffer(Pointer ptr) { assert isInParallelPhase(); - localBuffer.get().push(ptr); + buffer.push(ptr); } @Override @@ -102,12 +94,9 @@ public void startWorkerThreadsImpl() { workerCount = getWorkerCount(); ///cmp serialGC vs pargc with 1 worker allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); - for (int i = 0; i < workerCount; i++) { - BUFFERS[i].runtimeInit(); - if (i > 0) { - // We reuse the gc thread for one of the workers, so number of worker threads is `workerCount - 1` - startWorkerThread(i); - } + for (int i = 1; i < workerCount; i++) { + // We reuse the gc thread for worker #0 + startWorkerThread(i); } } @@ -130,8 +119,6 @@ private void startWorkerThread(int n) { VMThreads.SafepointBehavior.markThreadAsCrashed(); debugLog().string("WW start ").unsigned(n).newline(); - final ThreadLocalBuffer buffer = BUFFERS[n]; - localBuffer.set(buffer); getStats().install(); while (true) { try { @@ -143,7 +130,7 @@ private void startWorkerThread(int n) { } while ((busyWorkers.get() & (1 << n)) == 0); debugLog().string("WW run ").unsigned(n).newline(); - drain(buffer); + drainBuffer(); int witness = busyWorkers.get(); int expected; do { @@ -151,7 +138,9 @@ private void startWorkerThread(int n) { witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); } while (witness != expected); if (busyWorkers.get() == 0) { + mutex.lock(); seqPhase.signal(); + mutex.unlock(); } debugLog().string("WW idle ").unsigned(n).newline(); } catch (Throwable ex) { @@ -171,20 +160,19 @@ private void waitForIdleImpl() { busyWorkers.set(allWorkersBusy); parPhase.broadcast(); // let worker threads run - localBuffer.set(BUFFERS[0]); - drain(BUFFERS[0]); + drainBuffer(); + mutex.lock(); while (busyWorkers.get() != 0) { - mutex.lock(); debugLog().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); seqPhase.block(); // wait for them to become idle - mutex.unlock(); } + mutex.unlock(); inParallelPhase = false; } - private void drain(ThreadLocalBuffer buffer) { + private void drainBuffer() { Object obj; while ((obj = buffer.pop()) != null) { getVisitor().doVisitObject(obj); From 060bbd47366a8c19054ffcb4e611e5bd5833d266 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 16 Sep 2022 12:41:42 +0300 Subject: [PATCH 058/125] Use buffer for aligned chunks instead of objects --- .../core/genscavenge/AlignedHeapChunk.java | 2 +- .../genscavenge/GreyToBlackObjectVisitor.java | 10 ---- .../oracle/svm/core/genscavenge/Space.java | 19 +++---- .../genscavenge/parallel/ChunkBuffer.java | 24 ++++++--- .../genscavenge/parallel/ParallelGCImpl.java | 53 +++++++++++-------- 5 files changed, 55 insertions(+), 53 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 25f25d0a653d..09ccc58873d8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -147,7 +147,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") - static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { + public static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 0f725cc1cb92..870494eb4f1a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -59,16 +59,6 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") public boolean visitObjectInline(Object o) { - if (ParallelGC.isSupported() && !ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(o)); - return true; - } else { - return doVisitObject(o); - } - } - - @AlwaysInline("GC performance") - public boolean doVisitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index d81a25b28bc5..2021a340ad57 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -182,7 +182,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ - return allocateInNewChunk(objectSize); + return allocateInNewChunk(oldChunk, objectSize); } /** @@ -196,9 +196,12 @@ private Pointer freeMemory(UnsignedWord objectSize) { return AlignedHeapChunk.freeMemory(oldChunk, objectSize); } - private Pointer allocateInNewChunk(UnsignedWord objectSize) { + private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { if (ParallelGC.isSupported()) { ParallelGCImpl.mutex.lock(); + if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { + ParallelGCImpl.singleton().push(oldChunk); + } } AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); if (ParallelGC.isSupported()) { @@ -382,9 +385,6 @@ Object promoteAlignedObject(Object original, Space originalSpace) { Object copy = copyAlignedObject(original); if (copy != null) { ObjectHeaderImpl.installForwardingPointer(original, copy); - if (ParallelGC.isSupported()) { - ParallelGCImpl.singleton().push(Word.objectToUntrackedPointer(copy)); - } } return copy; } @@ -434,7 +434,6 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); } - ParallelGCImpl.singleton().pushToLocalBuffer(copyMemory); return copy; } else { // Retract speculatively allocated memory @@ -488,10 +487,8 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.singleton().push(chunk); ParallelGCImpl.mutex.unlock(); - if (!AlignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { - throw VMError.shouldNotReachHere(); - } } if (this.isOldSpace()) { @@ -514,10 +511,8 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { + assert false;/// ParallelGCImpl.mutex.unlock(); - if (!UnalignedHeapChunk.walkObjectsInline(chunk, GCImpl.getGCImpl().getGreyToBlackObjectVisitor())) { - throw VMError.shouldNotReachHere(); - } } if (this.isOldSpace()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 1da361a3609e..eccd42aebc23 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,37 +1,47 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.log.Log; -import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; /** * Synchronized buffer where chunks to be scanned are stored. */ public class ChunkBuffer { - private static final int SIZE = 1024 * 1024; ///handle overflow + reduce for chunks - private static final Pointer[] buffer = new Pointer[SIZE]; + private static final int SIZE = 10 * 1024; ///handle overflow + private static final HeapChunk.Header[] buffer = new HeapChunk.Header[SIZE]; private int top; private Log debugLog() { return ParallelGCImpl.debugLog(); } - void push(Pointer ptr) { + void push(HeapChunk.Header ptr) { ParallelGCImpl.mutex.lock(); assert top < SIZE; buffer[top++] = ptr; ParallelGCImpl.mutex.unlock(); } - Object pop() { + HeapChunk.Header pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - return buffer[--top].toObjectNonNull(); + return buffer[--top]; } else { - return null; + return WordFactory.nullPointer(); } } finally { ParallelGCImpl.mutex.unlock(); } } + + int size() { + ParallelGCImpl.mutex.lock(); + try { + return top; + } finally { + ParallelGCImpl.mutex.unlock(); + } + } } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 1d3148505bb4..b2c92634b1ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -6,6 +6,7 @@ import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; +import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; @@ -19,6 +20,7 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; @@ -40,8 +42,10 @@ public class ParallelGCImpl extends ParallelGC { /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ - private static final FastThreadLocalWord chunkTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.chunkTL"); + private static final FastThreadLocalWord allocChunkTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); + private static final FastThreadLocalWord scannedChunkTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); @@ -62,28 +66,19 @@ public static void waitForIdle() { singleton().waitForIdleImpl(); } - public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { - return chunkTL.get(); + public static AlignedHeapChunk.AlignedHeader getThreadLocalScannedChunk() { + return scannedChunkTL.get(); } - public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { - chunkTL.set(chunk); + public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + return allocChunkTL.get(); } - /** - * To be invoked in sequential GC phase. Pushes object pointer to one of the workers' buffers - * in round-robin fashion, attempting to balance load between workers. - */ - public void push(Pointer ptr) { - assert !isInParallelPhase(); - buffer.push(ptr); + public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + allocChunkTL.set(chunk); } - /** - * To be invoked in parallel GC phase. Pushes object pointer to current worker's thread local buffer. - */ - public void pushToLocalBuffer(Pointer ptr) { - assert isInParallelPhase(); + public void push(HeapChunk.Header ptr) { buffer.push(ptr); } @@ -173,11 +168,23 @@ private void waitForIdleImpl() { } private void drainBuffer() { - Object obj; - while ((obj = buffer.pop()) != null) { - getVisitor().doVisitObject(obj); - } - chunkTL.set(WordFactory.nullPointer()); + debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); + do { + HeapChunk.Header chunk; + while ((chunk = buffer.pop()).notEqual(WordFactory.nullPointer())) { + debugLog().string("WW drain chunk=").zhex(chunk).newline(); + AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) chunk, getVisitor()); + } + AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); + debugLog().string("WW drain tlab=").zhex(tlab).newline(); + if (tlab.notEqual(WordFactory.nullPointer())) { + scannedChunkTL.set(tlab); + AlignedHeapChunk.walkObjectsInline(tlab, getVisitor()); + } + } while (buffer.size() > 0 || allocChunkTL.get().notEqual(scannedChunkTL.get())); + + allocChunkTL.set(WordFactory.nullPointer()); + scannedChunkTL.set(WordFactory.nullPointer()); } private GreyToBlackObjectVisitor getVisitor() { From f0a26b72863b505cd66838c502b0aefe4568d880 Mon Sep 17 00:00:00 2001 From: peterz Date: Sat, 17 Sep 2022 16:34:07 +0300 Subject: [PATCH 059/125] Support for unaligned chunks --- .../com/oracle/svm/core/genscavenge/Space.java | 6 +++--- .../core/genscavenge/parallel/ChunkBuffer.java | 12 +++++++----- .../genscavenge/parallel/ParallelGCImpl.java | 16 +++++++++++----- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 2021a340ad57..0f265f52c7f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -200,7 +200,7 @@ private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, Unsi if (ParallelGC.isSupported()) { ParallelGCImpl.mutex.lock(); if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { - ParallelGCImpl.singleton().push(oldChunk); + ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } } AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); @@ -487,7 +487,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(chunk); + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); ParallelGCImpl.mutex.unlock(); } @@ -511,7 +511,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { - assert false;/// + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01)); ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index eccd42aebc23..69e6779d514e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -2,6 +2,7 @@ import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.log.Log; +import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; /** @@ -9,21 +10,22 @@ */ public class ChunkBuffer { private static final int SIZE = 10 * 1024; ///handle overflow - private static final HeapChunk.Header[] buffer = new HeapChunk.Header[SIZE]; + private static final Pointer[] buffer = new Pointer[SIZE]; private int top; private Log debugLog() { return ParallelGCImpl.debugLog(); } - void push(HeapChunk.Header ptr) { - ParallelGCImpl.mutex.lock(); + /** + * This method must be called with ParallelGCImpl.mutex locked + */ + void push(Pointer ptr) { assert top < SIZE; buffer[top++] = ptr; - ParallelGCImpl.mutex.unlock(); } - HeapChunk.Header pop() { + Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index b2c92634b1ed..d5c98447af63 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -7,6 +7,7 @@ import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; +import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; @@ -78,7 +79,7 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { allocChunkTL.set(chunk); } - public void push(HeapChunk.Header ptr) { + public void push(Pointer ptr) { buffer.push(ptr); } @@ -170,10 +171,15 @@ private void waitForIdleImpl() { private void drainBuffer() { debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); do { - HeapChunk.Header chunk; - while ((chunk = buffer.pop()).notEqual(WordFactory.nullPointer())) { - debugLog().string("WW drain chunk=").zhex(chunk).newline(); - AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) chunk, getVisitor()); + Pointer ptr; + while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { + debugLog().string("WW drain chunk=").zhex(ptr).newline(); + if (ptr.and(0x01).notEqual(0)) { + // unaligned chunk + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~0x01), getVisitor()); + } else { + AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); + } } AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); debugLog().string("WW drain tlab=").zhex(tlab).newline(); From ec5dd99ad7793c042879f96bce1b8804e55fcae1 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 19 Sep 2022 13:27:06 +0300 Subject: [PATCH 060/125] HyperAlloc works but load is badly unbalanced --- .../com/oracle/svm/core/genscavenge/Space.java | 2 +- .../core/genscavenge/parallel/ChunkBuffer.java | 3 +-- .../genscavenge/parallel/ParallelGCImpl.java | 16 ++++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 0f265f52c7f5..651da2242633 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -511,7 +511,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01)); + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01));///magic ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 69e6779d514e..c37fa9042790 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,12 +1,11 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.log.Log; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; /** - * Synchronized buffer where chunks to be scanned are stored. + * Synchronized buffer that stores "grey" heap chunks to be scanned. */ public class ChunkBuffer { private static final int SIZE = 10 * 1024; ///handle overflow diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index d5c98447af63..2431bbb5bff6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -170,11 +170,11 @@ private void waitForIdleImpl() { private void drainBuffer() { debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); - do { + while (true) { Pointer ptr; while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { debugLog().string("WW drain chunk=").zhex(ptr).newline(); - if (ptr.and(0x01).notEqual(0)) { + if (ptr.and(0x01).notEqual(0)) {///magic // unaligned chunk UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~0x01), getVisitor()); } else { @@ -183,13 +183,17 @@ private void drainBuffer() { } AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); debugLog().string("WW drain tlab=").zhex(tlab).newline(); - if (tlab.notEqual(WordFactory.nullPointer())) { + if (tlab.equal(WordFactory.nullPointer())) { + break; + } else { scannedChunkTL.set(tlab); AlignedHeapChunk.walkObjectsInline(tlab, getVisitor()); + if (allocChunkTL.get().equal(tlab)) { + // this tlab is now black, retire it + allocChunkTL.set(WordFactory.nullPointer()); + } } - } while (buffer.size() > 0 || allocChunkTL.get().notEqual(scannedChunkTL.get())); - - allocChunkTL.set(WordFactory.nullPointer()); + } scannedChunkTL.set(WordFactory.nullPointer()); } From d59cb3646c73f81846c05dc90c7f8c2a6a676fd6 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 19 Sep 2022 18:02:50 +0300 Subject: [PATCH 061/125] Report stats for relevant threads only --- .../com/oracle/svm/core/genscavenge/parallel/Stats.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java index a08cbd0c1ae7..6a8a1dcd9bf5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java @@ -71,9 +71,10 @@ public void print(Log log) { totalStats.reset(); log.string("PGC stats:").newline(); for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - StatsImpl stats = statsTL.get(vmThread); - if (stats != null) { - Thread jt = PlatformThreads.fromVMThread(vmThread); + Thread jt = PlatformThreads.fromVMThread(vmThread); + String jtName = jt.getName(); + if (jt == Thread.currentThread() || jtName != null && jtName.startsWith("ParallelGCWorker-")) { + StatsImpl stats = statsTL.get(vmThread); log.string(" ").string(jt.getName()).string(":").newline(); totalStats.objectVisitTime += stats.objectVisitTime; totalStats.allocTime += stats.allocTime; From c4c4b34768959490d94096fd10746f6087aed807 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 13:17:50 +0300 Subject: [PATCH 062/125] Fixed SerialGC compilation --- .../src/com/oracle/svm/core/genscavenge/Space.java | 13 +++++++------ .../core/genscavenge/parallel/ParallelGCImpl.java | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 651da2242633..88f7800d4fa0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -190,7 +190,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { */ @AlwaysInline("GC performance") private Pointer freeMemory(UnsignedWord objectSize) { - assert ParallelGCImpl.isInParallelPhase(); + assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.freeMemory(oldChunk, objectSize); @@ -378,7 +378,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -392,6 +392,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); + assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); @@ -481,12 +482,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); ParallelGCImpl.mutex.unlock(); } @@ -505,12 +506,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01));///magic ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 2431bbb5bff6..a19fa1dfd294 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -60,7 +60,7 @@ public static ParallelGCImpl singleton() { } public static boolean isInParallelPhase() { - return isSupported() && singleton().inParallelPhase; + return singleton().inParallelPhase; } public static void waitForIdle() { From c061049b0825d7974e94c622de5d14a5d01ccc8f Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 16:18:06 +0300 Subject: [PATCH 063/125] Cleanup --- .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 3 --- .../src/com/oracle/svm/core/genscavenge/Space.java | 2 +- .../svm/core/genscavenge/parallel/ParallelGCImpl.java | 10 +++++----- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index b66b85a79047..95b27b51c1e4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -370,9 +370,6 @@ private void printGCBefore(String cause) { verboseGCLog.string(" MinimumHeapSize: ").unsigned(getPolicy().getMinimumHeapSize()).newline(); verboseGCLog.string(" AlignedChunkSize: ").unsigned(HeapParameters.getAlignedHeapChunkSize()).newline(); verboseGCLog.string(" LargeArrayThreshold: ").unsigned(HeapParameters.getLargeArrayThreshold()).string("]").newline(); - boolean compressed = ReferenceAccess.singleton().haveCompressedReferences();///rm - boolean hasShift = ReferenceAccess.singleton().getCompressEncoding().hasShift(); - verboseGCLog.string(" has compressed refs: ").bool(compressed).string(", has shift: ").bool(hasShift).newline(); if (SerialGCOptions.PrintHeapShape.getValue()) { HeapImpl.getHeapImpl().logImageHeapPartitionBoundaries(verboseGCLog); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 88f7800d4fa0..266f5e6b447c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -512,7 +512,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(0x01));///magic + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); ParallelGCImpl.mutex.unlock(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index a19fa1dfd294..029079c72798 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -28,6 +28,7 @@ public class ParallelGCImpl extends ParallelGC { + public static final int UNALIGNED_BIT = 0x01; private static final int MAX_WORKERS_COUNT = 31; /** @@ -88,7 +89,7 @@ public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - workerCount = getWorkerCount(); ///cmp serialGC vs pargc with 1 worker + workerCount = getWorkerCount(); allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); for (int i = 1; i < workerCount; i++) { // We reuse the gc thread for worker #0 @@ -174,9 +175,8 @@ private void drainBuffer() { Pointer ptr; while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { debugLog().string("WW drain chunk=").zhex(ptr).newline(); - if (ptr.and(0x01).notEqual(0)) {///magic - // unaligned chunk - UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~0x01), getVisitor()); + if (ptr.and(UNALIGNED_BIT).notEqual(0)) { + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); } @@ -209,7 +209,7 @@ private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } - static Log debugLog() {///rm + static Log debugLog() { return Log.noopLog(); } } From 1dbd6c104ef8b103985620f2cb9abcfd5e647eb3 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 17:14:03 +0300 Subject: [PATCH 064/125] Determine buffer size at runtime --- .../genscavenge/parallel/ChunkBuffer.java | 30 ++++++++----------- .../genscavenge/parallel/ParallelGCImpl.java | 16 ++++++---- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index c37fa9042790..72480544e776 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,6 +1,6 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.log.Log; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -8,27 +8,30 @@ * Synchronized buffer that stores "grey" heap chunks to be scanned. */ public class ChunkBuffer { - private static final int SIZE = 10 * 1024; ///handle overflow - private static final Pointer[] buffer = new Pointer[SIZE]; - private int top; + private Pointer buffer; + private int size, top, step; - private Log debugLog() { - return ParallelGCImpl.debugLog(); + ChunkBuffer(int maxChunks, int refSize) { + this.step = refSize; + this.size = maxChunks * refSize; + this.buffer = UnmanagedMemory.malloc(this.size); } /** * This method must be called with ParallelGCImpl.mutex locked */ void push(Pointer ptr) { - assert top < SIZE; - buffer[top++] = ptr; + assert top < size; + buffer.writeWord(top, ptr); + top += step; } Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - return buffer[--top]; + top -= step; + return buffer.readWord(top); } else { return WordFactory.nullPointer(); } @@ -36,13 +39,4 @@ Pointer pop() { ParallelGCImpl.mutex.unlock(); } } - - int size() { - ParallelGCImpl.mutex.lock(); - try { - return top; - } finally { - ParallelGCImpl.mutex.unlock(); - } - } } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 029079c72798..844f9945328c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -6,7 +6,7 @@ import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; -import com.oracle.svm.core.genscavenge.HeapChunk; +import com.oracle.svm.core.genscavenge.HeapParameters; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; @@ -21,7 +21,6 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; -import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; import java.util.concurrent.atomic.AtomicInteger; @@ -39,8 +38,6 @@ public class ParallelGCImpl extends ParallelGC { private int allWorkersBusy; private final AtomicInteger busyWorkers = new AtomicInteger(0); - private static final ChunkBuffer buffer = new ChunkBuffer(); - /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ @@ -53,6 +50,7 @@ public class ParallelGCImpl extends ParallelGC { private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); + private ChunkBuffer buffer; private volatile boolean inParallelPhase; @Fold @@ -89,10 +87,17 @@ public void startWorkerThreadsImpl() { int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); VMError.guarantee(hubOffset == 0, "hub offset must be 0"); + // Allocate buffer large enough to store maximum possible number of heap chunks + long maxHeapSize = GCImpl.getPolicy().getMaximumHeapSize().rawValue(); + long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); + long unalignedChunkSize = HeapParameters.getLargeArrayThreshold().rawValue(); + long maxChunks = maxHeapSize / Math.min(alignedChunkSize, unalignedChunkSize) + 1; + buffer = new ChunkBuffer((int) maxChunks, ConfigurationValues.getObjectLayout().getReferenceSize()); + workerCount = getWorkerCount(); allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); + // We reuse the gc thread for worker #0 for (int i = 1; i < workerCount; i++) { - // We reuse the gc thread for worker #0 startWorkerThread(i); } } @@ -170,7 +175,6 @@ private void waitForIdleImpl() { } private void drainBuffer() { - debugLog().string("WW drain size=").unsigned(buffer.size()).newline(); while (true) { Pointer ptr; while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { From ca5edacade095b3e5df87ac5e9154b61ece62893 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 3 Oct 2022 17:19:46 +0300 Subject: [PATCH 065/125] Removed Stats --- .../genscavenge/parallel/ParallelGCImpl.java | 5 - .../svm/core/genscavenge/parallel/Stats.java | 205 ------------------ .../parallel/ThreadLocalBuffer.java | 61 ------ 3 files changed, 271 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 844f9945328c..bcd7c615cafc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -121,7 +121,6 @@ private void startWorkerThread(int n) { VMThreads.SafepointBehavior.markThreadAsCrashed(); debugLog().string("WW start ").unsigned(n).newline(); - getStats().install(); while (true) { try { do { @@ -205,10 +204,6 @@ private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } - public static Stats getStats() { - return Stats.stats(); - } - private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java deleted file mode 100644 index 6a8a1dcd9bf5..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/Stats.java +++ /dev/null @@ -1,205 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.thread.PlatformThreads; -import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; -import com.oracle.svm.core.threadlocal.FastThreadLocalObject; -import org.graalvm.nativeimage.IsolateThread; - -public abstract class Stats { - private static final Stats realStats = new RealStats(); - private static final Stats noopStats = new NoopStats(); - - static Stats stats() { - return realStats; - } - - static Stats noopStats() { - return noopStats; - } - - public void reset() {} - public void print(Log log) {} - void install() {} - public void noteAllocTime(long t) {} - public void noteForwardInstallTime(long t) {} - public void noteObjectVisitTime(long t) {} - public void noteQueueCopyTime(long t) {} - public void noteQueueScanTime(long t) {} - public void noteLostObject() {} - void notePut(int putIndex, int getIndex, int capacity) {} - void noteGet() {} -} - -class NoopStats extends Stats { -} - -class RealStats extends Stats { - private static final FastThreadLocalObject statsTL = - FastThreadLocalFactory.createObject(StatsImpl.class, "ParGC.Stats"); - - private final StatsImpl defaultStats = new StatsImpl(); - private final StatsImpl totalStats = new StatsImpl(); - - private StatsImpl impl() { - StatsImpl stats = statsTL.get(); - if (stats == null) { - stats = defaultStats; - statsTL.set(stats); - } - return stats; - } - - @Override - void install() { - statsTL.set(new StatsImpl()); - } - - @Override - public void reset() { - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - StatsImpl stats = statsTL.get(vmThread); - if (stats != null) { - stats.reset(); - } - } - } - - @Override - public void print(Log log) { - totalStats.reset(); - log.string("PGC stats:").newline(); - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - Thread jt = PlatformThreads.fromVMThread(vmThread); - String jtName = jt.getName(); - if (jt == Thread.currentThread() || jtName != null && jtName.startsWith("ParallelGCWorker-")) { - StatsImpl stats = statsTL.get(vmThread); - log.string(" ").string(jt.getName()).string(":").newline(); - totalStats.objectVisitTime += stats.objectVisitTime; - totalStats.allocTime += stats.allocTime; - totalStats.forwardInstallTime += stats.forwardInstallTime; - totalStats.queueScanTime += stats.queueScanTime; - totalStats.queueCopyTime += stats.queueCopyTime; - totalStats.lostObjectCount += stats.lostObjectCount; - totalStats.putCount += stats.putCount; - totalStats.getCount += stats.getCount; - totalStats.maxQueueSize = Math.max(totalStats.maxQueueSize, stats.maxQueueSize); - stats.print(log); - } - } - log.string(" total:").newline(); - totalStats.print(log); - } - - @Override - public void noteAllocTime(long t) { - impl().noteAllocTime(t); - } - - @Override - public void noteForwardInstallTime(long t) { - impl().noteForwardInstallTime(t); - } - - @Override - public void noteObjectVisitTime(long t) { - impl().noteObjectVisitTime(t); - } - - @Override - public void noteQueueCopyTime(long t) { - impl().noteQueueCopyTime(t); - } - - @Override - public void noteQueueScanTime(long t) { - impl().noteQueueScanTime(t); - } - - @Override - public void noteLostObject() { - impl().noteLostObject(); - } - - @Override - void notePut(int putIndex, int getIndex, int capacity) { - impl().notePut(putIndex, getIndex, capacity); - } - - @Override - public void noteGet() { - impl().noteGet(); - } -} - -class StatsImpl extends Stats { - int allocTime, forwardInstallTime, objectVisitTime, queueCopyTime, queueScanTime; - int putCount, getCount, lostObjectCount, maxQueueSize; - - @Override - public void reset() { - allocTime = forwardInstallTime = objectVisitTime = queueCopyTime = queueScanTime = 0; - putCount = getCount = lostObjectCount = maxQueueSize = 0; - } - - @Override - public void print(Log log) { - log.string(" visit ").unsigned(objectVisitTime) - .string(" alloc ").unsigned(allocTime) - .string(" fwptr ").unsigned(forwardInstallTime) - .string(" qscan ").unsigned(queueScanTime) - .string(" qcopy ").unsigned(queueCopyTime) - .string(" lost ").unsigned(lostObjectCount).newline(); - log.string(" put ").unsigned(putCount) - .string(" get ").unsigned(getCount) - .string(" max ").unsigned(maxQueueSize).newline(); - } - - @Override - public void noteAllocTime(long t) { - allocTime += t; - } - - @Override - public void noteForwardInstallTime(long t) { - forwardInstallTime += t; - } - - @Override - public void noteObjectVisitTime(long t) { - objectVisitTime += t; - } - - @Override - public void noteQueueCopyTime(long t) { - queueCopyTime += t; - } - - @Override - public void noteQueueScanTime(long t) { - queueScanTime += t; - } - - @Override - public void noteLostObject() { - lostObjectCount++; - } - - @Override - void notePut(int putIndex, int getIndex, int capacity) { - int size = putIndex - getIndex; - if (size < 0) { - size += capacity; - } - if (size > maxQueueSize) { - maxQueueSize = size; - } - putCount++; - } - - @Override - void noteGet() { - getCount++; - } -} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java deleted file mode 100644 index 1dc025567def..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ThreadLocalBuffer.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.heap.OutOfMemoryUtil; -import com.oracle.svm.core.log.Log; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; - -/** - * Thread local (non MT safe) buffer where pointers to grey objects are stored. - */ -public class ThreadLocalBuffer { - private static final int INITIAL_SIZE = 128 * 1024 * 8; // 128k words - private static final OutOfMemoryError OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate a ThreadLocalBuffer"); - - private Pointer buffer = WordFactory.nullPointer(); - private UnsignedWord size = WordFactory.unsigned(INITIAL_SIZE); - private UnsignedWord top = WordFactory.zero(); - - private Log log() { - return Log.log(); - } - - void runtimeInit() { - buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); - ensureBuffer(); - } - - private void ensureCapacity() { - if (top.aboveOrEqual(size)) { - size = size.multiply(2); - buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, size); - ensureBuffer(); - log().string("TLB grow to ").unsigned(size).newline(); - } - } - - private void ensureBuffer() { - if (buffer.isNull()) { - throw OutOfMemoryUtil.reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); - } - } - - void push(Pointer ptr) { - ensureCapacity(); - buffer.writeWord(top, ptr); - top = top.add(8); - } - - Object pop() { - if (top.aboveThan(0)) { - top = top.subtract(8); - Pointer ptr = buffer.readWord(top); - return ptr.toObjectNonNull(); - } else { - return null; - } - } -} \ No newline at end of file From 757075ffbc4c59961c853d156f61cc9cba8c441c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 4 Oct 2022 19:07:14 +0300 Subject: [PATCH 066/125] Cleanup --- .../oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 870494eb4f1a..88adc11c4a0b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -24,14 +24,11 @@ */ package com.oracle.svm.core.genscavenge; -import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.annotate.AlwaysInline; import com.oracle.svm.core.annotate.NeverInline; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; From 01627fce89cedc3c6dc43dba303cc94ca6c427d1 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 2 Nov 2022 10:47:20 +0600 Subject: [PATCH 067/125] Fixed crash due to unscanned chunks HyperAlloc works for hours with heap verification --- .../src/com/oracle/svm/core/genscavenge/Space.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 266f5e6b447c..ffe6a8e7fe85 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -487,9 +487,11 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); - ParallelGCImpl.mutex.unlock(); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } if (this.isOldSpace()) { @@ -511,9 +513,11 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isSupported()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); - ParallelGCImpl.mutex.unlock(); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } if (this.isOldSpace()) { From 80df70ec10a8e89952e1808818322e58dbad6274 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 3 Nov 2022 14:37:39 +0600 Subject: [PATCH 068/125] Added copyright notices --- .../genscavenge/parallel/ChunkBuffer.java | 26 +++++++++++++++++++ .../genscavenge/parallel/ParallelGCImpl.java | 26 +++++++++++++++++++ .../parallel/ParallelGCOptions.java | 26 +++++++++++++++++++ .../com/oracle/svm/core/heap/ParallelGC.java | 26 +++++++++++++++++++ 4 files changed, 104 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 72480544e776..205bed8a1d3a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.genscavenge.parallel; import org.graalvm.nativeimage.UnmanagedMemory; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index bcd7c615cafc..15b1d660f15d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java index 208f06fc633e..05002102799f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.option.RuntimeOptionKey; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index 3bd3cf42b34a..aa1af21c497a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,3 +1,29 @@ +/* + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateOptions; From 2a540d303e5c99b3ce958f993c72f08407144b80 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 10 Nov 2022 15:39:20 +0300 Subject: [PATCH 069/125] Fixed merge errors --- .../core/genscavenge/parallel/ParallelGCImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 15b1d660f15d..e6e4f86fbd91 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -27,8 +27,9 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; -import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -45,7 +46,8 @@ import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -239,8 +241,10 @@ static Log debugLog() { } } -@AutomaticFeature -class ParallelGCFeature implements Feature { +@Platforms(Platform.HOSTED_ONLY.class) +@AutomaticallyRegisteredFeature() +@SuppressWarnings("unused") +class ParallelGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { From f228f74322a147d35d8df1fc3c38d42bbe31311b Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 10 Nov 2022 16:31:41 +0300 Subject: [PATCH 070/125] Fixed Substrate options to work with ParallelGC --- .../core/genscavenge/JfrGCEventSupport.java | 2 +- .../SerialAndEpsilonGCOptions.java | 22 +++++++++---------- .../genscavenge/graal/BarrierSnippets.java | 2 +- .../graal/GenScavengeGCFeature.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 5 +++++ .../heap/Target_java_lang_ref_Reference.java | 6 ++--- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 5c813ee3c3fc..5270d0282244 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -141,7 +141,7 @@ private int popPhase() { class JfrGCEventFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue(); + return SubstrateOptions.useSerialOrParallelGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index e3179517289d..bc4cfb993692 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -40,13 +40,13 @@ /** Common options that can be specified for both the serial and the epsilon GC. */ public final class SerialAndEpsilonGCOptions { @Option(help = "The maximum heap size as percent of physical memory. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "The maximum size of the young generation as a percentage of the maximum heap size. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "The size of an aligned chunk. Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly) { + public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly) { @Override protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { int multiple = 4096; @@ -59,28 +59,28 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV * fit in an aligned chunk. */ @Option(help = "The size at or above which an array will be allocated in its own unaligned chunk. 0 implies (AlignedHeapChunkSize / 8). Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Fill unused memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Before use, fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "After use, Fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Bytes that can be allocated before (re-)querying the physical memory size. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly); @Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::serialOrEpsilonGCOnly); + public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::graalCeGCOnly); private SerialAndEpsilonGCOptions() { } - public static void serialOrEpsilonGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseSerialGC.getValue() && !SubstrateOptions.UseEpsilonGC.getValue()) { + private static void graalCeGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useGraalCeGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index 247da5e8440e..69adc25a7743 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -204,7 +204,7 @@ class BarrierSnippetCounters { class BarrierSnippetCountersFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.UseSerialGC.getValue() && SubstrateOptions.useRememberedSet(); + return SubstrateOptions.useSerialOrParallelGC() && SubstrateOptions.useRememberedSet(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index e311f8b0ccdf..f44e3ec53a28 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -155,7 +155,7 @@ private static RememberedSet createRememberedSet() { } private static PerfDataHolder createPerfData() { - if (SubstrateOptions.UseSerialGC.getValue()) { + if (SubstrateOptions.useSerialOrParallelGC()) { return new SerialGCPerfData(); } else { assert SubstrateOptions.UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 3fd681029fd8..91bebd8810c5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -338,6 +338,11 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }; + @Fold + public static boolean useSerialOrParallelGC() { + return UseSerialGC.getValue() || UseParallelGC.getValue(); + } + @Fold public static boolean useGraalCeGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index de86cf928f8c..3c2cf2e3ca6a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -95,7 +95,7 @@ public final class Target_java_lang_ref_Reference { @SuppressWarnings("unused") // @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) // - @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotSerialNotEpsilonGC.class) // + @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotGraalCeGC.class) // transient Target_java_lang_ref_Reference discovered; @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Custom, declClass = ComputeQueueValue.class) // @@ -229,9 +229,9 @@ public Object transform(Object receiver, Object originalValue) { } @Platforms(Platform.HOSTED_ONLY.class) -class NotSerialNotEpsilonGC implements BooleanSupplier { +class NotGraalCeGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.UseSerialGC.getValue() && !SubstrateOptions.UseEpsilonGC.getValue(); + return !SubstrateOptions.useGraalCeGC(); } } From 3cbe52e21055ca83835d9f9c58fcc31fe21d2732 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 Nov 2022 16:00:42 +0300 Subject: [PATCH 071/125] Post-review renaming and cleanup --- .../core/genscavenge/AlignedHeapChunk.java | 4 +-- .../oracle/svm/core/genscavenge/GCImpl.java | 3 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 2 +- .../core/genscavenge/PinnedObjectImpl.java | 2 +- .../SerialAndEpsilonGCOptions.java | 22 ++++++------- .../oracle/svm/core/genscavenge/Space.java | 31 +++++++++---------- .../core/genscavenge/UnalignedHeapChunk.java | 2 +- ...GC.java => UseMarkAndCopyOrEpsilonGC.java} | 4 +-- .../graal/GenScavengeGCFeature.java | 3 +- .../genscavenge/parallel/ChunkBuffer.java | 19 ++++++------ .../genscavenge/parallel/ParallelGCImpl.java | 22 ++++++------- .../parallel/ParallelGCOptions.java | 6 ++-- .../com/oracle/svm/core/SubstrateOptions.java | 19 +++++++++--- .../com/oracle/svm/core/heap/ParallelGC.java | 6 ++-- .../heap/Target_java_lang_ref_Reference.java | 6 ++-- .../oracle/svm/core/hub/LayoutEncoding.java | 1 + 16 files changed, 82 insertions(+), 70 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/{UseGraalCeGC.java => UseMarkAndCopyOrEpsilonGC.java} (92%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 0150e3c5add0..ba0180117ac8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -113,7 +113,7 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { } /** Retract the latest allocation. Used by parallel collector. */ - static Pointer freeMemory(AlignedHeader that, UnsignedWord size) { + static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); assert newTop.aboveThan(HeapChunk.asPointer(that)); HeapChunk.setTopPointer(that, newTop); @@ -160,7 +160,7 @@ static MemoryWalker.HeapChunkAccess getMemoryWal } /** Methods for a {@link MemoryWalker} to access an aligned heap chunk. */ - @AutomaticallyRegisteredImageSingleton(onlyWith = UseGraalCeGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index d7fee4f8a074..dbc9ea720977 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -74,7 +74,6 @@ import com.oracle.svm.core.heap.NoAllocationVerifier; import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.heap.ParallelGC; -import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceMapIndex; import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner; @@ -1043,7 +1042,7 @@ private static void prepareForPromotion(boolean isIncremental) { private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { ParallelGCImpl.waitForIdle(); } else if (isIncremental) { scanGreyObjectsLoop(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index fd7c91f59c27..29617435e89f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -878,7 +878,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } -@TargetClass(value = java.lang.Runtime.class, onlyWith = UseGraalCeGC.class) +@TargetClass(value = java.lang.Runtime.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) @SuppressWarnings("static-method") final class Target_java_lang_Runtime { @Substitute diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index ab349ead749b..a5686c28b8f5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -42,7 +42,7 @@ /** Support for pinning objects to a memory address with {@link PinnedObject}. */ final class PinnedObjectImpl implements PinnedObject { - @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseGraalCeGC.class) + @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) static class PinnedObjectSupportImpl implements PinnedObjectSupport { @Override public PinnedObject create(Object object) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index bc4cfb993692..e6b76ab52316 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -40,13 +40,13 @@ /** Common options that can be specified for both the serial and the epsilon GC. */ public final class SerialAndEpsilonGCOptions { @Option(help = "The maximum heap size as percent of physical memory. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "The maximum size of the young generation as a percentage of the maximum heap size. Serial and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "The size of an aligned chunk. Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly) { + public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly) { @Override protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { int multiple = 4096; @@ -59,28 +59,28 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV * fit in an aligned chunk. */ @Option(help = "The size at or above which an array will be allocated in its own unaligned chunk. 0 implies (AlignedHeapChunkSize / 8). Serial and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Fill unused memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Before use, fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "After use, Fill memory chunks with a sentinel value. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Bytes that can be allocated before (re-)querying the physical memory size. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); @Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::graalCeGCOnly); + public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); private SerialAndEpsilonGCOptions() { } - private static void graalCeGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.useGraalCeGC()) { + private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.UseMarkAndCopyOrEpsilonGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ce980f706f76..bdc75c7fef3f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -48,7 +48,6 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.util.VMError; /** * A Space is a collection of HeapChunks. @@ -170,7 +169,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk; - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { oldChunk = ParallelGCImpl.getThreadLocalChunk(); } else { oldChunk = getLastAlignedHeapChunk(); @@ -189,22 +188,22 @@ private Pointer allocateMemory(UnsignedWord objectSize) { * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") - private Pointer freeMemory(UnsignedWord objectSize) { - assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + private Pointer retractAllocation(UnsignedWord objectSize) { + assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); assert oldChunk.isNonNull(); - return AlignedHeapChunk.freeMemory(oldChunk, objectSize); + return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { ParallelGCImpl.mutex.lock(); if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } } AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); - if (ParallelGC.isSupported()) { + if (ParallelGC.isEnabled()) { ParallelGCImpl.mutex.unlock(); ParallelGCImpl.setThreadLocalChunk(newChunk); } @@ -231,7 +230,7 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); } appendAlignedHeapChunkUninterruptibly(aChunk); @@ -284,7 +283,7 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. */ - if (SubstrateOptions.MultiThreaded.getValue() && !SubstrateOptions.UseParallelGC.getValue()) { + if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); } appendUnalignedHeapChunkUninterruptibly(uChunk); @@ -378,7 +377,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { return promoteAlignedObjectParallel(original, originalSpace); } @@ -392,7 +391,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { @AlwaysInline("GC performance") Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert VMOperation.isGCInProgress(); - assert ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase(); + assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); @@ -438,7 +437,7 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { return copy; } else { // Retract speculatively allocated memory - freeMemory(size); + retractAllocation(size); return forward; } } @@ -482,12 +481,12 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported()) { + if (ParallelGCImpl.isEnabled()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); @@ -508,12 +507,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isSupported() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isSupported()) { + if (ParallelGCImpl.isEnabled()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 16bb628bc1b8..6f5f1956cf1e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -162,7 +162,7 @@ public static MemoryWalker.HeapChunkAccess g return ImageSingletons.lookup(UnalignedHeapChunk.MemoryWalkerAccessImpl.class); } - @AutomaticallyRegisteredImageSingleton(onlyWith = UseGraalCeGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java similarity index 92% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java index 8da5cca783c0..9e824c594140 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseGraalCeGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java @@ -32,9 +32,9 @@ import com.oracle.svm.core.SubstrateOptions; @Platforms(Platform.HOSTED_ONLY.class) -public class UseGraalCeGC implements BooleanSupplier { +public class UseMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.useGraalCeGC(); + return SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index f44e3ec53a28..e9efa8939ccf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; +import com.oracle.svm.core.genscavenge.UseMarkAndCopyOrEpsilonGC; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; @@ -69,7 +70,7 @@ class GenScavengeGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return new com.oracle.svm.core.genscavenge.UseGraalCeGC().getAsBoolean(); + return new UseMarkAndCopyOrEpsilonGC().getAsBoolean(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 205bed8a1d3a..b1bb86500d86 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,8 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.config.ConfigurationValues; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -35,28 +37,25 @@ */ public class ChunkBuffer { private Pointer buffer; - private int size, top, step; + private int size, top; - ChunkBuffer(int maxChunks, int refSize) { - this.step = refSize; - this.size = maxChunks * refSize; + ChunkBuffer(int maxChunks) { + this.size = maxChunks * ConfigurationValues.getTarget().wordSize; this.buffer = UnmanagedMemory.malloc(this.size); } - /** - * This method must be called with ParallelGCImpl.mutex locked - */ void push(Pointer ptr) { + ParallelGCImpl.mutex.assertIsOwner("Must hold ParallelGCImpl.mutex"); assert top < size; buffer.writeWord(top, ptr); - top += step; + top += ConfigurationValues.getTarget().wordSize; } Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - top -= step; + top -= ConfigurationValues.getTarget().wordSize; return buffer.readWord(top); } else { return WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index e6e4f86fbd91..bd2464d09c2a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -120,7 +120,7 @@ public void startWorkerThreadsImpl() { long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); long unalignedChunkSize = HeapParameters.getLargeArrayThreshold().rawValue(); long maxChunks = maxHeapSize / Math.min(alignedChunkSize, unalignedChunkSize) + 1; - buffer = new ChunkBuffer((int) maxChunks, ConfigurationValues.getObjectLayout().getReferenceSize()); + buffer = new ChunkBuffer((int) maxChunks); workerCount = getWorkerCount(); allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); @@ -131,7 +131,7 @@ public void startWorkerThreadsImpl() { } private int getWorkerCount() { - int setting = ParallelGCOptions.ParallelGCWorkers.getValue(); + int setting = ParallelGCOptions.ParallelGCThreads.getValue(); int workers = setting > 0 ? setting : getDefaultWorkerCount(); workers = Math.min(workers, MAX_WORKERS_COUNT); verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); @@ -212,15 +212,15 @@ private void drainBuffer() { AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); } } - AlignedHeapChunk.AlignedHeader tlab = allocChunkTL.get(); - debugLog().string("WW drain tlab=").zhex(tlab).newline(); - if (tlab.equal(WordFactory.nullPointer())) { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + debugLog().string("WW drain allocChunk=").zhex(allocChunk).newline(); + if (allocChunk.equal(WordFactory.nullPointer())) { break; } else { - scannedChunkTL.set(tlab); - AlignedHeapChunk.walkObjectsInline(tlab, getVisitor()); - if (allocChunkTL.get().equal(tlab)) { - // this tlab is now black, retire it + scannedChunkTL.set(allocChunk); + AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); + if (allocChunkTL.get().equal(allocChunk)) { + // this allocation chunk is now black, retire it allocChunkTL.set(WordFactory.nullPointer()); } } @@ -248,7 +248,7 @@ class ParallelGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return ParallelGC.isSupported(); + return ParallelGC.isEnabled(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java index 05002102799f..6fe4b04d6ae9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,8 +30,10 @@ import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionType; +import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.Immutable; + public class ParallelGCOptions { @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User) - public static final RuntimeOptionKey ParallelGCWorkers = new RuntimeOptionKey<>(0); + public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 91bebd8810c5..556b2fe13a46 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -37,8 +37,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.Set; import java.util.function.Predicate; +import com.oracle.svm.core.util.InterruptImageBuilding; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -326,25 +328,34 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @APIOption(name = "parallel", group = GCGroup.class, customHelp = "Parallel garbage collector")// @Option(help = "Use a parallel GC")// - public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false) { + public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false, SubstrateOptions::validateParallelGC) { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { SubstrateOptions.UseSerialGC.update(values, false); SubstrateOptions.UseEpsilonGC.update(values, false); - ConcealedOptions.UseRememberedSet.update(values, false); - SubstrateOptions.MultiThreaded.update(values, true); } } }; + private static void validateParallelGC(OptionKey unused) { + validateParallelGC(SubstrateOptions.MultiThreaded, true); + validateParallelGC(ConcealedOptions.UseRememberedSet, false); + } + + private static void validateParallelGC(HostedOptionKey optionKey, boolean expected) { + if (optionKey.getValue() != expected) { + throw UserError.abort("ParallelGC requires the option %s to be %s", optionKey, expected); + } + } + @Fold public static boolean useSerialOrParallelGC() { return UseSerialGC.getValue() || UseParallelGC.getValue(); } @Fold - public static boolean useGraalCeGC() { + public static boolean UseMarkAndCopyOrEpsilonGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java index aa1af21c497a..95d8de510c9f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,12 +33,12 @@ public abstract class ParallelGC { @Fold - public static boolean isSupported() { + public static boolean isEnabled() { return SubstrateOptions.UseParallelGC.getValue(); } public static void startWorkerThreads() { - if (isSupported()) { + if (isEnabled()) { ImageSingletons.lookup(ParallelGC.class).startWorkerThreadsImpl(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index 3c2cf2e3ca6a..2a3fdd519d37 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -95,7 +95,7 @@ public final class Target_java_lang_ref_Reference { @SuppressWarnings("unused") // @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) // - @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotGraalCeGC.class) // + @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotMarkAndCopyOrEpsilonGC.class) // transient Target_java_lang_ref_Reference discovered; @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Custom, declClass = ComputeQueueValue.class) // @@ -229,9 +229,9 @@ public Object transform(Object receiver, Object originalValue) { } @Platforms(Platform.HOSTED_ONLY.class) -class NotGraalCeGC implements BooleanSupplier { +class NotMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.useGraalCeGC(); + return !SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index a60463fdcdbb..b290969c630a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -274,6 +274,7 @@ public static UnsignedWord getSizeFromObjectInline(Object obj) { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromEncoding(Object obj, int encoding) { + assert encoding == KnownIntrinsics.readHub(obj).getLayoutEncoding(); if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj)); } else { From c8ae8a3b317da18b6977f31726010bf0cc4912ba Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 22 Nov 2022 15:39:42 +0300 Subject: [PATCH 072/125] Review: forwarding header installation was not atomic enough --- .../core/genscavenge/ObjectHeaderImpl.java | 13 +++---------- .../oracle/svm/core/genscavenge/Space.java | 19 +++++++++---------- .../genscavenge/parallel/ParallelGCImpl.java | 3 --- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index be3dc1f0e23c..4146e121a578 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -358,17 +358,10 @@ static Object installForwardingPointerParallel(Object original, UnsignedWord ori // try installing the new header Pointer originalPtr = Word.objectToUntrackedPointer(original); - long witness; - if (getReferenceSize() == Integer.BYTES) { - witness = originalPtr.compareAndSwapInt(getHubOffset(), - (int) originalHeader.rawValue(), (int) forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); - } else { - witness = originalPtr.compareAndSwapLong(getHubOffset(), - originalHeader.rawValue(), forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); - } + UnsignedWord witness = originalPtr.compareAndSwapWord(getHubOffset(), originalHeader, forwardHeader, LocationIdentity.ANY_LOCATION); assert isPointerToForwardedObject(originalPtr); - if (witness != originalHeader.rawValue()) { - return getForwardedObject(originalPtr, WordFactory.unsigned(witness)); + if (witness.notEqual(originalHeader)) { + return getForwardedObject(originalPtr, witness); } else if (hasShift) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index bdc75c7fef3f..b5d09eaabe72 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,6 +28,7 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.config.ConfigurationValues; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -395,7 +396,8 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); - UnsignedWord originalHeader = ObjectHeaderImpl.readHeaderFromPointer(originalMemory); + int hubOffset = ObjectHeaderImpl.getHubOffset(); + UnsignedWord originalHeader = originalMemory.readWord(hubOffset); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { return ObjectHeaderImpl.getForwardedObject(originalMemory, originalHeader); } @@ -414,19 +416,16 @@ Object promoteAlignedObjectParallel(Object original, Space originalSpace) { } // Install forwarding pointer into the original header - assert ObjectHeaderImpl.getHubOffset() == 0; Object forward = ObjectHeaderImpl.installForwardingPointerParallel(original, originalHeader, copy); - if (forward == copy) { // We have won the race, now we must copy the object bits. First install the original header - int headerSize = ObjectHeaderImpl.getReferenceSize(); - if (headerSize == Integer.BYTES) { - copyMemory.writeInt(0, (int) originalHeader.rawValue()); - } else { - copyMemory.writeWord(0, originalHeader); - } + copyMemory.writeWord(hubOffset, originalHeader); // Copy the rest of original object - UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(headerSize), copyMemory.add(headerSize), size.subtract(headerSize)); + if (hubOffset > 0) { + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); + } + int offset = hubOffset + ConfigurationValues.getTarget().wordSize; + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), size.subtract(offset)); if (isOldSpace()) { // If the object was promoted to the old gen, we need to take care of the remembered diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index bd2464d09c2a..168725c472e4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -112,9 +112,6 @@ public void push(Pointer ptr) { @Override public void startWorkerThreadsImpl() { - int hubOffset = ConfigurationValues.getObjectLayout().getHubOffset(); - VMError.guarantee(hubOffset == 0, "hub offset must be 0"); - // Allocate buffer large enough to store maximum possible number of heap chunks long maxHeapSize = GCImpl.getPolicy().getMaximumHeapSize().rawValue(); long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); From 1a22815ba06b0cc42d88993a1a54e2a9798d71de Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 23 Nov 2022 15:17:00 +0300 Subject: [PATCH 073/125] Worker thread shutdown --- .../oracle/svm/core/genscavenge/HeapImpl.java | 5 +++++ .../genscavenge/parallel/ParallelGCImpl.java | 22 +++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 29617435e89f..a176b61d6442 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -28,6 +28,8 @@ import java.util.ArrayList; import java.util.List; +import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.heap.ParallelGC; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.SuppressFBWarnings; @@ -200,6 +202,9 @@ boolean walkMemory(MemoryWalker.Visitor visitor) { @Override @Uninterruptible(reason = "Tear-down in progress.") public boolean tearDown() { + if (ParallelGC.isEnabled()) { + ParallelGCImpl.singleton().tearDown(); + } youngGeneration.tearDown(); oldGeneration.tearDown(); getChunkProvider().tearDown(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 168725c472e4..1bb512dbd632 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -27,7 +27,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; -import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; @@ -40,6 +40,7 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; @@ -51,7 +52,9 @@ import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -64,6 +67,7 @@ public class ParallelGCImpl extends ParallelGC { */ private int workerCount = 1; private int allWorkersBusy; + private List workers; private final AtomicInteger busyWorkers = new AtomicInteger(0); /** @@ -120,11 +124,9 @@ public void startWorkerThreadsImpl() { buffer = new ChunkBuffer((int) maxChunks); workerCount = getWorkerCount(); - allWorkersBusy = ~(-1 << workerCount) & (-1 << 1); - // We reuse the gc thread for worker #0 - for (int i = 1; i < workerCount; i++) { - startWorkerThread(i); - } + // We reuse the gc thread as the last worker + workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); + allWorkersBusy = ~(-1 << (workerCount - 1)); } private int getWorkerCount() { @@ -141,7 +143,7 @@ private int getDefaultWorkerCount() { return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; } - private void startWorkerThread(int n) { + private Thread startWorkerThread(int n) { Thread t = new Thread(() -> { VMThreads.SafepointBehavior.markThreadAsCrashed(); debugLog().string("WW start ").unsigned(n).newline(); @@ -177,6 +179,12 @@ private void startWorkerThread(int n) { t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); + return t; + } + + @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) + public void tearDown() { + workers.forEach(PlatformThreads::exit); } private void waitForIdleImpl() { From 00a8f37e34749c28eca73ab7f228d2c8b17a93f1 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 23 Nov 2022 15:56:20 +0300 Subject: [PATCH 074/125] Made `ChunkBuffer` grow as needed and free memory in the end --- .../genscavenge/parallel/ChunkBuffer.java | 52 ++++++++++++++++--- .../genscavenge/parallel/ParallelGCImpl.java | 9 +--- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index b1bb86500d86..60a1614ded48 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -26,9 +26,11 @@ package com.oracle.svm.core.genscavenge.parallel; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.os.CommittedMemoryProvider; import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -36,26 +38,38 @@ * Synchronized buffer that stores "grey" heap chunks to be scanned. */ public class ChunkBuffer { + private static final int INITIAL_SIZE = 1024 * wordSize(); + private Pointer buffer; private int size, top; - ChunkBuffer(int maxChunks) { - this.size = maxChunks * ConfigurationValues.getTarget().wordSize; - this.buffer = UnmanagedMemory.malloc(this.size); + @Fold + static int wordSize() { + return ConfigurationValues.getTarget().wordSize; + } + + ChunkBuffer() { + this.size = INITIAL_SIZE; + this.buffer = malloc(this.size); } void push(Pointer ptr) { - ParallelGCImpl.mutex.assertIsOwner("Must hold ParallelGCImpl.mutex"); - assert top < size; + assert !ParallelGCImpl.isInParallelPhase() || ParallelGCImpl.mutex.isOwner(); + if (top >= size) { + int oldSize = size; + size *= 2; + assert top < size; + buffer = realloc(buffer, oldSize, size); + } buffer.writeWord(top, ptr); - top += ConfigurationValues.getTarget().wordSize; + top += wordSize(); } Pointer pop() { ParallelGCImpl.mutex.lock(); try { if (top > 0) { - top -= ConfigurationValues.getTarget().wordSize; + top -= wordSize(); return buffer.readWord(top); } else { return WordFactory.nullPointer(); @@ -64,4 +78,26 @@ Pointer pop() { ParallelGCImpl.mutex.unlock(); } } + + void release() { + free(buffer, size); + } + + @AlwaysInline("GC performance") + private Pointer malloc(int size) { + return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(size)); + } + + @AlwaysInline("GC performance") + private Pointer realloc(Pointer orig, int origSize, int newSize) { + Pointer ptr = malloc(newSize); + UnmanagedMemoryUtil.copyLongsForward(orig, ptr, WordFactory.unsigned(origSize)); + free(orig, origSize); + return ptr; + } + + @AlwaysInline("GC performance") + private void free(Pointer ptr, int size) { + CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(size)); + } } \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 1bb512dbd632..0349b4293131 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -116,13 +116,7 @@ public void push(Pointer ptr) { @Override public void startWorkerThreadsImpl() { - // Allocate buffer large enough to store maximum possible number of heap chunks - long maxHeapSize = GCImpl.getPolicy().getMaximumHeapSize().rawValue(); - long alignedChunkSize = HeapParameters.getAlignedHeapChunkSize().rawValue(); - long unalignedChunkSize = HeapParameters.getLargeArrayThreshold().rawValue(); - long maxChunks = maxHeapSize / Math.min(alignedChunkSize, unalignedChunkSize) + 1; - buffer = new ChunkBuffer((int) maxChunks); - + buffer = new ChunkBuffer(); workerCount = getWorkerCount(); // We reuse the gc thread as the last worker workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); @@ -184,6 +178,7 @@ private Thread startWorkerThread(int n) { @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) public void tearDown() { + buffer.release(); workers.forEach(PlatformThreads::exit); } From 2af8a3064bc4c0322aa831efcfb5e73edb3b63a2 Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 25 Nov 2022 12:56:22 +0300 Subject: [PATCH 075/125] More robust worker thread routine --- .../genscavenge/parallel/ParallelGCImpl.java | 114 ++++++++---------- 1 file changed, 51 insertions(+), 63 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 0349b4293131..dd1668e2353e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; +import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapParameters; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; @@ -59,16 +60,9 @@ public class ParallelGCImpl extends ParallelGC { public static final int UNALIGNED_BIT = 0x01; - private static final int MAX_WORKERS_COUNT = 31; - /** - * Number of parallel GC workers. Default is 1 so that if GC happens before worker threads are started, - * it occurs on a single thread just like serial GC. - */ - private int workerCount = 1; - private int allWorkersBusy; private List workers; - private final AtomicInteger busyWorkers = new AtomicInteger(0); + private final AtomicInteger busyWorkers = new AtomicInteger(); /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -112,21 +106,22 @@ public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { public void push(Pointer ptr) { buffer.push(ptr); + if (inParallelPhase) { + parPhase.signal(); + } } @Override public void startWorkerThreadsImpl() { buffer = new ChunkBuffer(); - workerCount = getWorkerCount(); - // We reuse the gc thread as the last worker + int workerCount = getWorkerCount(); + busyWorkers.set(workerCount); workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); - allWorkersBusy = ~(-1 << (workerCount - 1)); } private int getWorkerCount() { int setting = ParallelGCOptions.ParallelGCThreads.getValue(); int workers = setting > 0 ? setting : getDefaultWorkerCount(); - workers = Math.min(workers, MAX_WORKERS_COUNT); verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); return workers; } @@ -144,27 +139,24 @@ private Thread startWorkerThread(int n) { while (true) { try { - do { - debugLog().string("WW block ").unsigned(n).newline(); + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && allocChunkTL.get().isNull()) { + if (busyWorkers.decrementAndGet() == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + debugLog().string("WW idle ").unsigned(n).newline(); mutex.lock(); parPhase.block(); mutex.unlock(); - } while ((busyWorkers.get() & (1 << n)) == 0); + busyWorkers.incrementAndGet(); + debugLog().string("WW run ").unsigned(n).newline(); + } - debugLog().string("WW run ").unsigned(n).newline(); - drainBuffer(); - int witness = busyWorkers.get(); - int expected; do { - expected = witness; - witness = busyWorkers.compareAndExchange(expected, expected & ~(1 << n)); - } while (witness != expected); - if (busyWorkers.get() == 0) { - mutex.lock(); - seqPhase.signal(); - mutex.unlock(); - } - debugLog().string("WW idle ").unsigned(n).newline(); + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); } catch (Throwable ex) { VMError.shouldNotReachHere(ex); } @@ -176,53 +168,43 @@ private Thread startWorkerThread(int n) { return t; } - @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) - public void tearDown() { - buffer.release(); - workers.forEach(PlatformThreads::exit); - } - private void waitForIdleImpl() { - inParallelPhase = true; + assert allocChunkTL.get().isNonNull(); + push(HeapChunk.asPointer(allocChunkTL.get())); + allocChunkTL.set(WordFactory.nullPointer()); debugLog().string("PP start workers\n"); - busyWorkers.set(allWorkersBusy); + inParallelPhase = true; parPhase.broadcast(); // let worker threads run - drainBuffer(); - mutex.lock(); - while (busyWorkers.get() != 0) { - debugLog().string("PP wait busy=").unsigned(busyWorkers.get()).newline(); + while (inParallelPhase) { + debugLog().string("PP wait\n"); seqPhase.block(); // wait for them to become idle } mutex.unlock(); - - inParallelPhase = false; } - private void drainBuffer() { - while (true) { - Pointer ptr; - while ((ptr = buffer.pop()).notEqual(WordFactory.nullPointer())) { - debugLog().string("WW drain chunk=").zhex(ptr).newline(); - if (ptr.and(UNALIGNED_BIT).notEqual(0)) { - UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); - } else { - AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); - } - } - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - debugLog().string("WW drain allocChunk=").zhex(allocChunk).newline(); - if (allocChunk.equal(WordFactory.nullPointer())) { - break; + private void scanChunk(Pointer ptr) { + if (ptr.isNonNull()) { + debugLog().string("WW scan chunk=").zhex(ptr).newline(); + if (ptr.and(UNALIGNED_BIT).notEqual(0)) { + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { - scannedChunkTL.set(allocChunk); - AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); - if (allocChunkTL.get().equal(allocChunk)) { - // this allocation chunk is now black, retire it - allocChunkTL.set(WordFactory.nullPointer()); - } + AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); + } + } + } + + private void scanAllocChunk() { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + if (allocChunk.isNonNull()) { + debugLog().string("WW scan alloc=").zhex(allocChunk).newline(); + scannedChunkTL.set(allocChunk); + AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); + if (allocChunkTL.get().equal(allocChunk)) { + // this allocation chunk is now black, retire it + allocChunkTL.set(WordFactory.nullPointer()); } } scannedChunkTL.set(WordFactory.nullPointer()); @@ -232,6 +214,12 @@ private GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } + @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) + public void tearDown() { + buffer.release(); + workers.forEach(PlatformThreads::exit); + } + private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } From b849c4a51c499110bf3820337d17c548b919ad37 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 7 Dec 2022 10:17:32 +0300 Subject: [PATCH 076/125] Thread safe Reference handling --- .../genscavenge/ReferenceObjectProcessing.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index e0bffb409384..4021ecf30644 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceInternals; import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicReference; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; @@ -48,7 +49,7 @@ /** Discovers and handles {@link Reference} objects during garbage collection. */ final class ReferenceObjectProcessing { /** Head of the linked list of discovered references that need to be revisited. */ - private static Reference rememberedRefsList; + private static final AtomicReference> rememberedRefsList = new AtomicReference<>(); /** * For a {@link SoftReference}, the longest duration after its last access to keep its referent @@ -141,9 +142,11 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { // are revisited after the GC finished promoting all strongly reachable objects. // null link means undiscovered, avoid for the last node with a cyclic reference - Reference next = (rememberedRefsList != null) ? rememberedRefsList : dr; - ReferenceInternals.setNextDiscovered(dr, next); - rememberedRefsList = dr; + Reference expected; + do { + expected = rememberedRefsList.get(); + ReferenceInternals.setNextDiscovered(dr, expected != null ? expected : dr); + } while (!rememberedRefsList.compareAndSet(expected, dr)); } /** @@ -154,8 +157,7 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { */ static Reference processRememberedReferences() { Reference pendingHead = null; - Reference current = rememberedRefsList; - rememberedRefsList = null; + Reference current = rememberedRefsList.getAndSet(null); while (current != null) { // Get the next node (the last node has a cyclic reference to self). @@ -180,7 +182,7 @@ static Reference processRememberedReferences() { } static void afterCollection(UnsignedWord freeBytes) { - assert rememberedRefsList == null; + assert rememberedRefsList.get() == null; UnsignedWord unused = freeBytes.unsignedDivide(1024 * 1024 /* MB */); maxSoftRefAccessIntervalMs = unused.multiply(SerialGCOptions.SoftRefLRUPolicyMSPerMB.getValue()); ReferenceInternals.updateSoftReferenceClock(); From 14c2047b801262e84f784e03c8406b7d73d9cd21 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 7 Dec 2022 12:39:02 +0300 Subject: [PATCH 077/125] Made SerialGC options work with ParallelGC --- .../core/genscavenge/JfrGCEventSupport.java | 2 +- .../SerialAndEpsilonGCOptions.java | 2 +- .../svm/core/genscavenge/SerialGCOptions.java | 38 +++++++++---------- .../UseMarkAndCopyOrEpsilonGC.java | 2 +- .../genscavenge/graal/BarrierSnippets.java | 2 +- .../graal/GenScavengeGCFeature.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 4 +- .../heap/Target_java_lang_ref_Reference.java | 2 +- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 5270d0282244..af3c72b7c4a3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -141,7 +141,7 @@ private int popPhase() { class JfrGCEventFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.useSerialOrParallelGC(); + return SubstrateOptions.useMarkAndCopyGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index e6b76ab52316..2985036e6a48 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -80,7 +80,7 @@ private SerialAndEpsilonGCOptions() { } private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseMarkAndCopyOrEpsilonGC()) { + if (!SubstrateOptions.useMarkAndCopyOrEpsilonGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java index d5e6151e0f54..7bed2ef84982 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java @@ -40,16 +40,16 @@ /** Options that are only valid for the serial GC (and not for the epsilon GC). */ public final class SerialGCOptions { @Option(help = "The garbage collection policy, either Adaptive (default) or BySpaceAndTime. Serial GC only.", type = OptionType.User)// - public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::serialGCOnly); + public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::markAndCopyGCOnly); @Option(help = "Percentage of total collection time that should be spent on young generation collections. Serial GC with collection policy 'BySpaceAndTime' only.", type = OptionType.User)// - public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::markAndCopyGCOnly); @Option(help = "The maximum free bytes reserved for allocations, in bytes (0 for automatic according to GC policy). Serial GC only.", type = OptionType.User)// - public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Maximum number of survivor spaces. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::serialGCOnly) { + public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly) { @Override public Integer getValueOrDefault(UnmodifiableEconomicMap, Object> values) { Integer value = (Integer) values.get(this); @@ -65,50 +65,50 @@ public Integer getValue(OptionValues values) { }; @Option(help = "Determines if a full GC collects the young generation separately or together with the old generation. Serial GC only.", type = OptionType.Expert) // - public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Enables card marking for image heap objects, which arranges them in chunks. Automatically enabled when supported. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); @Option(help = "This number of milliseconds multiplied by the free heap memory in MByte is the time span " + "for which a soft reference will keep its referent alive after its last access. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print the shape of the heap before and after each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print summary GC information after application main method returns. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print a time stamp at each collection, if +PrintGC or +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Print the time for each of the phases of each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Instrument write barriers with counters. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify the remembered set if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify all object references if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Verify write barriers. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Trace heap chunks during collections, if +VerboseGC and +PrintHeapShape. Serial GC only.", type = OptionType.Debug) // - public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); @Option(help = "Develop demographics of the object references visited. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly); + public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); private SerialGCOptions() { } - private static void serialGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.UseSerialGC.getValue()) { + private static void markAndCopyGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useMarkAndCopyGC()) { throw new InterruptImageBuilding("The option " + optionKey.getName() + " is garbage collector specific and cannot be specified if the " + Heap.getHeap().getGC().getName() + " is used at runtime."); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java index 9e824c594140..7ab0a5f6d284 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java @@ -35,6 +35,6 @@ public class UseMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); + return SubstrateOptions.useMarkAndCopyOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index 69adc25a7743..56a2b518cb2c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -204,7 +204,7 @@ class BarrierSnippetCounters { class BarrierSnippetCountersFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.useSerialOrParallelGC() && SubstrateOptions.useRememberedSet(); + return SubstrateOptions.useMarkAndCopyGC() && SubstrateOptions.useRememberedSet(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index e9efa8939ccf..4959d7eb3845 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -156,7 +156,7 @@ private static RememberedSet createRememberedSet() { } private static PerfDataHolder createPerfData() { - if (SubstrateOptions.useSerialOrParallelGC()) { + if (SubstrateOptions.useMarkAndCopyGC()) { return new SerialGCPerfData(); } else { assert SubstrateOptions.UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 556b2fe13a46..610237996463 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -350,12 +350,12 @@ private static void validateParallelGC(HostedOptionKey optionKey, boole } @Fold - public static boolean useSerialOrParallelGC() { + public static boolean useMarkAndCopyGC() { return UseSerialGC.getValue() || UseParallelGC.getValue(); } @Fold - public static boolean UseMarkAndCopyOrEpsilonGC() { + public static boolean useMarkAndCopyOrEpsilonGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index 2a3fdd519d37..557818613de9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -232,6 +232,6 @@ public Object transform(Object receiver, Object originalValue) { class NotMarkAndCopyOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.UseMarkAndCopyOrEpsilonGC(); + return !SubstrateOptions.useMarkAndCopyOrEpsilonGC(); } } From 271134d8f456892965562f3707ce6324d03bda6a Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 8 Dec 2022 13:33:26 +0300 Subject: [PATCH 078/125] Enabled incremental collections --- .../oracle/svm/core/genscavenge/GCImpl.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 57 +++++++++++++------ .../genscavenge/parallel/ParallelGCImpl.java | 16 +++--- .../com/oracle/svm/core/SubstrateOptions.java | 9 +-- .../oracle/svm/core/hub/LayoutEncoding.java | 1 - 5 files changed, 52 insertions(+), 37 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index dbc9ea720977..41382ef50abd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1042,10 +1042,10 @@ private static void prepareForPromotion(boolean isIncremental) { private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { - if (ParallelGC.isEnabled()) { - ParallelGCImpl.waitForIdle(); - } else if (isIncremental) { + if (isIncremental) { scanGreyObjectsLoop(); + } else if (ParallelGC.isEnabled()) { + ParallelGCImpl.singleton().waitForIdle(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index b5d09eaabe72..4c7798de2554 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -167,14 +167,27 @@ public Log report(Log log, boolean traceHeapChunks) { */ @AlwaysInline("GC performance") private Pointer allocateMemory(UnsignedWord objectSize) { + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + return allocateMemoryParallel(objectSize); + } Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk; - if (ParallelGC.isEnabled()) { - oldChunk = ParallelGCImpl.getThreadLocalChunk(); - } else { - oldChunk = getLastAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); + if (oldChunk.isNonNull()) { + result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } + if (result.isNonNull()) { + return result; + } + /* Slow-path: try allocating a new chunk for the requested memory. */ + return allocateInNewChunk(objectSize); + } + + @AlwaysInline("GC performance") + private Pointer allocateMemoryParallel(UnsignedWord objectSize) { + Pointer result = WordFactory.nullPointer(); + /* Fast-path: try allocating in the thread local allocation chunk. */ + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -182,7 +195,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return result; } /* Slow-path: try allocating a new chunk for the requested memory. */ - return allocateInNewChunk(oldChunk, objectSize); + return allocateInNewChunkParallel(oldChunk, objectSize); } /** @@ -191,24 +204,32 @@ private Pointer allocateMemory(UnsignedWord objectSize) { @AlwaysInline("GC performance") private Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getThreadLocalChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } - private Pointer allocateInNewChunk(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { - if (ParallelGC.isEnabled()) { - ParallelGCImpl.mutex.lock(); + private Pointer allocateInNewChunk(UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + if (newChunk.isNonNull()) { + return AlignedHeapChunk.allocateMemory(newChunk, objectSize); + } + return WordFactory.nullPointer(); + } + + private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader newChunk; + ParallelGCImpl.mutex.lock(); + try { if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } - } - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); - if (ParallelGC.isEnabled()) { + newChunk = requestAlignedHeapChunk(); + } finally { ParallelGCImpl.mutex.unlock(); - ParallelGCImpl.setThreadLocalChunk(newChunk); } if (newChunk.isNonNull()) { + ParallelGCImpl.setAllocationChunk(newChunk); return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } return WordFactory.nullPointer(); @@ -379,7 +400,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { - return promoteAlignedObjectParallel(original, originalSpace); + return copyAlignedObjectParallel(original); } Object copy = copyAlignedObject(original); @@ -390,7 +411,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { } @AlwaysInline("GC performance") - Object promoteAlignedObjectParallel(Object original, Space originalSpace) { + Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); @@ -485,7 +506,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); @@ -511,7 +532,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled()) { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); if (ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.unlock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index dd1668e2353e..24ad775b1e91 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -34,7 +34,6 @@ import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; -import com.oracle.svm.core.genscavenge.HeapParameters; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; @@ -88,23 +87,21 @@ public static boolean isInParallelPhase() { return singleton().inParallelPhase; } - public static void waitForIdle() { - singleton().waitForIdleImpl(); - } - public static AlignedHeapChunk.AlignedHeader getThreadLocalScannedChunk() { return scannedChunkTL.get(); } - public static AlignedHeapChunk.AlignedHeader getThreadLocalChunk() { + public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { return allocChunkTL.get(); } - public static void setThreadLocalChunk(AlignedHeapChunk.AlignedHeader chunk) { + public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { + assert chunk.isNonNull(); allocChunkTL.set(chunk); } public void push(Pointer ptr) { + assert ptr.isNonNull(); buffer.push(ptr); if (inParallelPhase) { parPhase.signal(); @@ -168,7 +165,10 @@ private Thread startWorkerThread(int n) { return t; } - private void waitForIdleImpl() { + /** + * Start parallel phase and wait until all chunks have been processed. Used by complete collections. + */ + public void waitForIdle() { assert allocChunkTL.get().isNonNull(); push(HeapChunk.asPointer(allocChunkTL.get())); allocChunkTL.set(WordFactory.nullPointer()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 610237996463..2fae63ed226e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -339,13 +339,8 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o }; private static void validateParallelGC(OptionKey unused) { - validateParallelGC(SubstrateOptions.MultiThreaded, true); - validateParallelGC(ConcealedOptions.UseRememberedSet, false); - } - - private static void validateParallelGC(HostedOptionKey optionKey, boolean expected) { - if (optionKey.getValue() != expected) { - throw UserError.abort("ParallelGC requires the option %s to be %s", optionKey, expected); + if (!SubstrateOptions.MultiThreaded.getValue()) { + throw new InterruptImageBuilding("ParallelGC requires the option MultiThreaded to be set."); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index b290969c630a..a60463fdcdbb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -274,7 +274,6 @@ public static UnsignedWord getSizeFromObjectInline(Object obj) { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromEncoding(Object obj, int encoding) { - assert encoding == KnownIntrinsics.readHub(obj).getLayoutEncoding(); if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj)); } else { From bced7d1fc28e09f9a130b867b5f6f6b299414a42 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 12 Dec 2022 13:23:17 +0300 Subject: [PATCH 079/125] Use try-finally for mutex locking --- .../oracle/svm/core/genscavenge/Space.java | 30 +++++++------ .../genscavenge/parallel/ParallelGCImpl.java | 43 +++++++++++-------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 4c7798de2554..828c65e92c49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -504,12 +504,15 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + try { + originalSpace.extractAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk); + } finally { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } } @@ -530,12 +533,15 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { ParallelGCImpl.mutex.lock(); } - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + try { + originalSpace.extractUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk); + } finally { + if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); + if (ParallelGCImpl.isInParallelPhase()) { + ParallelGCImpl.mutex.unlock(); + } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 24ad775b1e91..07f7cac21fed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -53,7 +53,6 @@ import org.graalvm.word.WordFactory; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; public class ParallelGCImpl extends ParallelGC { @@ -61,7 +60,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int UNALIGNED_BIT = 0x01; private List workers; - private final AtomicInteger busyWorkers = new AtomicInteger(); + private int busyWorkers; /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. @@ -112,7 +111,7 @@ public void push(Pointer ptr) { public void startWorkerThreadsImpl() { buffer = new ChunkBuffer(); int workerCount = getWorkerCount(); - busyWorkers.set(workerCount); + busyWorkers = workerCount; workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); } @@ -138,16 +137,19 @@ private Thread startWorkerThread(int n) { try { Pointer ptr; while (!inParallelPhase || (ptr = buffer.pop()).isNull() && allocChunkTL.get().isNull()) { - if (busyWorkers.decrementAndGet() == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - debugLog().string("WW idle ").unsigned(n).newline(); mutex.lock(); - parPhase.block(); - mutex.unlock(); - busyWorkers.incrementAndGet(); - debugLog().string("WW run ").unsigned(n).newline(); + try { + if (--busyWorkers == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + debugLog().string("WW idle ").unsigned(n).newline(); + parPhase.block(); + ++busyWorkers; + debugLog().string("WW run ").unsigned(n).newline(); + } finally { + mutex.unlock(); + } } do { @@ -174,15 +176,18 @@ public void waitForIdle() { allocChunkTL.set(WordFactory.nullPointer()); debugLog().string("PP start workers\n"); - inParallelPhase = true; - parPhase.broadcast(); // let worker threads run - mutex.lock(); - while (inParallelPhase) { - debugLog().string("PP wait\n"); - seqPhase.block(); // wait for them to become idle + try { + inParallelPhase = true; + parPhase.broadcast(); // let worker threads run + + while (inParallelPhase) { + debugLog().string("PP wait\n"); + seqPhase.block(); // wait for them to become idle + } + } finally { + mutex.unlock(); } - mutex.unlock(); } private void scanChunk(Pointer ptr) { From 165a8dcca7d99f0900c81001825504c3aa3c99fc Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 13 Dec 2022 15:08:19 +0300 Subject: [PATCH 080/125] Added assertion to `ParallelGCImpl.getScannedChunk()` --- .../src/com/oracle/svm/core/genscavenge/Space.java | 2 +- .../oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 828c65e92c49..682a32ed4c72 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -221,7 +221,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu AlignedHeapChunk.AlignedHeader newChunk; ParallelGCImpl.mutex.lock(); try { - if (oldChunk.notEqual(ParallelGCImpl.getThreadLocalScannedChunk())) { + if (oldChunk.notEqual(ParallelGCImpl.getScannedChunk())) { ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); } newChunk = requestAlignedHeapChunk(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 07f7cac21fed..2a0c9af3c30b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -86,7 +86,8 @@ public static boolean isInParallelPhase() { return singleton().inParallelPhase; } - public static AlignedHeapChunk.AlignedHeader getThreadLocalScannedChunk() { + public static AlignedHeapChunk.AlignedHeader getScannedChunk() { + assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); return scannedChunkTL.get(); } From 8ec4ce900c7f4d5314900f6f5a0ce916a65adcb4 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 13 Dec 2022 15:44:55 +0300 Subject: [PATCH 081/125] Wait for workers to be blocked before starting parallel phase --- .../svm/core/genscavenge/parallel/ParallelGCImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index 2a0c9af3c30b..ed0b2bbbefd8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -176,15 +176,20 @@ public void waitForIdle() { push(HeapChunk.asPointer(allocChunkTL.get())); allocChunkTL.set(WordFactory.nullPointer()); - debugLog().string("PP start workers\n"); mutex.lock(); try { + while (busyWorkers > 0) { // wait for worker threads to become ready + debugLog().string("PP wait for workers\n"); + seqPhase.block(); + } + + debugLog().string("PP start workers\n"); inParallelPhase = true; - parPhase.broadcast(); // let worker threads run + parPhase.broadcast(); // let worker threads run while (inParallelPhase) { debugLog().string("PP wait\n"); - seqPhase.block(); // wait for them to become idle + seqPhase.block(); // wait for them to become idle } } finally { mutex.unlock(); From e4ec854b206013117445e4776a9b90fdd18a4699 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 26 Jan 2023 10:49:16 +0300 Subject: [PATCH 082/125] Don't retire scanned allocation chunks Instead, remember top position and rescan starting from it if needed --- .../oracle/svm/core/genscavenge/Space.java | 4 +- .../genscavenge/parallel/ParallelGCImpl.java | 64 +++++++++++++------ 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 682a32ed4c72..a0838fb9cb36 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -221,9 +221,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu AlignedHeapChunk.AlignedHeader newChunk; ParallelGCImpl.mutex.lock(); try { - if (oldChunk.notEqual(ParallelGCImpl.getScannedChunk())) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(oldChunk)); - } + ParallelGCImpl.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); } finally { ParallelGCImpl.mutex.unlock(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java index ed0b2bbbefd8..91cb59ab317c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java @@ -50,6 +50,7 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import java.util.List; @@ -59,7 +60,7 @@ public class ParallelGCImpl extends ParallelGC { public static final int UNALIGNED_BIT = 0x01; - private List workers; + private Thread[] workers; private int busyWorkers; /** @@ -69,6 +70,8 @@ public class ParallelGCImpl extends ParallelGC { FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); private static final FastThreadLocalWord scannedChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); + private static final FastThreadLocalWord allocChunkScanOffsetTL = + FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); @@ -86,11 +89,6 @@ public static boolean isInParallelPhase() { return singleton().inParallelPhase; } - public static AlignedHeapChunk.AlignedHeader getScannedChunk() { - assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); - return scannedChunkTL.get(); - } - public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { return allocChunkTL.get(); } @@ -98,6 +96,7 @@ public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { assert chunk.isNonNull(); allocChunkTL.set(chunk); + allocChunkScanOffsetTL.set(AlignedHeapChunk.getObjectsStartOffset()); } public void push(Pointer ptr) { @@ -108,12 +107,23 @@ public void push(Pointer ptr) { } } + public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { + assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + if (chunk.notEqual(scannedChunkTL.get())) { + UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + assert scanOffset.aboveThan(0); + if (chunk.getTopOffset().aboveThan(scanOffset)) { + push(HeapChunk.asPointer(chunk).add(scanOffset)); + } + } + } + @Override public void startWorkerThreadsImpl() { buffer = new ChunkBuffer(); int workerCount = getWorkerCount(); busyWorkers = workerCount; - workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toList(); + workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toArray(Thread[]::new); } private int getWorkerCount() { @@ -137,7 +147,7 @@ private Thread startWorkerThread(int n) { while (true) { try { Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && allocChunkTL.get().isNull()) { + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { mutex.lock(); try { if (--busyWorkers == 0) { @@ -174,7 +184,6 @@ private Thread startWorkerThread(int n) { public void waitForIdle() { assert allocChunkTL.get().isNonNull(); push(HeapChunk.asPointer(allocChunkTL.get())); - allocChunkTL.set(WordFactory.nullPointer()); mutex.lock(); try { @@ -194,6 +203,11 @@ public void waitForIdle() { } finally { mutex.unlock(); } + // clean up thread local allocation chunks + allocChunkTL.set(WordFactory.nullPointer()); + for (Thread t: workers) { + allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); + } } private void scanChunk(Pointer ptr) { @@ -202,23 +216,35 @@ private void scanChunk(Pointer ptr) { if (ptr.and(UNALIGNED_BIT).notEqual(0)) { UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { - AlignedHeapChunk.walkObjectsInline((AlignedHeapChunk.AlignedHeader) ptr, getVisitor()); + AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); + if (chunk.equal(ptr)) { + ptr = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); + } + HeapChunk.walkObjectsFromInline(chunk, ptr, getVisitor()); } } } private void scanAllocChunk() { - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - if (allocChunk.isNonNull()) { - debugLog().string("WW scan alloc=").zhex(allocChunk).newline(); + if (allocChunkNeedsScanning()) { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + assert scanOffset.aboveThan(0); + Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); + debugLog().string("WW scan alloc=").zhex(allocChunk).string(" from offset ").unsigned(scanOffset).newline(); scannedChunkTL.set(allocChunk); - AlignedHeapChunk.walkObjectsInline(allocChunk, getVisitor()); + HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, getVisitor()); + scannedChunkTL.set(WordFactory.nullPointer()); if (allocChunkTL.get().equal(allocChunk)) { - // this allocation chunk is now black, retire it - allocChunkTL.set(WordFactory.nullPointer()); + // remember top offset so that we don't scan the same objects again + allocChunkScanOffsetTL.set(allocChunk.getTopOffset()); } } - scannedChunkTL.set(WordFactory.nullPointer()); + } + + private boolean allocChunkNeedsScanning() { + AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); + return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); } private GreyToBlackObjectVisitor getVisitor() { @@ -228,7 +254,9 @@ private GreyToBlackObjectVisitor getVisitor() { @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) public void tearDown() { buffer.release(); - workers.forEach(PlatformThreads::exit); + for (Thread t: workers) { + PlatformThreads.exit(t); + } } private static Log verboseGCLog() { From dd322190026be28d615aa12344ff1e5220ac1ee8 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Mar 2023 14:12:45 +0300 Subject: [PATCH 083/125] Merged ParallelGC and ParallelGCImpl --- .../oracle/svm/core/genscavenge/GCImpl.java | 11 +++-- .../oracle/svm/core/genscavenge/HeapImpl.java | 10 ++-- .../oracle/svm/core/genscavenge/Space.java | 45 +++++++++--------- .../genscavenge/parallel/ChunkBuffer.java | 6 +-- .../{ParallelGCImpl.java => ParallelGC.java} | 21 +++++---- .../graal/snippets/CEntryPointSnippets.java | 6 +-- .../src/com/oracle/svm/core/heap/Heap.java | 2 + .../com/oracle/svm/core/heap/ParallelGC.java | 47 ------------------- 8 files changed, 57 insertions(+), 91 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/{ParallelGCImpl.java => ParallelGC.java} (95%) delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 98ac4586a023..090d269a271e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -64,7 +64,7 @@ import com.oracle.svm.core.genscavenge.BasicCollectionPolicies.NeverCollect; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.graal.RuntimeCompilation; import com.oracle.svm.core.heap.CodeReferenceMapDecoder; @@ -72,7 +72,6 @@ import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.NoAllocationVerifier; import com.oracle.svm.core.heap.OutOfMemoryUtil; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceMapIndex; import com.oracle.svm.core.heap.RestrictHeapAccess; @@ -146,6 +145,12 @@ public void collect(GCCause cause) { collect(cause, false); } + public void initialize() { + if (ParallelGC.isEnabled()) { + ParallelGC.singleton().startWorkerThreads(); + } + } + public void maybeCollectOnAllocation() { boolean outOfMemory = false; if (hasNeverCollectPolicy()) { @@ -1055,7 +1060,7 @@ private void scanGreyObjects(boolean isIncremental) { if (isIncremental) { scanGreyObjectsLoop(); } else if (ParallelGC.isEnabled()) { - ParallelGCImpl.singleton().waitForIdle(); + ParallelGC.singleton().waitForIdle(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index c73583c9d8ac..2eadb6f75e1b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -49,7 +49,6 @@ import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; @@ -59,7 +58,7 @@ import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; import com.oracle.svm.core.genscavenge.graal.ForcedSerialPostWriteBarrier; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets; import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; @@ -204,7 +203,7 @@ boolean walkMemory(MemoryWalker.Visitor visitor) { @Uninterruptible(reason = "Tear-down in progress.") public boolean tearDown() { if (ParallelGC.isEnabled()) { - ParallelGCImpl.singleton().tearDown(); + ParallelGC.singleton().tearDown(); } youngGeneration.tearDown(); oldGeneration.tearDown(); @@ -243,6 +242,11 @@ GCImpl getGCImpl() { return gcImpl; } + @Override + public void initGC() { + gcImpl.initialize(); + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAllocationDisallowed() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 26201ad5a6d3..ec067cfbe591 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -44,10 +44,9 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; -import com.oracle.svm.core.genscavenge.parallel.ParallelGCImpl; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; @@ -191,7 +190,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { private Pointer allocateMemoryParallel(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the thread local allocation chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -207,8 +206,8 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { */ @AlwaysInline("GC performance") private Pointer retractAllocation(UnsignedWord objectSize) { - assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGCImpl.getAllocationChunk(); + assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } @@ -223,15 +222,15 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk; - ParallelGCImpl.mutex.lock(); + ParallelGC.mutex.lock(); try { - ParallelGCImpl.singleton().pushAllocChunk(oldChunk); + ParallelGC.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); } finally { - ParallelGCImpl.mutex.unlock(); + ParallelGC.mutex.unlock(); } if (newChunk.isNonNull()) { - ParallelGCImpl.setAllocationChunk(newChunk); + ParallelGC.setAllocationChunk(newChunk); return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } return WordFactory.nullPointer(); @@ -401,7 +400,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { return copyAlignedObjectParallel(original); } @@ -415,7 +414,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { @AlwaysInline("GC performance") Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); - assert ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase(); + assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); Pointer originalMemory = Word.objectToUntrackedPointer(original); @@ -522,17 +521,17 @@ private Object copyAlignedObject(Object originalObj) { void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.lock(); + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.lock(); } try { originalSpace.extractAlignedHeapChunk(chunk); appendAlignedHeapChunk(chunk); } finally { - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); + if (ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.unlock(); } } } @@ -551,17 +550,17 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGCImpl.isEnabled() && ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.lock(); + if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.lock(); } try { originalSpace.extractUnalignedHeapChunk(chunk); appendUnalignedHeapChunk(chunk); } finally { - if (ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGCImpl.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGCImpl.UNALIGNED_BIT)); - if (ParallelGCImpl.isInParallelPhase()) { - ParallelGCImpl.mutex.unlock(); + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); + if (ParallelGC.isInParallelPhase()) { + ParallelGC.mutex.unlock(); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 60a1614ded48..17dbc628ef08 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -54,7 +54,7 @@ static int wordSize() { } void push(Pointer ptr) { - assert !ParallelGCImpl.isInParallelPhase() || ParallelGCImpl.mutex.isOwner(); + assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.isOwner(); if (top >= size) { int oldSize = size; size *= 2; @@ -66,7 +66,7 @@ void push(Pointer ptr) { } Pointer pop() { - ParallelGCImpl.mutex.lock(); + ParallelGC.mutex.lock(); try { if (top > 0) { top -= wordSize(); @@ -75,7 +75,7 @@ Pointer pop() { return WordFactory.nullPointer(); } } finally { - ParallelGCImpl.mutex.unlock(); + ParallelGC.mutex.unlock(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java similarity index 95% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 91cb59ab317c..5de72d3949ae 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -27,6 +27,7 @@ package com.oracle.svm.core.genscavenge.parallel; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; @@ -35,7 +36,6 @@ import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; -import com.oracle.svm.core.heap.ParallelGC; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; @@ -53,10 +53,9 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import java.util.List; import java.util.stream.IntStream; -public class ParallelGCImpl extends ParallelGC { +public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; @@ -81,8 +80,13 @@ public class ParallelGCImpl extends ParallelGC { private volatile boolean inParallelPhase; @Fold - public static ParallelGCImpl singleton() { - return (ParallelGCImpl) ImageSingletons.lookup(ParallelGC.class); + public static ParallelGC singleton() { + return ImageSingletons.lookup(ParallelGC.class); + } + + @Fold + public static boolean isEnabled() { + return SubstrateOptions.UseParallelGC.getValue(); } public static boolean isInParallelPhase() { @@ -108,7 +112,7 @@ public void push(Pointer ptr) { } public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { - assert ParallelGCImpl.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + assert ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); if (chunk.notEqual(scannedChunkTL.get())) { UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); assert scanOffset.aboveThan(0); @@ -118,8 +122,7 @@ public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { } } - @Override - public void startWorkerThreadsImpl() { + public void startWorkerThreads() { buffer = new ChunkBuffer(); int workerCount = getWorkerCount(); busyWorkers = workerCount; @@ -280,6 +283,6 @@ public boolean isInConfiguration(IsInConfigurationAccess access) { @Override public void afterRegistration(AfterRegistrationAccess access) { - ImageSingletons.add(ParallelGC.class, new ParallelGCImpl()); + ImageSingletons.add(ParallelGC.class, new ParallelGC()); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 9514d24d7248..e46003720772 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -86,7 +86,7 @@ import com.oracle.svm.core.graal.nodes.CEntryPointEnterNode; import com.oracle.svm.core.graal.nodes.CEntryPointLeaveNode; import com.oracle.svm.core.graal.nodes.CEntryPointUtilityNode; -import com.oracle.svm.core.heap.ParallelGC; +import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.heap.RestrictHeapAccess; @@ -325,8 +325,8 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } - /* Number of workers can be affected by a runtime option, so start threads after options are parsed */ - ParallelGC.startWorkerThreads(); + /* Number of parallel GC workers can be affected by a runtime option, so call this after options are parsed */ + Heap.getHeap().initGC(); boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); if (firstIsolate) { // let other isolates (if any) initialize now diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index 4902b9c13dd5..99fa80d694c0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -80,6 +80,8 @@ protected Heap() { public abstract GC getGC(); + public abstract void initGC(); + public abstract RuntimeCodeInfoGCSupport getRuntimeCodeInfoGCSupport(); /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java deleted file mode 100644 index 95d8de510c9f..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ParallelGC.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.heap; - -import com.oracle.svm.core.SubstrateOptions; -import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; - -public abstract class ParallelGC { - - @Fold - public static boolean isEnabled() { - return SubstrateOptions.UseParallelGC.getValue(); - } - - public static void startWorkerThreads() { - if (isEnabled()) { - ImageSingletons.lookup(ParallelGC.class).startWorkerThreadsImpl(); - } - } - - public abstract void startWorkerThreadsImpl(); -} From d9f38f327dd83b3a6d45dcd8a96cacfff85c102f Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Mar 2023 15:47:48 +0300 Subject: [PATCH 084/125] Introduced special SafepointBehavior for worker threads --- .../svm/core/genscavenge/parallel/ParallelGC.java | 2 +- .../src/com/oracle/svm/core/thread/Safepoint.java | 6 ++++-- .../src/com/oracle/svm/core/thread/VMThreads.java | 13 +++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 5de72d3949ae..299ae2e88a1c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -144,7 +144,7 @@ private int getDefaultWorkerCount() { private Thread startWorkerThread(int n) { Thread t = new Thread(() -> { - VMThreads.SafepointBehavior.markThreadAsCrashed(); + VMThreads.SafepointBehavior.useAsParallelGCThread(); debugLog().string("WW start ").unsigned(n).newline(); while (true) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 01f45420aa5f..7f4c0ea82a20 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -760,7 +760,8 @@ private static void waitForSafepoints(String reason) { int safepointBehavior = SafepointBehavior.getSafepointBehaviorVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; @@ -905,7 +906,8 @@ public static int countingVMOperation() { int status = StatusSupport.getStatusVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints += 1; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 1c2df2c689fc..c0b1bc3502df 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -865,6 +865,9 @@ public static class SafepointBehavior { */ static final int THREAD_CRASHED = 2; + /** Similar to THREAD_CRASHED, used by ParallelGC worker threads. */ + static final int PARALLEL_GC_THREAD = 3; + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean ignoresSafepoints() { return safepointBehaviorTL.getVolatile() != ALLOW_SAFEPOINT; @@ -915,6 +918,14 @@ public static void markThreadAsCrashed() { safepointBehaviorTL.setVolatile(THREAD_CRASHED); } + /** + * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, + * so it must not keep references to movable heap objects on its stack. + */ + public static void useAsParallelGCThread() { + safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static String toString(int safepointBehavior) { switch (safepointBehavior) { @@ -924,6 +935,8 @@ public static String toString(int safepointBehavior) { return "PREVENT_VM_FROM_REACHING_SAFEPOINT"; case THREAD_CRASHED: return "THREAD_CRASHED"; + case PARALLEL_GC_THREAD: + return "PARALLEL_GC_THREAD"; default: return "Invalid safepoint behavior"; } From ec3b2a59dd3feee4d13d1e8575a9eba5e1daac1c Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 7 Mar 2023 15:48:33 +0300 Subject: [PATCH 085/125] Style fixes --- .../oracle/svm/core/genscavenge/GCImpl.java | 1 + .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../oracle/svm/core/genscavenge/Space.java | 1 + .../genscavenge/parallel/ChunkBuffer.java | 15 +++++++------- .../core/genscavenge/parallel/ParallelGC.java | 20 +++++++++++-------- .../com/oracle/svm/core/SubstrateOptions.java | 8 ++++---- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 090d269a271e..c6e5b1f337db 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -145,6 +145,7 @@ public void collect(GCCause cause) { collect(cause, false); } + @SuppressWarnings("static-method") public void initialize() { if (ParallelGC.isEnabled()) { ParallelGC.singleton().startWorkerThreads(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 1b35a0807d4f..7778ba7e46cc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -33,7 +33,6 @@ import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ec067cfbe591..185e885ad33a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -205,6 +205,7 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") + @SuppressWarnings("static-method") private Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 17dbc628ef08..6a6fdc06aa84 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -41,7 +41,8 @@ public class ChunkBuffer { private static final int INITIAL_SIZE = 1024 * wordSize(); private Pointer buffer; - private int size, top; + private int size; + private int top; @Fold static int wordSize() { @@ -84,12 +85,12 @@ void release() { } @AlwaysInline("GC performance") - private Pointer malloc(int size) { - return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(size)); + private static Pointer malloc(int bytes) { + return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(bytes)); } @AlwaysInline("GC performance") - private Pointer realloc(Pointer orig, int origSize, int newSize) { + private static Pointer realloc(Pointer orig, int origSize, int newSize) { Pointer ptr = malloc(newSize); UnmanagedMemoryUtil.copyLongsForward(orig, ptr, WordFactory.unsigned(origSize)); free(orig, origSize); @@ -97,7 +98,7 @@ private Pointer realloc(Pointer orig, int origSize, int newSize) { } @AlwaysInline("GC performance") - private void free(Pointer ptr, int size) { - CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(size)); + private static void free(Pointer ptr, int bytes) { + CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(bytes)); } -} \ No newline at end of file +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 299ae2e88a1c..f57ff42c0f3a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -129,14 +129,14 @@ public void startWorkerThreads() { workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toArray(Thread[]::new); } - private int getWorkerCount() { + private static int getWorkerCount() { int setting = ParallelGCOptions.ParallelGCThreads.getValue(); - int workers = setting > 0 ? setting : getDefaultWorkerCount(); - verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(workers).string("]").newline(); - return workers; + int count = setting > 0 ? setting : getDefaultWorkerCount(); + verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(count).string("]").newline(); + return count; } - private int getDefaultWorkerCount() { + private static int getDefaultWorkerCount() { // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() int cpus = Jvm.JVM_ActiveProcessorCount(); return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; @@ -213,21 +213,24 @@ public void waitForIdle() { } } + @SuppressWarnings("static-method") private void scanChunk(Pointer ptr) { if (ptr.isNonNull()) { debugLog().string("WW scan chunk=").zhex(ptr).newline(); if (ptr.and(UNALIGNED_BIT).notEqual(0)) { UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); } else { + Pointer start = ptr; AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); if (chunk.equal(ptr)) { - ptr = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); + start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); } - HeapChunk.walkObjectsFromInline(chunk, ptr, getVisitor()); + HeapChunk.walkObjectsFromInline(chunk, start, getVisitor()); } } } + @SuppressWarnings("static-method") private void scanAllocChunk() { if (allocChunkNeedsScanning()) { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); @@ -245,12 +248,13 @@ private void scanAllocChunk() { } } + @SuppressWarnings("static-method") private boolean allocChunkNeedsScanning() { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); } - private GreyToBlackObjectVisitor getVisitor() { + private static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 10f0049ca1c4..af819661c0de 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -36,7 +36,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.Set; import java.util.function.Predicate; import com.oracle.svm.core.util.InterruptImageBuilding; @@ -317,7 +316,7 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @APIOption(name = "parallel", group = GCGroup.class, customHelp = "Parallel garbage collector")// @Option(help = "Use a parallel GC")// - public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false, SubstrateOptions::validateParallelGC) { + public static final HostedOptionKey UseParallelGC = new HostedOptionKey<>(false, SubstrateOptions::requireMultiThreading) { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { @@ -327,9 +326,10 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }; - private static void validateParallelGC(OptionKey unused) { + private static void requireMultiThreading(OptionKey optionKey) { if (!SubstrateOptions.MultiThreaded.getValue()) { - throw new InterruptImageBuilding("ParallelGC requires the option MultiThreaded to be set."); + throw new InterruptImageBuilding( + "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); } } From 1462c2f124f4443c362cf3863ad2019e1e191cf4 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 9 Mar 2023 15:04:16 +0300 Subject: [PATCH 086/125] Fall back to serial GC if collection occurs before worker threads have been started --- .../oracle/svm/core/genscavenge/GCImpl.java | 8 +++-- .../genscavenge/parallel/ChunkBuffer.java | 6 ++++ .../core/genscavenge/parallel/ParallelGC.java | 36 +++++++++++++------ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c6e5b1f337db..eee115fc8d88 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1060,8 +1060,8 @@ private void scanGreyObjects(boolean isIncremental) { try { if (isIncremental) { scanGreyObjectsLoop(); - } else if (ParallelGC.isEnabled()) { - ParallelGC.singleton().waitForIdle(); + } else if (ParallelGC.isEnabled() && ParallelGC.singleton().waitForIdle()) { + // GC can happen before parallel worker threads have been started. In this case, fall back to serial GC. } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } @@ -1159,6 +1159,10 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } + + if (ParallelGC.isEnabled()) { + ParallelGC.singleton().cleanupAfterCollection(); + } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 6a6fdc06aa84..757e3e02c324 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -67,6 +67,7 @@ void push(Pointer ptr) { } Pointer pop() { + assert ParallelGC.isInParallelPhase(); ParallelGC.mutex.lock(); try { if (top > 0) { @@ -80,6 +81,11 @@ Pointer pop() { } } + boolean isEmpty() { + assert !ParallelGC.isInParallelPhase(); + return top == 0; + } + void release() { free(buffer, size); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index f57ff42c0f3a..86affe89570a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -59,9 +59,6 @@ public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; - private Thread[] workers; - private int busyWorkers; - /** * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. */ @@ -77,6 +74,8 @@ public class ParallelGC { private final VMCondition parPhase = new VMCondition(mutex); private ChunkBuffer buffer; + private Thread[] workers; + private int busyWorkers; private volatile boolean inParallelPhase; @Fold @@ -105,14 +104,16 @@ public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { public void push(Pointer ptr) { assert ptr.isNonNull(); - buffer.push(ptr); - if (inParallelPhase) { - parPhase.signal(); + if (buffer != null) { + buffer.push(ptr); + if (inParallelPhase) { + parPhase.signal(); + } } } public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { - assert ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); if (chunk.notEqual(scannedChunkTL.get())) { UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); assert scanOffset.aboveThan(0); @@ -182,9 +183,15 @@ private Thread startWorkerThread(int n) { } /** - * Start parallel phase and wait until all chunks have been processed. Used by complete collections. + * Start parallel phase and wait until all chunks have been processed. + * @return false if worker threads have not been started yet. This can happen if GC happens very early + * during application startup. */ - public void waitForIdle() { + public boolean waitForIdle() { + if (workers == null) { + return false; + } + assert allocChunkTL.get().isNonNull(); push(HeapChunk.asPointer(allocChunkTL.get())); @@ -206,11 +213,13 @@ public void waitForIdle() { } finally { mutex.unlock(); } + + assert buffer.isEmpty(); // clean up thread local allocation chunks - allocChunkTL.set(WordFactory.nullPointer()); for (Thread t: workers) { allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); } + return true; } @SuppressWarnings("static-method") @@ -258,10 +267,15 @@ private static GreyToBlackObjectVisitor getVisitor() { return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); } + @SuppressWarnings("static-method") + public void cleanupAfterCollection() { + allocChunkTL.set(WordFactory.nullPointer()); + } + @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) public void tearDown() { buffer.release(); - for (Thread t: workers) { + for (Thread t : workers) { PlatformThreads.exit(t); } } From df092558bdc5f0428eab25a094c9a96448afd7ca Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 14 Mar 2023 13:08:56 +0300 Subject: [PATCH 087/125] Style fixes --- .../oracle/svm/core/genscavenge/GCImpl.java | 3 +- .../SerialAndEpsilonGCOptions.java | 3 +- .../oracle/svm/core/genscavenge/Space.java | 3 +- .../core/genscavenge/parallel/ParallelGC.java | 71 +++++++++---------- .../parallel/ParallelGCOptions.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 2 +- .../graal/snippets/CEntryPointSnippets.java | 3 +- .../com/oracle/svm/core/thread/Safepoint.java | 4 +- .../com/oracle/svm/core/thread/VMThreads.java | 8 +-- 9 files changed, 51 insertions(+), 48 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index eee115fc8d88..5e4249786a0a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1061,7 +1061,8 @@ private void scanGreyObjects(boolean isIncremental) { if (isIncremental) { scanGreyObjectsLoop(); } else if (ParallelGC.isEnabled() && ParallelGC.singleton().waitForIdle()) { - // GC can happen before parallel worker threads have been started. In this case, fall back to serial GC. + // GC can happen before parallel worker threads have been started. In this case, + // fall back to serial GC. } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index 8cc7d3229f42..f1f7d372a8a0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -81,7 +81,8 @@ private SerialAndEpsilonGCOptions() { private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { if (!SubstrateOptions.useMarkAndCopyOrEpsilonGC()) { throw new InterruptImageBuilding( - "The option '" + optionKey.getName() + "' can only be used together with the serial ('--gc=serial'), parallel ('--gc=parallel'), or the epsilon garbage collector ('--gc=epsilon')."); + "The option '" + optionKey.getName() + + "' can only be used together with the serial ('--gc=serial'), parallel ('--gc=parallel'), or the epsilon garbage collector ('--gc=epsilon')."); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 185e885ad33a..b744cae07cea 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -442,7 +442,8 @@ Object copyAlignedObjectParallel(Object original) { // Install forwarding pointer into the original header Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original header + // We have won the race, now we must copy the object bits. First install the original + // header copyMemory.writeWord(hubOffset, originalHeader); // Copy the rest of original object if (hubOffset > 0) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 86affe89570a..ff25c67e4eff 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -60,14 +60,12 @@ public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; /** - * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new chunk needs to be allocated. + * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new + * chunk needs to be allocated. */ - private static final FastThreadLocalWord allocChunkTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); - private static final FastThreadLocalWord scannedChunkTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); - private static final FastThreadLocalWord allocChunkScanOffsetTL = - FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); + private static final FastThreadLocalWord allocChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); + private static final FastThreadLocalWord scannedChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); + private static final FastThreadLocalWord allocChunkScanOffsetTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); @@ -145,36 +143,36 @@ private static int getDefaultWorkerCount() { private Thread startWorkerThread(int n) { Thread t = new Thread(() -> { - VMThreads.SafepointBehavior.useAsParallelGCThread(); - debugLog().string("WW start ").unsigned(n).newline(); - - while (true) { - try { - Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { - mutex.lock(); - try { - if (--busyWorkers == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - debugLog().string("WW idle ").unsigned(n).newline(); - parPhase.block(); - ++busyWorkers; - debugLog().string("WW run ").unsigned(n).newline(); - } finally { - mutex.unlock(); + VMThreads.SafepointBehavior.useAsParallelGCThread(); + debugLog().string("WW start ").unsigned(n).newline(); + + while (true) { + try { + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { + mutex.lock(); + try { + if (--busyWorkers == 0) { + inParallelPhase = false; + seqPhase.signal(); } + debugLog().string("WW idle ").unsigned(n).newline(); + parPhase.block(); + ++busyWorkers; + debugLog().string("WW run ").unsigned(n).newline(); + } finally { + mutex.unlock(); } - - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); - } catch (Throwable ex) { - VMError.shouldNotReachHere(ex); } + + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); + } catch (Throwable ex) { + VMError.shouldNotReachHere(ex); } + } }); t.setName("ParallelGCWorker-" + n); t.setDaemon(true); @@ -184,8 +182,9 @@ private Thread startWorkerThread(int n) { /** * Start parallel phase and wait until all chunks have been processed. - * @return false if worker threads have not been started yet. This can happen if GC happens very early - * during application startup. + * + * @return false if worker threads have not been started yet. This can happen if GC happens very + * early during application startup. */ public boolean waitForIdle() { if (workers == null) { @@ -216,7 +215,7 @@ public boolean waitForIdle() { assert buffer.isEmpty(); // clean up thread local allocation chunks - for (Thread t: workers) { + for (Thread t : workers) { allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); } return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java index 6fe4b04d6ae9..3857cd35abc2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java @@ -34,6 +34,6 @@ public class ParallelGCOptions { - @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User) + @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index af819661c0de..8b4e3967cec3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -329,7 +329,7 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o private static void requireMultiThreading(OptionKey optionKey) { if (!SubstrateOptions.MultiThreaded.getValue()) { throw new InterruptImageBuilding( - "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); + "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index e46003720772..c8506b69a77e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -325,7 +325,8 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } - /* Number of parallel GC workers can be affected by a runtime option, so call this after options are parsed */ + // Number of parallel GC workers can be affected by a runtime option, + // so call this after options are parsed Heap.getHeap().initGC(); boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 7f4c0ea82a20..40591ee0e12e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -761,7 +761,7 @@ private static void waitForSafepoints(String reason) { if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; @@ -907,7 +907,7 @@ public static int countingVMOperation() { if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { ignoreSafepoints += 1; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index c0b1bc3502df..f89f63b671e7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -887,7 +887,7 @@ public static int getSafepointBehaviorVolatile(IsolateThread vmThread) { * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * thread will also actively prevent the VM from reaching a safepoint (regardless of its * thread status). - * + * * NOTE: Be careful with this method and make sure that this thread does not allocate any * Java objects as this could result deadlocks. This method will only work prevent * safepoints reliably if it is called from a thread with @@ -902,7 +902,7 @@ public static void preventSafepoints() { /** * Marks the thread as crashed. This method may only be used in places where it is not * possible to safely detach a thread. - * + * * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * safepoint handling will ignore the thread so that the VM can reach a safepoint regardless * of the status of this thread. @@ -919,8 +919,8 @@ public static void markThreadAsCrashed() { } /** - * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, - * so it must not keep references to movable heap objects on its stack. + * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, so + * it must not keep references to movable heap objects on its stack. */ public static void useAsParallelGCThread() { safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); From 34ad44de4a77eabf28c3537ed9a4a03b0f9021d6 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 13 Mar 2023 16:50:45 +0100 Subject: [PATCH 088/125] Fix after rebasing to master. --- .../src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 167323c0ea64..4bf74c62a9f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -450,7 +450,7 @@ Object installForwardingPointerParallel(Object original, UnsignedWord originalHe boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - forwardHeader = WordFactory.unsigned(0xf0f0f0f0f0f0f0f0L); + forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); hasShift = true; } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); From 35325d64e9dc63ca8e4d44ffd37b2e3d958b321c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 15 Mar 2023 12:40:35 +0100 Subject: [PATCH 089/125] Make the inner-most part of the GC uninterruptible. --- .../genscavenge/AbstractCollectionPolicy.java | 2 + .../core/genscavenge/AlignedHeapChunk.java | 4 + .../genscavenge/BasicCollectionPolicies.java | 3 + .../core/genscavenge/ChunksAccounting.java | 3 + .../core/genscavenge/CollectionPolicy.java | 3 + .../svm/core/genscavenge/GCAccounting.java | 2 + .../oracle/svm/core/genscavenge/GCImpl.java | 4 + .../svm/core/genscavenge/Generation.java | 6 + .../genscavenge/GreyToBlackObjRefVisitor.java | 22 ++++ .../genscavenge/GreyToBlackObjectVisitor.java | 2 + .../svm/core/genscavenge/HeapChunk.java | 4 +- .../core/genscavenge/HeapChunkProvider.java | 29 +---- .../svm/core/genscavenge/HeapParameters.java | 1 + .../core/genscavenge/ObjectHeaderImpl.java | 19 ++- .../svm/core/genscavenge/OldGeneration.java | 8 +- .../ReferenceObjectProcessing.java | 6 + .../oracle/svm/core/genscavenge/Space.java | 29 +++-- .../core/genscavenge/UnalignedHeapChunk.java | 1 + .../svm/core/genscavenge/YoungGeneration.java | 10 +- .../genscavenge/parallel/ChunkBuffer.java | 38 +++--- .../core/genscavenge/parallel/ParallelGC.java | 113 +++++++++--------- .../remset/AlignedChunkRememberedSet.java | 9 ++ .../core/genscavenge/remset/CardTable.java | 6 + .../remset/CardTableBasedRememberedSet.java | 7 ++ .../genscavenge/remset/FirstObjectTable.java | 12 ++ .../genscavenge/remset/NoRememberedSet.java | 8 ++ .../genscavenge/remset/RememberedSet.java | 8 ++ .../remset/UnalignedChunkRememberedSet.java | 6 + .../posix/pthread/PthreadVMLockSupport.java | 1 + .../core/windows/WindowsVMLockSupport.java | 1 + .../oracle/svm/core/config/ObjectLayout.java | 2 + .../heap/InstanceReferenceMapDecoder.java | 4 +- .../oracle/svm/core/heap/ObjectHeader.java | 1 + .../oracle/svm/core/heap/OutOfMemoryUtil.java | 3 + .../svm/core/heap/PodReferenceMapDecoder.java | 7 +- .../svm/core/heap/ReferenceInternals.java | 8 +- .../svm/core/heap/ReferenceMapIndex.java | 3 + .../com/oracle/svm/core/hub/DynamicHub.java | 2 + .../svm/core/hub/InteriorObjRefWalker.java | 7 ++ .../oracle/svm/core/hub/LayoutEncoding.java | 5 + .../svm/core/jdk/UninterruptibleUtils.java | 7 ++ .../locks/SingleThreadedVMLockSupport.java | 1 + .../oracle/svm/core/locks/VMCondition.java | 1 + .../os/AbstractCommittedMemoryProvider.java | 1 + .../com/oracle/svm/core/thread/VMThreads.java | 1 + .../svm/core/util/TypedMemoryReader.java | 5 + 46 files changed, 306 insertions(+), 119 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index bee7b2871e1b..1f8e4b4308fd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -243,6 +243,7 @@ public UnsignedWord getCurrentHeapCapacity() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { assert VMOperation.isGCInProgress() : "use only during GC"; guaranteeSizeParametersInitialized(); @@ -290,6 +291,7 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { assert VMOperation.isGCInProgress() : "use only during GC"; return tenuringThreshold; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index bee8b8cbe57d..1a64269b47af 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -83,6 +83,7 @@ private AlignedHeapChunk() { // all static public interface AlignedHeader extends HeapChunk.Header { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) { HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), chunkSize); } @@ -101,6 +102,7 @@ public static Pointer getObjectsEnd(AlignedHeader that) { } /** Allocate uninitialized memory within this AlignedHeapChunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { Pointer result = WordFactory.nullPointer(); UnsignedWord available = HeapChunk.availableObjectMemory(that); @@ -113,6 +115,7 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { } /** Retract the latest allocation. Used by parallel collector. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); assert newTop.aboveThan(HeapChunk.asPointer(that)); @@ -137,6 +140,7 @@ public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { } /** Return the offset of an object within the objects part of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPointer) { Pointer objectsStart = getObjectsStart(that); return objectPointer.subtract(objectsStart); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java index ae89d3740679..8aa186b7509c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java @@ -32,6 +32,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.heap.ReferenceAccess; @@ -167,6 +168,7 @@ public UnsignedWord getMaximumSurvivorSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { return WordFactory.zero(); } @@ -202,6 +204,7 @@ public final UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { return 1; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java index 17c24b576832..99b3f184cb1c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java @@ -60,6 +60,7 @@ public void reset() { unalignedChunkBytes = WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getChunkBytes() { return getAlignedChunkBytes().add(getUnalignedChunkBytes()); } @@ -117,10 +118,12 @@ private void noteUnaligned(UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { unnoteUnaligned(UnalignedHeapChunk.getCommittedObjectMemory(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void unnoteUnaligned(UnsignedWord size) { unalignedCount--; unalignedChunkBytes = unalignedChunkBytes.subtract(size); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index 1d11025ef0f7..ba738c3df223 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.util.UserError; @@ -176,6 +177,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * survivor-to spaces of all ages. In other words, when copying during a collection, up to 2x * this amount can be used for surviving objects. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) UnsignedWord getSurvivorSpacesCapacity(); /** The capacity of the young generation, comprising the eden and survivor spaces. */ @@ -200,6 +202,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * 1 (straight from eden) and the {@linkplain HeapParameters#getMaxSurvivorSpaces() number of * survivor spaces + 1}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getTenuringAge(); /** Called at the beginning of a collection, in the safepoint operation. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java index 128df06b6087..c006ab6faa54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; /** @@ -154,6 +155,7 @@ void beforeCollection(boolean completeCollection) { /** Called after an object has been promoted from the young generation to the old generation. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void onSurvivorOverflowed() { lastIncrementalCollectionOverflowedSurvivors = true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c8859d924d5f..3dd868593ce1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -535,6 +535,7 @@ public void collectCompletely(GCCause cause) { collect(cause, true); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isCompleteCollection() { return completeCollection; } @@ -1085,6 +1086,7 @@ private static void scanGreyObjectsLoop() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); @@ -1118,6 +1120,7 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); @@ -1234,6 +1237,7 @@ public static boolean hasNeverCollectPolicy() { return getPolicy() instanceof NeverCollect; } + @Fold public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { return greyToBlackObjectVisitor; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java index 35b694891c19..e487a4c7f1d2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java @@ -27,6 +27,8 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; @@ -66,6 +68,8 @@ public String getName() { * promotion was done by copying, or {@code null} if there was insufficient capacity in * this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace); /** @@ -79,6 +83,8 @@ public String getName() { * was promoted through HeapChunk motion, or {@code null} if there was insufficient * capacity in this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 7778ba7e46cc..cad4c249834e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -30,6 +30,7 @@ import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; @@ -57,12 +58,14 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { return visitObjectReferenceInline(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); @@ -128,16 +131,22 @@ public interface Counters extends AutoCloseable { @Override void close(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteObjRef(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNullReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteForwardedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNonHeapReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteCopiedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteUnmodifiedReference(); void toLog(); @@ -182,32 +191,39 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { objRef += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { nullReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { forwardedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { nonHeapReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { copiedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { + // TEMP (chaeubl): this counter would break unmodifiedReference += 1L; } @@ -241,26 +257,32 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 9a8b48845d46..bc432b866261 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; @@ -55,6 +56,7 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 69104f59be47..43df19d7b1fe 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -41,8 +41,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.struct.PinnedObjectField; @@ -177,6 +177,7 @@ public interface Header> extends HeaderPadding { void setIdentityHashSalt(UnsignedWord value, LocationIdentity identity); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(Header chunk, Pointer objectsStart, UnsignedWord chunkSize) { HeapChunk.setEndOffset(chunk, chunkSize); HeapChunk.setTopPointer(chunk, objectsStart); @@ -306,6 +307,7 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 4df6532b8e51..1aeb21c28ad6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.heap.OutOfMemoryUtil; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -35,11 +34,11 @@ import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicUnsigned; import com.oracle.svm.core.log.Log; @@ -89,23 +88,15 @@ public UnsignedWord getBytesInUnusedChunks() { return bytesInUnusedAlignedChunks.get(); } - @AlwaysInline("Remove all logging when noopLog is returned by this method") - private static Log log() { - return Log.noopLog(); - } - private static final OutOfMemoryError ALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an aligned heap chunk"); private static final OutOfMemoryError UNALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an unaligned heap chunk"); /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); - log().string("[HeapChunkProvider.produceAlignedChunk chunk size: ").unsigned(chunkSize).newline(); - AlignedHeader result = popUnusedAlignedChunk(); - log().string(" unused chunk: ").zhex(result).newline(); - if (result.isNull()) { /* Unused list was empty, need to allocate memory. */ noteFirstAllocationTime(); @@ -113,7 +104,6 @@ AlignedHeader produceAlignedChunk() { if (result.isNull()) { throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); } - log().string(" new chunk: ").zhex(result).newline(); AlignedHeapChunk.initialize(result, chunkSize); } @@ -123,8 +113,6 @@ AlignedHeader produceAlignedChunk() { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" result chunk: ").zhex(result).string(" ]").newline(); return result; } @@ -200,13 +188,10 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { if (SubstrateOptions.MultiThreaded.getValue()) { VMThreads.guaranteeOwnsThreadMutex("Should hold the lock when pushing to the global list."); } - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); HeapChunk.setNext(chunk, unusedAlignedChunks.get()); unusedAlignedChunks.set(chunk); bytesInUnusedAlignedChunks.addAndGet(HeapParameters.getAlignedHeapChunkSize()); - - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); } /** @@ -218,15 +203,13 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { * garbage collections, I avoid the ABA problem by making the kernel of this method * uninterruptible so it can not be interrupted by a safepoint. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeader popUnusedAlignedChunk() { - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); - AlignedHeader result = popUnusedAlignedChunkUninterruptibly(); if (result.isNull()) { return WordFactory.nullPointer(); } else { bytesInUnusedAlignedChunks.subtractAndGet(HeapParameters.getAlignedHeapChunkSize()); - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); return result; } } @@ -268,7 +251,6 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) { /** Acquire an UnalignedHeapChunk from the operating system. */ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize); - log().string("[HeapChunkProvider.produceUnalignedChunk objectSize: ").unsigned(objectSize).string(" chunkSize: ").zhex(chunkSize).newline(); noteFirstAllocationTime(); UnalignedHeader result = (UnalignedHeader) CommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); @@ -282,8 +264,6 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" returns ").zhex(result).string(" ]").newline(); return result; } @@ -300,10 +280,10 @@ static void consumeUnalignedChunks(UnalignedHeader firstChunk) { freeUnalignedChunkList(firstChunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void zap(Header chunk, WordBase value) { Pointer start = HeapChunk.getTopPointer(chunk); Pointer limit = HeapChunk.getEndPointer(chunk); - log().string(" zap chunk: ").zhex(chunk).string(" start: ").zhex(start).string(" limit: ").zhex(limit).string(" value: ").zhex(value).newline(); for (Pointer p = start; p.belowThan(limit); p = p.add(FrameAccess.wordSize())) { p.writeWord(0, value); } @@ -332,6 +312,7 @@ boolean walkHeapChunks(MemoryWalker.Visitor visitor) { return continueVisiting; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void noteFirstAllocationTime() { if (firstAllocationTime == 0L) { firstAllocationTime = System.nanoTime(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java index 9c2be3f9a442..e5817b56068d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java @@ -137,6 +137,7 @@ public static UnsignedWord getLargeArrayThreshold() { * Zapping */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean getZapProducedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 4bf74c62a9f3..d097f56a7284 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -363,6 +363,7 @@ public static boolean isAlignedObject(Object o) { return !isUnalignedObject(o); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isAlignedHeader(UnsignedWord header) { return !isUnalignedHeader(header); } @@ -378,12 +379,14 @@ public static boolean isUnalignedHeader(UnsignedWord header) { return header.and(UNALIGNED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setRememberedSetBit(Object o) { UnsignedWord oldHeader = readHeaderFromObject(o); UnsignedWord newHeader = oldHeader.or(REMEMBERED_SET_BIT); writeHeaderToObject(o, newHeader); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean hasRememberedSet(UnsignedWord header) { return header.and(REMEMBERED_SET_BIT).notEqual(0); } @@ -399,14 +402,18 @@ public static boolean isForwardedHeader(UnsignedWord header) { return header.and(FORWARDED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr) { return getForwardedObject(ptr, readHeaderFromPointer(ptr)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // TEMP (chaeubl): + // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (false) { // References compressed with shift have no bits to spare, so the forwarding // reference is stored separately, after the object header ObjectLayout layout = ConfigurationValues.getObjectLayout(); @@ -423,11 +430,14 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // TEMP (chaeubl): + // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (false) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); @@ -444,12 +454,15 @@ void installForwardingPointer(Object original, Object copy) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { // create forwarding header UnsignedWord forwardHeader; boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + // TEMP (chaeubl): + // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (false) { forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); hasShift = true; } else { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 395e39ef03fe..f8d44ad04c42 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -31,8 +31,8 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.remset.RememberedSet; @@ -73,6 +73,7 @@ public boolean walkObjects(ObjectVisitor visitor) { /** Promote an Object to ToSpace if it is not already in ToSpace. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -80,6 +81,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -158,13 +160,13 @@ UnsignedWord getChunkBytes() { return fromBytes.add(toBytes); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") AlignedHeapChunk.AlignedHeader requestAlignedChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { - Log.log().string("[! OldGeneration.requestAlignedChunk: failure to allocate aligned chunk!]"); - throw VMError.shouldNotReachHere("Promotion failure"); + throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } RememberedSet.get().enableRememberedSetForChunk(chunk); return chunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 31b7ca07cdc2..83c2169b227c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -36,6 +36,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectReferenceVisitor; @@ -79,6 +80,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -87,6 +89,8 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header @@ -219,6 +223,7 @@ private static boolean processRememberedRef(Reference dr) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr) { ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); UnsignedWord header = ohi.readHeaderFromPointer(referentAddr); @@ -230,6 +235,7 @@ private static boolean maybeUpdateForwardedReference(Reference dr, Pointer re return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean willSurviveThisCollection(Object obj) { HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); Space space = HeapChunk.getSpace(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index b744cae07cea..0eedcf0381b7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -28,9 +28,6 @@ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; -import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.hub.DynamicHub; import org.graalvm.compiler.word.ObjectAccess; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; @@ -39,14 +36,17 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; @@ -110,6 +110,7 @@ boolean isEdenSpace() { return age == 0; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isYoungSpace() { return age <= HeapParameters.getMaxSurvivorSpaces(); } @@ -123,14 +124,17 @@ public boolean isOldSpace() { return age == (HeapParameters.getMaxSurvivorSpaces() + 1); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getAge() { return age; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getNextAgeForPromotion() { return age + 1; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isFromSpace() { return isFromSpace; } @@ -169,6 +173,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemory(UnsignedWord objectSize) { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { return allocateMemoryParallel(objectSize); @@ -187,6 +192,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemoryParallel(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the thread local allocation chunk. */ @@ -205,14 +211,15 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { * Retract the latest allocation. Used by parallel collector. */ @AlwaysInline("GC performance") - @SuppressWarnings("static-method") - private Pointer retractAllocation(UnsignedWord objectSize) { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); if (newChunk.isNonNull()) { @@ -221,9 +228,10 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk; - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); try { ParallelGC.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); @@ -329,6 +337,7 @@ private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.Unaligne } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; extractUnalignedHeapChunkUninterruptibly(uChunk); @@ -397,6 +406,7 @@ private void setLastUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) /** Promote an aligned Object to this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); @@ -413,6 +423,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); @@ -467,6 +478,7 @@ Object copyAlignedObjectParallel(Object original) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromHeader(Object obj, Word header) { DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(header); int encoding = hub.getLayoutEncoding(); @@ -474,6 +486,7 @@ public static UnsignedWord getSizeFromHeader(Object obj, Word header) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(originalObj); @@ -549,11 +562,12 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } /** Promote an UnalignedHeapChunk by moving it to this Space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { originalSpace.extractUnalignedHeapChunk(chunk); @@ -577,6 +591,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index fd405abfe1a2..1117387771ca 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -145,6 +145,7 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index f25fbcf3fcd6..d80e2bf5024c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -29,8 +29,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.heap.ObjectVisitor; @@ -237,6 +237,7 @@ UnsignedWord computeSurvivorObjectBytes() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") public boolean contains(Object object) { if (!HeapImpl.usesImageHeapCardMarking()) { @@ -250,6 +251,7 @@ public boolean contains(Object object) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -265,6 +267,7 @@ protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedH } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -304,17 +307,22 @@ private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { return unalignedChunkFitsInSurvivors((UnalignedHeapChunk.UnalignedHeader) chunk); } + // TEMP (chaeubl): this is problematic but it isn't called or? + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean alignedChunkFitsInSurvivors() { UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(HeapParameters.getAlignedHeapChunkSize()); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + // TEMP (chaeubl): this is problematic but it isn't called or? + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean unalignedChunkFitsInSurvivors(UnalignedHeapChunk.UnalignedHeader chunk) { UnsignedWord size = UnalignedHeapChunk.getCommittedObjectMemory(chunk); UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(size); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeapChunk.AlignedHeader requestAlignedSurvivorChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; if (!alignedChunkFitsInSurvivors()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 757e3e02c324..0f00e4e1d4e5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -26,14 +26,15 @@ package com.oracle.svm.core.genscavenge.parallel; -import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.UnmanagedMemoryUtil; -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.os.CommittedMemoryProvider; import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; + /** * Synchronized buffer that stores "grey" heap chunks to be scanned. */ @@ -54,21 +55,23 @@ static int wordSize() { this.buffer = malloc(this.size); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void push(Pointer ptr) { - assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.isOwner(); + assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.hasOwner(); if (top >= size) { int oldSize = size; size *= 2; assert top < size; - buffer = realloc(buffer, oldSize, size); + buffer = realloc(buffer, size); } buffer.writeWord(top, ptr); top += wordSize(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Pointer pop() { assert ParallelGC.isInParallelPhase(); - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); try { if (top > 0) { top -= wordSize(); @@ -87,24 +90,21 @@ boolean isEmpty() { } void release() { - free(buffer, size); + free(buffer); } - @AlwaysInline("GC performance") private static Pointer malloc(int bytes) { - return CommittedMemoryProvider.get().allocateUnalignedChunk(WordFactory.unsigned(bytes)); + // TEMP (chaeubl): needs proper error handling + return ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(bytes)); } - @AlwaysInline("GC performance") - private static Pointer realloc(Pointer orig, int origSize, int newSize) { - Pointer ptr = malloc(newSize); - UnmanagedMemoryUtil.copyLongsForward(orig, ptr, WordFactory.unsigned(origSize)); - free(orig, origSize); - return ptr; + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer realloc(Pointer orig, int newSize) { + // TEMP (chaeubl): needs proper error handling + return ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(orig, WordFactory.unsigned(newSize)); } - @AlwaysInline("GC performance") - private static void free(Pointer ptr, int bytes) { - CommittedMemoryProvider.get().freeUnalignedChunk(ptr, WordFactory.unsigned(bytes)); + private static void free(Pointer ptr) { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index ff25c67e4eff..443c4f63a203 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -26,6 +26,16 @@ package com.oracle.svm.core.genscavenge.parallel; +import java.util.stream.IntStream; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; @@ -33,7 +43,6 @@ import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; -import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.jdk.Jvm; @@ -45,15 +54,6 @@ import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; import com.oracle.svm.core.threadlocal.FastThreadLocalWord; import com.oracle.svm.core.util.VMError; -import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; - -import java.util.stream.IntStream; public class ParallelGC { @@ -86,20 +86,24 @@ public static boolean isEnabled() { return SubstrateOptions.UseParallelGC.getValue(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isInParallelPhase() { return singleton().inParallelPhase; } + @Uninterruptible(reason = "Called from a GC worker thread.") public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { return allocChunkTL.get(); } + @Uninterruptible(reason = "Called from a GC worker thread.") public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { assert chunk.isNonNull(); allocChunkTL.set(chunk); allocChunkScanOffsetTL.set(AlignedHeapChunk.getObjectsStartOffset()); } + @Uninterruptible(reason = "Called from a GC worker thread.") public void push(Pointer ptr) { assert ptr.isNonNull(); if (buffer != null) { @@ -110,6 +114,7 @@ public void push(Pointer ptr) { } } + @Uninterruptible(reason = "Called from a GC worker thread.") public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); if (chunk.notEqual(scannedChunkTL.get())) { @@ -142,44 +147,43 @@ private static int getDefaultWorkerCount() { } private Thread startWorkerThread(int n) { - Thread t = new Thread(() -> { - VMThreads.SafepointBehavior.useAsParallelGCThread(); - debugLog().string("WW start ").unsigned(n).newline(); - - while (true) { - try { - Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { - mutex.lock(); - try { - if (--busyWorkers == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - debugLog().string("WW idle ").unsigned(n).newline(); - parPhase.block(); - ++busyWorkers; - debugLog().string("WW run ").unsigned(n).newline(); - } finally { - mutex.unlock(); - } - } - - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); - } catch (Throwable ex) { - VMError.shouldNotReachHere(ex); - } - } - }); + Thread t = new Thread(this::work); t.setName("ParallelGCWorker-" + n); t.setDaemon(true); t.start(); return t; } + @Uninterruptible(reason = "Called from a GC worker thread.") + private void work() { + VMThreads.SafepointBehavior.useAsParallelGCThread(); + while (true) { + try { + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { + mutex.lockNoTransitionUnspecifiedOwner(); + try { + if (--busyWorkers == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + parPhase.blockNoTransition(); + ++busyWorkers; + } finally { + mutex.unlock(); + } + } + + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); + } catch (Throwable ex) { + VMError.shouldNotReachHere(ex); + } + } + } + /** * Start parallel phase and wait until all chunks have been processed. * @@ -221,33 +225,31 @@ public boolean waitForIdle() { return true; } - @SuppressWarnings("static-method") - private void scanChunk(Pointer ptr) { + @Uninterruptible(reason = "Called from a GC worker thread.") + private static void scanChunk(Pointer ptr) { if (ptr.isNonNull()) { - debugLog().string("WW scan chunk=").zhex(ptr).newline(); if (ptr.and(UNALIGNED_BIT).notEqual(0)) { - UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), getVisitor()); + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); } else { Pointer start = ptr; AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); if (chunk.equal(ptr)) { start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); } - HeapChunk.walkObjectsFromInline(chunk, start, getVisitor()); + HeapChunk.walkObjectsFromInline(chunk, start, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); } } } - @SuppressWarnings("static-method") - private void scanAllocChunk() { + @Uninterruptible(reason = "Called from a GC worker thread.") + private static void scanAllocChunk() { if (allocChunkNeedsScanning()) { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); assert scanOffset.aboveThan(0); Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); - debugLog().string("WW scan alloc=").zhex(allocChunk).string(" from offset ").unsigned(scanOffset).newline(); scannedChunkTL.set(allocChunk); - HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, getVisitor()); + HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); scannedChunkTL.set(WordFactory.nullPointer()); if (allocChunkTL.get().equal(allocChunk)) { // remember top offset so that we don't scan the same objects again @@ -256,16 +258,12 @@ private void scanAllocChunk() { } } - @SuppressWarnings("static-method") - private boolean allocChunkNeedsScanning() { + @Uninterruptible(reason = "Called from a GC worker thread.") + private static boolean allocChunkNeedsScanning() { AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); } - private static GreyToBlackObjectVisitor getVisitor() { - return GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); - } - @SuppressWarnings("static-method") public void cleanupAfterCollection() { allocChunkTL.set(WordFactory.nullPointer()); @@ -283,6 +281,7 @@ private static Log verboseGCLog() { return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); } + @Uninterruptible(reason = "Called from a GC worker thread.") static Log debugLog() { return Log.noopLog(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index d244d47c7cb3..b03aa738fdca 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -37,6 +37,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -86,6 +87,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -95,6 +97,7 @@ public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(AlignedHeader chunk) { // Completely clean the card table and the first object table as further objects may be // added later on to this chunk. @@ -118,6 +121,7 @@ public static void clearRememberedSet(AlignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object object, boolean verifyOnly) { Pointer objectPointer = Word.objectToUntrackedPointer(object); AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(objectPointer); @@ -203,6 +207,7 @@ public static boolean verify(AlignedHeader chunk) { } /** Return the index of an object within the tables of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getObjectIndex(AlignedHeader chunk, Pointer objectPointer) { UnsignedWord offset = AlignedHeapChunk.getObjectOffset(chunk, objectPointer); return CardTable.memoryOffsetToIndex(offset); @@ -260,18 +265,22 @@ static UnsignedWord getCardTableLimitOffset() { return UnsignedUtils.roundUp(tableLimit, alignment); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(AlignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(AlignedHeader chunk) { return getFirstObjectTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(Pointer chunk) { return chunk.add(getFirstObjectTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index fa707695db94..f5421338675e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapImpl; @@ -83,10 +84,12 @@ final class CardTable { private CardTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void cleanTable(Pointer tableStart, UnsignedWord size) { UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setDirty(Pointer table, UnsignedWord index) { byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); // Using a likely probability should typically avoid placing the write below at a separate @@ -100,6 +103,7 @@ public static void setClean(Pointer table, UnsignedWord index) { table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isDirty(Pointer table, UnsignedWord index) { int entry = readEntry(table, index); return entry == DIRTY_ENTRY; @@ -110,10 +114,12 @@ private static boolean isClean(Pointer table, UnsignedWord index) { return entry == CLEAN_ENTRY; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int readEntry(Pointer table, UnsignedWord index) { return table.readByte(index); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 40ec1153d282..04281e745f20 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -32,6 +32,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -86,17 +87,20 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { AlignedChunkRememberedSet.enableRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { UnalignedChunkRememberedSet.enableRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj); } @@ -107,12 +111,14 @@ public void clearRememberedSet(AlignedHeader chunk) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { UnalignedChunkRememberedSet.clearRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return ObjectHeaderImpl.hasRememberedSet(header); } @@ -131,6 +137,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { if (holderObject == null || object == null) { return; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 2c5f3ff3b141..3f5136bf6e54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -30,6 +30,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.hub.LayoutEncoding; @@ -163,6 +164,7 @@ final class FirstObjectTable { private FirstObjectTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initializeTable(Pointer table, UnsignedWord size) { if (SubstrateUtil.HOSTED) { // Initialize this table unconditionally as this simplifies a few things. @@ -173,12 +175,14 @@ public static void initializeTable(Pointer table, UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean doInitializeTable(Pointer table, UnsignedWord size) { UnmanagedMemoryUtil.fill(table, size, (byte) UNINITIALIZED_ENTRY); return true; } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setTableForObject(Pointer table, UnsignedWord startOffset, UnsignedWord endOffset) { assert startOffset.belowThan(endOffset); UnsignedWord startIndex = memoryOffsetToIndex(startOffset); @@ -328,15 +332,18 @@ private static UnsignedWord getTableSizeForMemoryRange(Pointer memoryStart, Poin * The multiplier from memory offsets to byte offsets into the previous card. This is the * granularity to which I can point to the start of an object. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetScale() { return ConfigurationValues.getObjectLayout().getAlignment(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getEntryAtIndex(Pointer table, UnsignedWord index) { return table.readByte(indexToTableOffset(index)); } /** Set the table entry at a given index. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void setEntryAtIndex(Pointer table, UnsignedWord index, int value) { assert isValidEntry(value) : "Invalid entry"; assert isUninitializedIndex(table, index) || getEntryAtIndex(table, index) == value : "Overwriting!"; @@ -363,6 +370,7 @@ private static boolean isMemoryOffsetEntry(int entry) { return MEMORY_OFFSET_MIN <= entry && entry <= MEMORY_OFFSET_MAX; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int biasExponent(int exponent) { assert EXPONENT_MIN <= exponent && exponent <= EXPONENT_MAX : "Exponent out of bounds."; return exponent + EXPONENT_BIAS; @@ -379,18 +387,22 @@ private static UnsignedWord exponentToOffset(int n) { return WordFactory.unsigned(1L << n); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToTableOffset(UnsignedWord index) { return index.multiply(ENTRY_SIZE_BYTES); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToMemoryOffset(UnsignedWord index) { return index.multiply(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetToEntry(UnsignedWord memoryOffset) { assert memoryOffset.belowThan(BYTES_COVERED_BY_ENTRY) : "Offset out of bounds."; UnsignedWord scaledOffset = memoryOffset.unsignedDivide(memoryOffsetScale()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index 8fec4384578c..d0f911951ca4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -34,6 +34,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -85,16 +86,20 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { // Nothing to do. } @Override + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { // Nothing to do. } @@ -105,12 +110,14 @@ public void clearRememberedSet(AlignedHeader chunk) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { // Nothing to do. } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return false; } @@ -127,6 +134,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { // Nothing to do. } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java index d6b1725a2967..f2d4e74e00fa 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.Space; @@ -76,28 +77,34 @@ static RememberedSet get() { * Enables remembered set tracking for an aligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(AlignedHeader chunk); /** * Enables remembered set tracking for an unaligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(UnalignedHeader chunk); /** * Enables remembered set tracking for a single object in an aligned chunk. Must be called when * an object is added to the image heap or old generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForObject(AlignedHeader chunk, Object obj); /** Clears the remembered set of an aligned chunk. */ void clearRememberedSet(AlignedHeader chunk); /** Clears the remembered set of an unaligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void clearRememberedSet(UnalignedHeader chunk); /** Checks if remembered set tracking is enabled for an object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean hasRememberedSet(UnsignedWord header); /** @@ -122,6 +129,7 @@ static RememberedSet get() { * tracking is enabled. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void dirtyCardIfNecessary(Object holderObject, Object object); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 1e993fb3f30f..1ee648709ae3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; @@ -59,6 +60,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk) { // The remembered set bit in the header will be set by the code that writes the objects. } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); // Unaligned chunks don't have a first object table. @@ -67,6 +69,7 @@ public static void enableRememberedSet(UnalignedHeader chunk) { ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -75,6 +78,7 @@ public static void clearRememberedSet(UnalignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object obj, boolean verifyOnly) { UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunk(obj); Pointer cardTableStart = getCardTableStart(chunk); @@ -132,10 +136,12 @@ static UnsignedWord getObjectIndex() { return WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(UnalignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java index b588e0172475..132a8f5d2186 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java @@ -367,6 +367,7 @@ private static long remainingNanos(long waitNanos, long startNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { PthreadVMLockSupport.checkResult(Pthread.pthread_cond_signal(getStructPointer()), "pthread_cond_signal"); } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java index c0e97bdb7d1d..5d85820652b1 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java @@ -350,6 +350,7 @@ public long blockNoTransition(long waitNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { Process.NoTransitions.WakeConditionVariable(getStructPointer()); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java index febd8215da9b..abf3f93fc044 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java @@ -75,11 +75,13 @@ public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int ob } /** The minimum alignment of objects (instances and arrays). */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getAlignment() { return objectAlignment; } /** Tests if the given offset or address is aligned according to {@link #getAlignment()}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAligned(final long value) { return (value % getAlignment() == 0L); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java index c203d92c0fe2..af810ff48722 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java @@ -29,15 +29,17 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.util.DuplicatedInNativeCode; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.NonmovableByteArrayReader; import com.oracle.svm.core.util.TypedMemoryReader; @DuplicatedInNativeCode public class InstanceReferenceMapDecoder { @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArray referenceMapEncoding, long referenceMapIndex, ObjectReferenceVisitor visitor, Object holderObject) { assert ReferenceMapIndex.denotesValidReferenceMap(referenceMapIndex); assert referenceMapEncoding.isNonNull(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java index 5e83ff7a0932..4c3f686e0351 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java @@ -63,6 +63,7 @@ protected ObjectHeader() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public abstract DynamicHub dynamicHubFromObjectHeader(Word header); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static DynamicHub readDynamicHubFromObject(Object o) { return KnownIntrinsics.readHub(o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 86fb9c99d2d3..aa11bdddb98e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.JDKUtils; import com.oracle.svm.core.log.Log; @@ -38,6 +39,8 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } + // TEMP (chaeubl): + @Uninterruptible(reason = "TEMP (chaeubl)", calleeMustBe = false) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index bd10eb301a5a..9a827a3b9c3b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -34,16 +34,19 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.JavaMemoryUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.graal.nodes.NewPodInstanceNode; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.UnsignedUtils; public final class PodReferenceMapDecoder { @DuplicatedInNativeCode @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) { int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); @@ -55,8 +58,8 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco int gap; do { mapOffset = mapOffset.subtract(2); - gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); - nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); + gap = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); + nrefs = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); for (int i = 0; i < nrefs; i++) { if (!visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj)) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java index 68605e09b182..f0c879cd8387 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java @@ -69,6 +69,7 @@ static Reference uncast(Target_java_lang_ref_Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#referent} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset))); } @@ -79,7 +80,7 @@ public static T getReferent(Reference instance) { } /** Write {@link Target_java_lang_ref_Reference#referent}. */ - @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setReferent(Reference instance, Object value) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), value); } @@ -105,6 +106,7 @@ public static void clear(Reference instance) { ObjectAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), null); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentFieldAddress(Reference instance) { return Word.objectToUntrackedPointer(instance).add(WordFactory.unsigned(Target_java_lang_ref_Reference.referentFieldOffset)); } @@ -119,6 +121,7 @@ public static Reference getNextDiscovered(Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#discovered} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getDiscoveredPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset))); } @@ -140,6 +143,7 @@ public static boolean isAnyReferenceFieldOffset(long offset) { } /** Write {@link Target_java_lang_ref_Reference#discovered}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setNextDiscovered(Reference instance, Reference newNext) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset), newNext); } @@ -251,6 +255,7 @@ public static boolean waitForReferenceProcessing() throws InterruptedException { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceClock() { return Target_java_lang_ref_SoftReference.clock; } @@ -262,6 +267,7 @@ public static void updateSoftReferenceClock() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceTimestamp(SoftReference instance) { Target_java_lang_ref_SoftReference ref = SubstrateUtil.cast(instance, Target_java_lang_ref_SoftReference.class); return ref.timestamp; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java index 4989c633cbbe..e80b89687eeb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + public class ReferenceMapIndex { /** * Marker value returned by @@ -44,6 +46,7 @@ public static boolean denotesEmptyReferenceMap(long referenceMapIndex) { return referenceMapIndex == EMPTY_REFERENCE_MAP || referenceMapIndex == NO_REFERENCE_MAP; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean denotesValidReferenceMap(long referenceMapIndex) { return referenceMapIndex != NO_REFERENCE_MAP; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index e4c4e2d9a431..c3fa24fa8ab8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -634,6 +634,7 @@ public DynamicHub getArrayHub() { return arrayHub; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getReferenceMapIndex() { return referenceMapIndex; } @@ -678,6 +679,7 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getHubType() { return hubType; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index fe2a8f6c4509..0aeff0c573fc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.InstanceReferenceMapDecoder; @@ -66,6 +67,7 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final Pointer objPointer = Word.objectToUntrackedPointer(obj); @@ -107,6 +109,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { NonmovableArray referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding(); long referenceMapIndex = objHub.getReferenceMapIndex(); @@ -116,6 +119,7 @@ private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { if (!Pod.RuntimeSupport.isPresent()) { throw VMError.shouldNotReachHere("Pod objects cannot be in the heap if the pod support is disabled."); @@ -124,6 +128,7 @@ private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, Dynam } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) { if (!Continuation.isSupported()) { throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); @@ -132,11 +137,13 @@ private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkOther() { throw VMError.shouldNotReachHere("Unexpected object with hub type 'other' in the heap."); } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { int length = ArrayLengthNode.arrayLength(obj); int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 9931f360ea54..61cc8c8b458f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -329,16 +329,19 @@ public static UnsignedWord getMomentarySizeFromObject(Object obj) { return getSizeFromObject(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInGC(Object obj) { return getSizeFromObjectInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj) { return getSizeFromObjectInlineInGC(obj, false); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOptionalIdHashField) { boolean withOptionalIdHashField = addOptionalIdHashField || (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && checkOptionalIdentityHashField(obj)); @@ -371,11 +374,13 @@ private static boolean checkOptionalIdentityHashField(Object obj) { return oh.hasOptionalIdentityHashField(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInGC(Object obj) { return getObjectEndInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInlineInGC(Object obj) { UnsignedWord size = getSizeFromObjectInlineInGC(obj, false); return Word.objectToUntrackedPointer(obj).add(size); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java index edb8c09bb17b..838d452afd34 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java @@ -435,6 +435,13 @@ public static long abs(long a) { } } + public static class Byte { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static int toUnsignedInt(byte x) { + return ((int) x) & 0xff; + } + } + public static class Long { /** Uninterruptible version of {@link java.lang.Long#numberOfLeadingZeros(long)}. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java index e20eafb49af8..f048ec19f3a4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java @@ -177,6 +177,7 @@ public long blockNoTransition(long nanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { /* Nothing to do. */ } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java index 6a5872171303..f85d789026dd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java @@ -98,6 +98,7 @@ public void blockNoTransitionUnspecifiedOwner() { /** * Wakes up a single thread that is waiting on this condition. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { throw VMError.shouldNotReachHere("VMCondition cannot be used during native image generation"); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index f381b6ed1b4d..9e2be0270d71 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -103,6 +103,7 @@ public int protect(PointerBase start, UnsignedWord nbytes, EnumSet acces } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { return allocate(nbytes, alignment, false); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 2833afb1377b..326de8007910 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -921,6 +921,7 @@ public static void markThreadAsCrashed() { * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, so * it must not keep references to movable heap objects on its stack. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void useAsParallelGCThread() { safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java index 22f004c74e27..8bc6c682604d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java @@ -26,6 +26,9 @@ import org.graalvm.word.Pointer; +import com.oracle.svm.core.Uninterruptible; + +// TEMP (chaeubl): this class could be removed (e.g., combined with native coder) public class TypedMemoryReader { public static int getS1(Pointer ptr) { return ptr.readByte(0); @@ -35,6 +38,7 @@ public static int getS2(Pointer ptr) { return ptr.readShort(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static int getS4(Pointer ptr) { return ptr.readInt(0); } @@ -51,6 +55,7 @@ public static int getU2(Pointer ptr) { return getS2(ptr) & 0xFFFF; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getU4(Pointer ptr) { return getS4(ptr) & 0xFFFFFFFFL; } From 5b16c2910b7cf072370ecf9669a717e0ed978b75 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 15 Mar 2023 17:19:49 +0100 Subject: [PATCH 090/125] Use unattached threads as GC worker threads. --- .../oracle/svm/core/genscavenge/GCImpl.java | 11 +- .../genscavenge/GreyToBlackObjRefVisitor.java | 1 - .../oracle/svm/core/genscavenge/HeapImpl.java | 11 +- .../core/genscavenge/ObjectHeaderImpl.java | 6 +- .../oracle/svm/core/genscavenge/Space.java | 6 +- .../svm/core/genscavenge/YoungGeneration.java | 2 - .../genscavenge/parallel/ChunkBuffer.java | 31 +- .../core/genscavenge/parallel/ParallelGC.java | 280 ++++++++++++------ .../parallel/ParallelGCOptions.java | 39 --- .../posix/thread/PosixPlatformThreads.java | 30 ++ .../svm/core/IsolateArgumentParser.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 5 +- .../graal/snippets/CEntryPointSnippets.java | 4 - .../src/com/oracle/svm/core/heap/Heap.java | 2 - .../oracle/svm/core/heap/OutOfMemoryUtil.java | 4 +- .../svm/core/thread/PlatformThreads.java | 33 +++ .../com/oracle/svm/core/thread/Safepoint.java | 6 +- .../com/oracle/svm/core/thread/VMThreads.java | 14 - .../svm/core/util/TypedMemoryReader.java | 1 - 19 files changed, 290 insertions(+), 198 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 3dd868593ce1..f6de4b5379f2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -146,13 +146,6 @@ public void collect(GCCause cause) { collect(cause, false); } - @SuppressWarnings("static-method") - public void initialize() { - if (ParallelGC.isEnabled()) { - ParallelGC.singleton().startWorkerThreads(); - } - } - public void maybeCollectOnAllocation() { boolean outOfMemory = false; if (hasNeverCollectPolicy()) { @@ -208,6 +201,10 @@ private void collectOperation(CollectionVMOperationData data) { assert VMOperation.isGCInProgress() : "Collection should be a VMOperation."; assert getCollectionEpoch().equal(data.getRequestingEpoch()); + if (SubstrateOptions.UseParallelGC.getValue()) { + ParallelGC.singleton().initialize(); + } + timers.mutator.closeAt(data.getRequestingNanoTime()); startCollectionOrExit(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index cad4c249834e..cd13fb0e5989 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -223,7 +223,6 @@ public void noteCopiedReferent() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { - // TEMP (chaeubl): this counter would break unmodifiedReference += 1L; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 76d81c9d14ec..538d223704be 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -42,17 +42,16 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateDiagnostics; import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunk; import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunkRegistry; import com.oracle.svm.core.SubstrateDiagnostics.ErrorContext; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor; @@ -70,6 +69,7 @@ import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.heap.ReferenceInternals; +import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicReference; import com.oracle.svm.core.locks.VMCondition; @@ -242,11 +242,6 @@ GCImpl getGCImpl() { return gcImpl; } - @Override - public void initGC() { - gcImpl.initialize(); - } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAllocationDisallowed() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index d097f56a7284..5a5ffaee2744 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -411,7 +411,7 @@ Object getForwardedObject(Pointer ptr) { Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TEMP (chaeubl): + // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { // References compressed with shift have no bits to spare, so the forwarding @@ -435,7 +435,7 @@ void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); UnsignedWord forwardHeader; if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TEMP (chaeubl): + // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { // Compression with a shift uses all bits of a reference, so store the forwarding @@ -460,7 +460,7 @@ Object installForwardingPointerParallel(Object original, UnsignedWord originalHe UnsignedWord forwardHeader; boolean hasShift = false; if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TEMP (chaeubl): + // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 0eedcf0381b7..9afe74af5c4a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -196,7 +196,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { private Pointer allocateMemoryParallel(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the thread local allocation chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); if (oldChunk.isNonNull()) { result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); } @@ -214,7 +214,7 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer retractAllocation(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.getAllocationChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); assert oldChunk.isNonNull(); return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } @@ -239,7 +239,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu ParallelGC.mutex.unlock(); } if (newChunk.isNonNull()) { - ParallelGC.setAllocationChunk(newChunk); + ParallelGC.singleton().setAllocationChunk(newChunk); return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } return WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index d80e2bf5024c..6a73b2e98912 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -307,14 +307,12 @@ private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { return unalignedChunkFitsInSurvivors((UnalignedHeapChunk.UnalignedHeader) chunk); } - // TEMP (chaeubl): this is problematic but it isn't called or? @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean alignedChunkFitsInSurvivors() { UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(HeapParameters.getAlignedHeapChunkSize()); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } - // TEMP (chaeubl): this is problematic but it isn't called or? @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean unalignedChunkFitsInSurvivors(UnalignedHeapChunk.UnalignedHeader chunk) { UnsignedWord size = UnalignedHeapChunk.getCommittedObjectMemory(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 0f00e4e1d4e5..9abcf88dea44 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -28,6 +28,8 @@ import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -50,9 +52,15 @@ static int wordSize() { return ConfigurationValues.getTarget().wordSize; } + @Platforms(Platform.HOSTED_ONLY.class) ChunkBuffer() { + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void initialize() { this.size = INITIAL_SIZE; - this.buffer = malloc(this.size); + // TODO (petermz): needs proper error handling + this.buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(this.size)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -62,7 +70,8 @@ void push(Pointer ptr) { int oldSize = size; size *= 2; assert top < size; - buffer = realloc(buffer, size); + // TODO (petermz): needs proper error handling + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, WordFactory.unsigned(size)); } buffer.writeWord(top, ptr); top += wordSize(); @@ -89,22 +98,8 @@ boolean isEmpty() { return top == 0; } - void release() { - free(buffer); - } - - private static Pointer malloc(int bytes) { - // TEMP (chaeubl): needs proper error handling - return ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(bytes)); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static Pointer realloc(Pointer orig, int newSize) { - // TEMP (chaeubl): needs proper error handling - return ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(orig, WordFactory.unsigned(newSize)); - } - - private static void free(Pointer ptr) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + void release() { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 443c4f63a203..59f58f1b1c34 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -26,56 +26,73 @@ package com.oracle.svm.core.genscavenge.parallel; -import java.util.stream.IntStream; +import java.util.function.BooleanSupplier; import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Isolate; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.IsolateArgumentParser; +import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.PlatformThreads; -import com.oracle.svm.core.thread.VMThreads; -import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; -import com.oracle.svm.core.threadlocal.FastThreadLocalWord; +import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandle; +import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandlePointer; +import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; import com.oracle.svm.core.util.VMError; public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; - /** - * Each GC worker allocates memory in its own thread local chunk, entering mutex only when new - * chunk needs to be allocated. - */ - private static final FastThreadLocalWord allocChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkTL"); - private static final FastThreadLocalWord scannedChunkTL = FastThreadLocalFactory.createWord("ParallelGCImpl.scannedChunkTL"); - private static final FastThreadLocalWord allocChunkScanOffsetTL = FastThreadLocalFactory.createWord("ParallelGCImpl.allocChunkScanOffsetTL"); + private final CEntryPointLiteral gcWorkerRunFunc; public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); - - private ChunkBuffer buffer; - private Thread[] workers; - private int busyWorkers; + private final ChunkBuffer buffer = new ChunkBuffer(); + + private boolean initialized; + private OSThreadHandlePointer workerThreads; + private GCWorkerThreadState workerStates; + private int numWorkerThreads; + private volatile int busyWorkerThreads; + private ThreadLocalKey workerStateTL; private volatile boolean inParallelPhase; + @Platforms(Platform.HOSTED_ONLY.class) + public ParallelGC() { + gcWorkerRunFunc = CEntryPointLiteral.create(ParallelGC.class, "gcWorkerRun", GCWorkerThreadState.class); + } + @Fold public static ParallelGC singleton() { return ImageSingletons.lookup(ParallelGC.class); @@ -92,15 +109,17 @@ public static boolean isInParallelPhase() { } @Uninterruptible(reason = "Called from a GC worker thread.") - public static AlignedHeapChunk.AlignedHeader getAllocationChunk() { - return allocChunkTL.get(); + public AlignedHeapChunk.AlignedHeader getAllocationChunk() { + GCWorkerThreadState state = getWorkerThreadState(); + return state.getAllocChunk(); } @Uninterruptible(reason = "Called from a GC worker thread.") - public static void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { + public void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { assert chunk.isNonNull(); - allocChunkTL.set(chunk); - allocChunkScanOffsetTL.set(AlignedHeapChunk.getObjectsStartOffset()); + GCWorkerThreadState state = getWorkerThreadState(); + state.setAllocChunk(chunk); + state.setAllocChunkScanOffset(AlignedHeapChunk.getObjectsStartOffset()); } @Uninterruptible(reason = "Called from a GC worker thread.") @@ -117,8 +136,9 @@ public void push(Pointer ptr) { @Uninterruptible(reason = "Called from a GC worker thread.") public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); - if (chunk.notEqual(scannedChunkTL.get())) { - UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + GCWorkerThreadState state = getWorkerThreadState(); + if (chunk.notEqual(state.getScannedChunk())) { + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); assert scanOffset.aboveThan(0); if (chunk.getTopOffset().aboveThan(scanOffset)) { push(HeapChunk.asPointer(chunk).add(scanOffset)); @@ -126,61 +146,95 @@ public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { } } - public void startWorkerThreads() { - buffer = new ChunkBuffer(); - int workerCount = getWorkerCount(); - busyWorkers = workerCount; - workers = IntStream.range(0, workerCount).mapToObj(this::startWorkerThread).toArray(Thread[]::new); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private GCWorkerThreadState getWorkerThreadState() { + if (CurrentIsolate.getCurrentThread().isNull()) { + return PlatformThreads.singleton().getUnmanagedThreadLocalValue(workerStateTL); + } + return workerStates.addressOf(numWorkerThreads); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void initialize() { + if (initialized) { + return; + } + + initialized = true; + buffer.initialize(); + + workerStateTL = PlatformThreads.singleton().createUnmanagedThreadLocal(); + + numWorkerThreads = getWorkerCount(); + busyWorkerThreads = numWorkerThreads; + + /* Allocate one struct per worker thread and one struct for the main GC thread. */ + int numWorkerStates = numWorkerThreads + 1; + // TODO (petermz): add error handling for allocation + workerStates = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(SizeOf.unsigned(GCWorkerThreadState.class).multiply(numWorkerStates)); + for (int i = 0; i < numWorkerStates; i++) { + workerStates.addressOf(i).setIsolate(CurrentIsolate.getIsolate()); + } + + // TODO (petermz): add error handling for allocation + workerThreads = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(OSThreadHandlePointer.class).multiply(numWorkerThreads)); + for (int i = 0; i < numWorkerThreads; i++) { + OSThreadHandle thread = PlatformThreads.singleton().startThreadUnmanaged(gcWorkerRunFunc.getFunctionPointer(), workerStates.addressOf(i), 0); + workerThreads.write(i, thread); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getWorkerCount() { - int setting = ParallelGCOptions.ParallelGCThreads.getValue(); + int index = IsolateArgumentParser.getOptionIndex(SubstrateOptions.ParallelGCThreads); + int setting = IsolateArgumentParser.getIntOptionValue(index); int count = setting > 0 ? setting : getDefaultWorkerCount(); - verboseGCLog().string("[Number of ParallelGC workers: ").unsigned(count).string("]").newline(); return count; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getDefaultWorkerCount() { // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() int cpus = Jvm.JVM_ActiveProcessorCount(); return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; } - private Thread startWorkerThread(int n) { - Thread t = new Thread(this::work); - t.setName("ParallelGCWorker-" + n); - t.setDaemon(true); - t.start(); - return t; + @Uninterruptible(reason = "Heap base is not set up yet.") + @CEntryPoint(include = UseParallelGC.class, publishAs = CEntryPoint.Publish.NotPublished) + @CEntryPointOptions(prologue = GCWorkerThreadPrologue.class, epilogue = CEntryPointOptions.NoEpilogue.class) + private static void gcWorkerRun(GCWorkerThreadState state) { + try { + ParallelGC.singleton().work(state); + } catch (Throwable e) { + throw VMError.shouldNotReachHere(e); + } } + @NeverInline("Prevent reads from floating up.") @Uninterruptible(reason = "Called from a GC worker thread.") - private void work() { - VMThreads.SafepointBehavior.useAsParallelGCThread(); + private void work(GCWorkerThreadState state) { + PlatformThreads.singleton().setUnmanagedThreadLocalValue(workerStateTL, state); + while (true) { - try { - Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning()) { - mutex.lockNoTransitionUnspecifiedOwner(); - try { - if (--busyWorkers == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - parPhase.blockNoTransition(); - ++busyWorkers; - } finally { - mutex.unlock(); + Pointer ptr; + while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { + mutex.lockNoTransitionUnspecifiedOwner(); + try { + if (--busyWorkerThreads == 0) { + inParallelPhase = false; + seqPhase.signal(); } + parPhase.blockNoTransition(); + ++busyWorkerThreads; + } finally { + mutex.unlock(); } - - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); - } catch (Throwable ex) { - VMError.shouldNotReachHere(ex); } + + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); } } @@ -191,16 +245,13 @@ private void work() { * early during application startup. */ public boolean waitForIdle() { - if (workers == null) { - return false; - } - - assert allocChunkTL.get().isNonNull(); - push(HeapChunk.asPointer(allocChunkTL.get())); + GCWorkerThreadState state = getWorkerThreadState(); + assert state.getAllocChunk().isNonNull(); + push(HeapChunk.asPointer(state.getAllocChunk())); mutex.lock(); try { - while (busyWorkers > 0) { // wait for worker threads to become ready + while (busyWorkerThreads > 0) { // wait for worker threads to become ready debugLog().string("PP wait for workers\n"); seqPhase.block(); } @@ -218,9 +269,10 @@ public boolean waitForIdle() { } assert buffer.isEmpty(); - // clean up thread local allocation chunks - for (Thread t : workers) { - allocChunkTL.set(PlatformThreads.getIsolateThreadUnsafe(t), WordFactory.nullPointer()); + // Reset thread local allocation chunks. + state.setAllocChunk(WordFactory.nullPointer()); + for (int i = 0; i < numWorkerThreads; i++) { + workerStates.addressOf(i).setAllocChunk(WordFactory.nullPointer()); } return true; } @@ -242,49 +294,101 @@ private static void scanChunk(Pointer ptr) { } @Uninterruptible(reason = "Called from a GC worker thread.") - private static void scanAllocChunk() { - if (allocChunkNeedsScanning()) { - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - UnsignedWord scanOffset = allocChunkScanOffsetTL.get(); + private void scanAllocChunk() { + GCWorkerThreadState state = getWorkerThreadState(); + if (allocChunkNeedsScanning(state)) { + AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); assert scanOffset.aboveThan(0); Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); - scannedChunkTL.set(allocChunk); + state.setScannedChunk(allocChunk); HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); - scannedChunkTL.set(WordFactory.nullPointer()); - if (allocChunkTL.get().equal(allocChunk)) { + state.setScannedChunk(WordFactory.nullPointer()); + if (state.getAllocChunk().equal(allocChunk)) { // remember top offset so that we don't scan the same objects again - allocChunkScanOffsetTL.set(allocChunk.getTopOffset()); + state.setAllocChunkScanOffset(allocChunk.getTopOffset()); } } } @Uninterruptible(reason = "Called from a GC worker thread.") - private static boolean allocChunkNeedsScanning() { - AlignedHeapChunk.AlignedHeader allocChunk = allocChunkTL.get(); - return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(allocChunkScanOffsetTL.get()); + private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { + AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); + return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } @SuppressWarnings("static-method") public void cleanupAfterCollection() { - allocChunkTL.set(WordFactory.nullPointer()); + getWorkerThreadState().setAllocChunk(WordFactory.nullPointer()); } - @Uninterruptible(reason = "Tear-down in progress.", calleeMustBe = false) + @Uninterruptible(reason = "Tear-down in progress.") public void tearDown() { - buffer.release(); - for (Thread t : workers) { - PlatformThreads.exit(t); - } - } + if (initialized) { + initialized = false; + + buffer.release(); + + // TODO (petermz): signal the worker threads so that they can shut down. + // TODO (petermz): use PlatformThreads.singleton().joinThreadUnmanaged(...) - private static Log verboseGCLog() { - return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); + workerThreads = WordFactory.nullPointer(); + + PlatformThreads.singleton().deleteUnmanagedThreadLocal(workerStateTL); + workerStateTL = WordFactory.nullPointer(); + } } @Uninterruptible(reason = "Called from a GC worker thread.") static Log debugLog() { return Log.noopLog(); } + + @RawStructure + private interface GCWorkerThreadState extends PointerBase { + @RawField + Isolate getIsolate(); + + @RawField + void setIsolate(Isolate value); + + @RawField + AlignedHeapChunk.AlignedHeader getAllocChunk(); + + @RawField + void setAllocChunk(AlignedHeapChunk.AlignedHeader value); + + @RawField + AlignedHeapChunk.AlignedHeader getScannedChunk(); + + @RawField + void setScannedChunk(AlignedHeapChunk.AlignedHeader value); + + @RawField + UnsignedWord getAllocChunkScanOffset(); + + @RawField + void setAllocChunkScanOffset(UnsignedWord value); + + GCWorkerThreadState addressOf(int index); + } + + private static class GCWorkerThreadPrologue implements CEntryPointOptions.Prologue { + @Uninterruptible(reason = "prologue") + @SuppressWarnings("unused") + public static void enter(GCWorkerThreadState state) { + CEntryPointSnippets.setHeapBase(state.getIsolate()); + WriteCurrentVMThreadNode.writeCurrentVMThread(WordFactory.nullPointer()); + } + } + + private static class UseParallelGC implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return SubstrateOptions.UseParallelGC.getValue(); + } + } } @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java deleted file mode 100644 index 3857cd35abc2..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGCOptions.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2022, 2022, BELLSOFT. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.genscavenge.parallel; - -import com.oracle.svm.core.option.RuntimeOptionKey; -import org.graalvm.compiler.options.Option; -import org.graalvm.compiler.options.OptionType; - -import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.Immutable; - -public class ParallelGCOptions { - - @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// - public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); -} diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index 3be9f86f296f..b4469e0828c9 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -39,6 +39,7 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; +import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; @@ -267,6 +268,35 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre return status == 0; } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); + PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); + return (ThreadLocalKey) key.read(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); + PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unchecked") + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + return (T) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); + PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); + } + @Override @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java index 73d52698a480..9e0bbf3e0905 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java @@ -55,7 +55,7 @@ @AutomaticallyRegisteredImageSingleton public class IsolateArgumentParser { private static final RuntimeOptionKey[] OPTIONS = {SubstrateGCOptions.MinHeapSize, SubstrateGCOptions.MaxHeapSize, SubstrateGCOptions.MaxNewSize, - SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData}; + SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData, SubstrateOptions.ParallelGCThreads}; private static final int OPTION_COUNT = OPTIONS.length; private static final CGlobalData OPTION_NAMES = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNames); private static final CGlobalData OPTION_NAME_POSITIONS = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNamePosition); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 24135d51ba16..1c64045f2f5f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.function.Predicate; -import com.oracle.svm.core.util.InterruptImageBuilding; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; @@ -66,6 +65,7 @@ import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.thread.VMOperationControl; +import com.oracle.svm.core.util.InterruptImageBuilding; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -326,6 +326,9 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }; + @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// + public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); + private static void requireMultiThreading(OptionKey optionKey) { if (!SubstrateOptions.MultiThreaded.getValue()) { throw new InterruptImageBuilding( diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 9d0ddee83631..1a818b24a79f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -343,10 +343,6 @@ private static int initializeIsolate(CEntryPointCreateIsolateParameters paramete } } - // Number of parallel GC workers can be affected by a runtime option, - // so call this after options are parsed - Heap.getHeap().initGC(); - boolean success = PlatformNativeLibrarySupport.singleton().initializeBuiltinLibraries(); if (firstIsolate) { // let other isolates (if any) initialize now state = success ? FirstIsolateInitStates.SUCCESSFUL : FirstIsolateInitStates.FAILED; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index aca6de0d06e1..42e28322d963 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -81,8 +81,6 @@ protected Heap() { public abstract GC getGC(); - public abstract void initGC(); - public abstract RuntimeCodeInfoGCSupport getRuntimeCodeInfoGCSupport(); /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index aa11bdddb98e..1093b1d57c50 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -39,8 +39,8 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } - // TEMP (chaeubl): - @Uninterruptible(reason = "TEMP (chaeubl)", calleeMustBe = false) + // TODO (petermz): may only be executed by the main thread + @Uninterruptible(reason = "TODO (petermz)", calleeMustBe = false) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index d22e987f66af..486b3528e35b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -64,10 +64,12 @@ import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawPointerTo; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; +import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.NeverInline; @@ -559,6 +561,26 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.joinThreadUnmanaged directly."); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.createNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.deleteNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.getNativeThreadLocalValue directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.setNativeThreadLocalValue directly."); + } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void closeOSThreadHandle(OSThreadHandle threadHandle) { @@ -1189,8 +1211,19 @@ static void blockedOn(Target_sun_nio_ch_Interruptible b) { } } + @RawStructure public interface OSThreadHandle extends PointerBase { } + + @RawPointerTo(OSThreadHandle.class) + public interface OSThreadHandlePointer extends PointerBase { + void write(int index, OSThreadHandle value); + + OSThreadHandle read(int index); + } + + public interface ThreadLocalKey extends PointerBase { + } } @TargetClass(value = ThreadPoolExecutor.class, innerClass = "Worker") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java index 729f9da14597..d31707793545 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Safepoint.java @@ -760,8 +760,7 @@ private static void waitForSafepoints(String reason) { int safepointBehavior = SafepointBehavior.getSafepointBehaviorVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints++; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; @@ -906,8 +905,7 @@ public static int countingVMOperation() { int status = StatusSupport.getStatusVolatile(vmThread); if (safepointBehavior == SafepointBehavior.PREVENT_VM_FROM_REACHING_SAFEPOINT) { notAtSafepoint++; - } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED || - safepointBehavior == SafepointBehavior.PARALLEL_GC_THREAD) { + } else if (safepointBehavior == SafepointBehavior.THREAD_CRASHED) { ignoreSafepoints += 1; } else { assert safepointBehavior == SafepointBehavior.ALLOW_SAFEPOINT; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 326de8007910..f22515193945 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -865,9 +865,6 @@ public static class SafepointBehavior { */ static final int THREAD_CRASHED = 2; - /** Similar to THREAD_CRASHED, used by ParallelGC worker threads. */ - static final int PARALLEL_GC_THREAD = 3; - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean ignoresSafepoints() { return safepointBehaviorTL.getVolatile() != ALLOW_SAFEPOINT; @@ -917,15 +914,6 @@ public static void markThreadAsCrashed() { safepointBehaviorTL.setVolatile(THREAD_CRASHED); } - /** - * Marks the thread as ParallelGC worker thread. The thread won't freeze at safepoints, so - * it must not keep references to movable heap objects on its stack. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void useAsParallelGCThread() { - safepointBehaviorTL.setVolatile(PARALLEL_GC_THREAD); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static String toString(int safepointBehavior) { switch (safepointBehavior) { @@ -935,8 +923,6 @@ public static String toString(int safepointBehavior) { return "PREVENT_VM_FROM_REACHING_SAFEPOINT"; case THREAD_CRASHED: return "THREAD_CRASHED"; - case PARALLEL_GC_THREAD: - return "PARALLEL_GC_THREAD"; default: return "Invalid safepoint behavior"; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java index 8bc6c682604d..43665437715d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/TypedMemoryReader.java @@ -28,7 +28,6 @@ import com.oracle.svm.core.Uninterruptible; -// TEMP (chaeubl): this class could be removed (e.g., combined with native coder) public class TypedMemoryReader { public static int getS1(Pointer ptr) { return ptr.readByte(0); From f4322206a803dc5e23d4338e87338a8485e06556 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 16 Mar 2023 15:29:53 +0300 Subject: [PATCH 091/125] Cleanup and assertion fixes --- .../svm/core/genscavenge/ChunksAccounting.java | 1 + .../src/com/oracle/svm/core/genscavenge/GCImpl.java | 9 ++------- .../src/com/oracle/svm/core/genscavenge/Space.java | 11 +++++++---- .../svm/core/genscavenge/parallel/ChunkBuffer.java | 2 +- .../svm/core/genscavenge/parallel/ParallelGC.java | 12 +++--------- .../remset/AlignedChunkRememberedSet.java | 1 + .../remset/CardTableBasedRememberedSet.java | 1 + .../core/genscavenge/remset/FirstObjectTable.java | 3 +++ .../svm/core/genscavenge/remset/NoRememberedSet.java | 1 + 9 files changed, 20 insertions(+), 21 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java index 99b3f184cb1c..02244cea6047 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java @@ -97,6 +97,7 @@ void noteAlignedHeapChunk() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteAlignedHeapChunk() { alignedCount--; if (parent != null) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index f6de4b5379f2..f0ec2c931709 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -1060,9 +1060,8 @@ private void scanGreyObjects(boolean isIncremental) { try { if (isIncremental) { scanGreyObjectsLoop(); - } else if (ParallelGC.isEnabled() && ParallelGC.singleton().waitForIdle()) { - // GC can happen before parallel worker threads have been started. In this case, - // fall back to serial GC. + } else if (ParallelGC.isEnabled()) { + ParallelGC.singleton().waitForIdle(); } else { HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects(); } @@ -1162,10 +1161,6 @@ private void releaseSpaces() { if (completeCollection) { heap.getOldGeneration().releaseSpaces(chunkReleaser); } - - if (ParallelGC.isEnabled()) { - ParallelGC.singleton().cleanupAfterCollection(); - } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 9afe74af5c4a..9ee8acae84d4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -115,6 +115,7 @@ public boolean isYoungSpace() { return age <= HeapParameters.getMaxSurvivorSpaces(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isSurvivorSpace() { return age > 0 && age <= HeapParameters.getMaxSurvivorSpaces(); } @@ -236,7 +237,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu ParallelGC.singleton().pushAllocChunk(oldChunk); newChunk = requestAlignedHeapChunk(); } finally { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } if (newChunk.isNonNull()) { ParallelGC.singleton().setAllocationChunk(newChunk); @@ -284,6 +285,7 @@ private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeade } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { assert VMOperation.isGCInProgress() : "Should only be called by the collector."; extractAlignedHeapChunkUninterruptibly(aChunk); @@ -533,11 +535,12 @@ private Object copyAlignedObject(Object originalObj) { } /** Promote an AlignedHeapChunk by moving it to this space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.lock(); + ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { originalSpace.extractAlignedHeapChunk(chunk); @@ -546,7 +549,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); if (ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } } } @@ -576,7 +579,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); if (ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 9abcf88dea44..d087fde521d0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -89,7 +89,7 @@ Pointer pop() { return WordFactory.nullPointer(); } } finally { - ParallelGC.mutex.unlock(); + ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 59f58f1b1c34..023cc79ad2d9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -224,10 +224,10 @@ private void work(GCWorkerThreadState state) { inParallelPhase = false; seqPhase.signal(); } - parPhase.blockNoTransition(); + parPhase.blockNoTransitionUnspecifiedOwner(); ++busyWorkerThreads; } finally { - mutex.unlock(); + mutex.unlockNoTransitionUnspecifiedOwner(); } } @@ -244,7 +244,7 @@ private void work(GCWorkerThreadState state) { * @return false if worker threads have not been started yet. This can happen if GC happens very * early during application startup. */ - public boolean waitForIdle() { + public void waitForIdle() { GCWorkerThreadState state = getWorkerThreadState(); assert state.getAllocChunk().isNonNull(); push(HeapChunk.asPointer(state.getAllocChunk())); @@ -274,7 +274,6 @@ public boolean waitForIdle() { for (int i = 0; i < numWorkerThreads; i++) { workerStates.addressOf(i).setAllocChunk(WordFactory.nullPointer()); } - return true; } @Uninterruptible(reason = "Called from a GC worker thread.") @@ -317,11 +316,6 @@ private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } - @SuppressWarnings("static-method") - public void cleanupAfterCollection() { - getWorkerThreadState().setAllocChunk(WordFactory.nullPointer()); - } - @Uninterruptible(reason = "Tear-down in progress.") public void tearDown() { if (initialized) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index b03aa738fdca..f9a39b6dd305 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -113,6 +113,7 @@ public static void enableRememberedSet(AlignedHeader chunk) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(AlignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 04281e745f20..6b5e3c3b1100 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -106,6 +106,7 @@ public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { AlignedChunkRememberedSet.clearRememberedSet(chunk); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 3f5136bf6e54..6a5274856fb3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -350,16 +350,19 @@ private static void setEntryAtIndex(Pointer table, UnsignedWord index, int value table.writeByte(indexToTableOffset(index), (byte) value); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedIndex(Pointer table, UnsignedWord index) { int entry = getEntryAtIndex(table, index); return isUninitializedEntry(entry); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isValidEntry(int entry) { return ENTRY_MIN <= entry && entry <= ENTRY_MAX; } /** May only be used for assertions. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedEntry(int entry) { assert isValidEntry(entry) : "Invalid entry"; return entry == UNINITIALIZED_ENTRY; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index d0f911951ca4..8cbf01310aaf 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -105,6 +105,7 @@ public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { // Nothing to do. } From 22d9dc527ddc5c41c382c4f384487ecc22460c55 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 20 Mar 2023 11:31:06 +0300 Subject: [PATCH 092/125] Addressed minor review comments --- .../core/genscavenge/AlignedHeapChunk.java | 2 +- .../genscavenge/GreyToBlackObjRefVisitor.java | 55 ++++++++++--------- .../core/genscavenge/ObjectHeaderImpl.java | 5 +- .../oracle/svm/core/genscavenge/Space.java | 7 +-- .../genscavenge/parallel/ChunkBuffer.java | 11 ++-- .../core/genscavenge/parallel/ParallelGC.java | 4 +- 6 files changed, 41 insertions(+), 43 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 1a64269b47af..e87e6b922d30 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -118,7 +118,7 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); - assert newTop.aboveThan(HeapChunk.asPointer(that)); + assert newTop.aboveOrEqual(getObjectsStart(that)); HeapChunk.setTopPointer(that, newTop); return newTop; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index cd13fb0e5989..367599924ea9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -155,13 +156,13 @@ public interface Counters extends AutoCloseable { } public static class RealCounters implements Counters { - private long objRef; - private long nullObjRef; - private long nullReferent; - private long forwardedReferent; - private long nonHeapReferent; - private long copiedReferent; - private long unmodifiedReference; + private UninterruptibleUtils.AtomicLong objRef; + private UninterruptibleUtils.AtomicLong nullObjRef; + private UninterruptibleUtils.AtomicLong nullReferent; + private UninterruptibleUtils.AtomicLong forwardedReferent; + private UninterruptibleUtils.AtomicLong nonHeapReferent; + private UninterruptibleUtils.AtomicLong copiedReferent; + private UninterruptibleUtils.AtomicLong unmodifiedReference; RealCounters() { reset(); @@ -169,13 +170,13 @@ public static class RealCounters implements Counters { @Override public void reset() { - objRef = 0L; - nullObjRef = 0L; - nullReferent = 0L; - forwardedReferent = 0L; - nonHeapReferent = 0L; - copiedReferent = 0L; - unmodifiedReference = 0L; + objRef.set(0L); + nullObjRef.set(0L); + nullReferent.set(0L); + forwardedReferent.set(0L); + nonHeapReferent.set(0L); + copiedReferent.set(0L); + unmodifiedReference.set(0L); } @Override @@ -193,50 +194,50 @@ public void close() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { - objRef += 1L; + objRef.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { - nullReferent += 1L; + nullReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { - forwardedReferent += 1L; + forwardedReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { - nonHeapReferent += 1L; + nonHeapReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { - copiedReferent += 1L; + copiedReferent.incrementAndGet(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { - unmodifiedReference += 1L; + unmodifiedReference.incrementAndGet(); } @Override public void toLog() { Log log = Log.log(); log.string("[GreyToBlackObjRefVisitor.counters:"); - log.string(" objRef: ").signed(objRef); - log.string(" nullObjRef: ").signed(nullObjRef); - log.string(" nullReferent: ").signed(nullReferent); - log.string(" forwardedReferent: ").signed(forwardedReferent); - log.string(" nonHeapReferent: ").signed(nonHeapReferent); - log.string(" copiedReferent: ").signed(copiedReferent); - log.string(" unmodifiedReference: ").signed(unmodifiedReference); + log.string(" objRef: ").signed(objRef.get()); + log.string(" nullObjRef: ").signed(nullObjRef.get()); + log.string(" nullReferent: ").signed(nullReferent.get()); + log.string(" forwardedReferent: ").signed(forwardedReferent.get()); + log.string(" nonHeapReferent: ").signed(nonHeapReferent.get()); + log.string(" copiedReferent: ").signed(copiedReferent.get()); + log.string(" unmodifiedReference: ").signed(unmodifiedReference.get()); log.string("]").newline(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 5a5ffaee2744..f8d4d56ebdac 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -66,6 +66,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { private static final UnsignedWord UNALIGNED_BIT = WordFactory.unsigned(0b00001); private static final UnsignedWord REMEMBERED_SET_BIT = WordFactory.unsigned(0b00010); private static final UnsignedWord FORWARDED_BIT = WordFactory.unsigned(0b00100); + private static final UnsignedWord SHIFT_FORWARD_HEADER = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); /** * Optional: per-object identity hash code state to avoid a fixed field, initially implicitly @@ -440,7 +441,7 @@ void installForwardingPointer(Object original, Object copy) { if (false) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. - forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); + forwardHeader = SHIFT_FORWARD_HEADER; ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); @@ -463,7 +464,7 @@ Object installForwardingPointerParallel(Object original, UnsignedWord originalHe // TODO (chaeubl): fix this // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { if (false) { - forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); + forwardHeader = SHIFT_FORWARD_HEADER; hasShift = true; } else { forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 9ee8acae84d4..5a30cfbfc3d9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -447,12 +447,9 @@ Object copyAlignedObjectParallel(Object original) { if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { return null; } - Object copy = copyMemory.toObject(); - if (copy == null) { - return null; - } // Install forwarding pointer into the original header + Object copy = copyMemory.toObject(); Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { // We have won the race, now we must copy the object bits. First install the original @@ -462,7 +459,7 @@ Object copyAlignedObjectParallel(Object original) { if (hubOffset > 0) { UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } - int offset = hubOffset + ConfigurationValues.getTarget().wordSize; + int offset = hubOffset + ConfigurationValues.getObjectLayout().getReferenceSize(); UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), size.subtract(offset)); if (isOldSpace()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index d087fde521d0..7a99261a7afc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -36,6 +36,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.VMError; /** * Synchronized buffer that stores "grey" heap chunks to be scanned. @@ -44,7 +45,7 @@ public class ChunkBuffer { private static final int INITIAL_SIZE = 1024 * wordSize(); private Pointer buffer; - private int size; + private int size = INITIAL_SIZE; private int top; @Fold @@ -58,20 +59,18 @@ static int wordSize() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void initialize() { - this.size = INITIAL_SIZE; - // TODO (petermz): needs proper error handling - this.buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(this.size)); + buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(size)); + VMError.guarantee(buffer.isNonNull()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void push(Pointer ptr) { assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.hasOwner(); if (top >= size) { - int oldSize = size; size *= 2; assert top < size; - // TODO (petermz): needs proper error handling buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, WordFactory.unsigned(size)); + VMError.guarantee(buffer.isNonNull()); } buffer.writeWord(top, ptr); top += wordSize(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 023cc79ad2d9..bcbb1cb850b4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -170,14 +170,14 @@ public void initialize() { /* Allocate one struct per worker thread and one struct for the main GC thread. */ int numWorkerStates = numWorkerThreads + 1; - // TODO (petermz): add error handling for allocation workerStates = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(SizeOf.unsigned(GCWorkerThreadState.class).multiply(numWorkerStates)); + VMError.guarantee(workerStates.isNonNull()); for (int i = 0; i < numWorkerStates; i++) { workerStates.addressOf(i).setIsolate(CurrentIsolate.getIsolate()); } - // TODO (petermz): add error handling for allocation workerThreads = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(OSThreadHandlePointer.class).multiply(numWorkerThreads)); + VMError.guarantee(workerThreads.isNonNull()); for (int i = 0; i < numWorkerThreads; i++) { OSThreadHandle thread = PlatformThreads.singleton().startThreadUnmanaged(gcWorkerRunFunc.getFunctionPointer(), workerStates.addressOf(i), 0); workerThreads.write(i, thread); From 43b944dde642a36260cb062f8b4cf97bbb1be736 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 20 Mar 2023 17:12:52 +0300 Subject: [PATCH 093/125] Adapted to IdentityHash changes --- .../oracle/svm/core/genscavenge/Space.java | 37 +++++++++++-------- .../oracle/svm/core/hub/LayoutEncoding.java | 13 ++++++- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 5a30cfbfc3d9..492782153e22 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -441,9 +441,18 @@ Object copyAlignedObjectParallel(Object original) { // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. // If another thread copies the object first, we retract the allocation later. - UnsignedWord size = getSizeFromHeader(original, originalHeader); - assert size.aboveThan(0); - Pointer copyMemory = allocateMemory(size); + UnsignedWord originalSize = LayoutEncoding.getSizeFromHeader(original, originalHeader, false); + UnsignedWord copySize = originalSize; + boolean addIdentityHashField = false; + if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) { + if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(originalHeader))) { + addIdentityHashField = true; + copySize = LayoutEncoding.getSizeFromHeader(original, originalHeader, true); + } + } + + assert copySize.aboveThan(0); + Pointer copyMemory = allocateMemoryParallel(copySize); if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { return null; } @@ -452,15 +461,21 @@ Object copyAlignedObjectParallel(Object original) { Object copy = copyMemory.toObject(); Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original - // header + // We have won the race, now we must copy the object bits. First install the original header copyMemory.writeWord(hubOffset, originalHeader); // Copy the rest of original object if (hubOffset > 0) { UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } int offset = hubOffset + ConfigurationValues.getObjectLayout().getReferenceSize(); - UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), size.subtract(offset)); + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); + + if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(original); + offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); + ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); + } if (isOldSpace()) { // If the object was promoted to the old gen, we need to take care of the remembered @@ -471,19 +486,11 @@ Object copyAlignedObjectParallel(Object original) { return copy; } else { // Retract speculatively allocated memory - retractAllocation(size); + retractAllocation(originalSize); return forward; } } - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getSizeFromHeader(Object obj, Word header) { - DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(header); - int encoding = hub.getLayoutEncoding(); - return LayoutEncoding.getSizeFromEncoding(obj, hub, encoding, false); - } - @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObject(Object originalObj) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 61cc8c8b458f..c9dc2160814d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -348,6 +348,17 @@ public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOp return getSizeFromObjectInline(obj, withOptionalIdHashField); } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getSizeFromHeader(Object obj, Word header, boolean addOptionalIdHashField) { + ObjectHeader oh = Heap.getHeap().getObjectHeader(); + DynamicHub hub = oh.dynamicHubFromObjectHeader(header); + int encoding = hub.getLayoutEncoding(); + boolean withOptionalIdHashField = addOptionalIdHashField || + (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && oh.hasOptionalIdentityHashField(header)); + return getSizeFromEncoding(obj, hub, encoding, withOptionalIdHashField); + } + @AlwaysInline("Actual inlining decided by callers.") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getSizeFromObjectInline(Object obj, boolean withOptionalIdHashField) { @@ -358,7 +369,7 @@ private static UnsignedWord getSizeFromObjectInline(Object obj, boolean withOpti @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getSizeFromEncoding(Object obj, DynamicHub hub, int encoding, boolean withOptionalIdHashField) { + private static UnsignedWord getSizeFromEncoding(Object obj, DynamicHub hub, int encoding, boolean withOptionalIdHashField) { if (isArrayLike(encoding)) { return getArraySize(encoding, ArrayLengthNode.arrayLength(obj), withOptionalIdHashField); } else { From c824b1bfd0be6f969bdb40950788d3fbff9e081e Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 20 Mar 2023 17:13:43 +0300 Subject: [PATCH 094/125] Worker thread shutdown routine --- .../svm/core/genscavenge/parallel/ParallelGC.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index bcbb1cb850b4..37b72aa6a2b9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -84,9 +84,10 @@ public class ParallelGC { private OSThreadHandlePointer workerThreads; private GCWorkerThreadState workerStates; private int numWorkerThreads; - private volatile int busyWorkerThreads; + private int busyWorkerThreads; private ThreadLocalKey workerStateTL; private volatile boolean inParallelPhase; + private volatile boolean shutdown; @Platforms(Platform.HOSTED_ONLY.class) public ParallelGC() { @@ -225,6 +226,9 @@ private void work(GCWorkerThreadState state) { seqPhase.signal(); } parPhase.blockNoTransitionUnspecifiedOwner(); + if (shutdown) { + return; + } ++busyWorkerThreads; } finally { mutex.unlockNoTransitionUnspecifiedOwner(); @@ -323,8 +327,13 @@ public void tearDown() { buffer.release(); - // TODO (petermz): signal the worker threads so that they can shut down. - // TODO (petermz): use PlatformThreads.singleton().joinThreadUnmanaged(...) + // signal the worker threads so that they can shut down + shutdown = true; + parPhase.broadcast(); + for (int i = 0; i < numWorkerThreads; i++) { + OSThreadHandle thread = workerThreads.read(i); + PlatformThreads.singleton().joinThreadUnmanaged(thread, WordFactory.nullPointer()); + } ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); workerThreads = WordFactory.nullPointer(); From 5954d3620c2a5342420b619edcd24d4cf0609522 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 21 Mar 2023 11:46:34 +0300 Subject: [PATCH 095/125] Report OOME on main GC thread only --- .../core/genscavenge/HeapChunkProvider.java | 11 ++++++- .../svm/core/genscavenge/OldGeneration.java | 4 +-- .../oracle/svm/core/genscavenge/Space.java | 8 ++--- .../core/genscavenge/parallel/ParallelGC.java | 33 ++++++++++++++----- .../oracle/svm/core/heap/OutOfMemoryUtil.java | 3 +- 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 1aeb21c28ad6..9fcc5490b669 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -95,6 +95,11 @@ public UnsignedWord getBytesInUnusedChunks() { /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { + return produceAlignedChunk(true); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + AlignedHeader produceAlignedChunk(boolean reportOutOfMemory) { UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); AlignedHeader result = popUnusedAlignedChunk(); if (result.isNull()) { @@ -102,7 +107,11 @@ AlignedHeader produceAlignedChunk() { noteFirstAllocationTime(); result = (AlignedHeader) CommittedMemoryProvider.get().allocateAlignedChunk(chunkSize, HeapParameters.getAlignedHeapChunkAlignment()); if (result.isNull()) { - throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); + if (reportOutOfMemory) { + throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); + } else { + throw ALIGNED_OUT_OF_MEMORY_ERROR; + } } AlignedHeapChunk.initialize(result, chunkSize); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index f8d44ad04c42..2bd8eba27116 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -162,9 +162,9 @@ UnsignedWord getChunkBytes() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") - AlignedHeapChunk.AlignedHeader requestAlignedChunk() { + AlignedHeapChunk.AlignedHeader requestAlignedChunk(boolean reportOutOfMemory) { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; - AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); + AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(reportOutOfMemory); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 492782153e22..ea73c12e2000 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -222,7 +222,7 @@ private static Pointer retractAllocation(UnsignedWord objectSize) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(true); if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } @@ -235,7 +235,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); try { ParallelGC.singleton().pushAllocChunk(oldChunk); - newChunk = requestAlignedHeapChunk(); + newChunk = requestAlignedHeapChunk(false); } finally { ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } @@ -599,13 +599,13 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { + private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk(boolean reportOutOfMemory) { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { assert isSurvivorSpace(); chunk = HeapImpl.getHeapImpl().getYoungGeneration().requestAlignedSurvivorChunk(); } else { - chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(); + chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(reportOutOfMemory); } if (chunk.isNonNull()) { appendAlignedHeapChunk(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 37b72aa6a2b9..36e3c41579f3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -59,6 +59,7 @@ import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; @@ -88,6 +89,7 @@ public class ParallelGC { private ThreadLocalKey workerStateTL; private volatile boolean inParallelPhase; private volatile boolean shutdown; + private volatile Throwable error; @Platforms(Platform.HOSTED_ONLY.class) public ParallelGC() { @@ -218,7 +220,7 @@ private void work(GCWorkerThreadState state) { while (true) { Pointer ptr; - while (!inParallelPhase || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { + while (!inParallelPhase || error != null || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { mutex.lockNoTransitionUnspecifiedOwner(); try { if (--busyWorkerThreads == 0) { @@ -235,18 +237,20 @@ private void work(GCWorkerThreadState state) { } } - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); + try { + do { + scanChunk(ptr); + } while ((ptr = buffer.pop()).isNonNull()); + scanAllocChunk(); + } catch (Throwable e) { + error = e; + shutdown = true; + } } } /** * Start parallel phase and wait until all chunks have been processed. - * - * @return false if worker threads have not been started yet. This can happen if GC happens very - * early during application startup. */ public void waitForIdle() { GCWorkerThreadState state = getWorkerThreadState(); @@ -272,6 +276,8 @@ public void waitForIdle() { mutex.unlock(); } + haltOnError(); + assert buffer.isEmpty(); // Reset thread local allocation chunks. state.setAllocChunk(WordFactory.nullPointer()); @@ -320,6 +326,17 @@ private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } + /** + * Make sure errors are thrown on main GC thread. + */ + private void haltOnError() { + if (error instanceof OutOfMemoryError oome) { + throw OutOfMemoryUtil.reportOutOfMemoryError(oome); + } else if (error instanceof Error) { + throw VMError.shouldNotReachHere(error); + } + } + @Uninterruptible(reason = "Tear-down in progress.") public void tearDown() { if (initialized) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 1093b1d57c50..8ca6e8f525f1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -39,8 +39,7 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } - // TODO (petermz): may only be executed by the main thread - @Uninterruptible(reason = "TODO (petermz)", calleeMustBe = false) + @Uninterruptible(reason = "Called from uninterruptible code.", calleeMustBe = false, mayBeInlined = true) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { From cf486e6047174d2af54f0f4329d2bb9259b93ac8 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 21 Mar 2023 13:06:32 +0300 Subject: [PATCH 096/125] Added javadoc for ParallelGC --- .../core/genscavenge/parallel/ParallelGC.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 36e3c41579f3..2b63dd122cd0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -70,6 +70,22 @@ import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; import com.oracle.svm.core.util.VMError; +/** + * A garbage collector that tries to shorten GC pause by executing work on multiple "worker threads". + * Currently the only phase supported is scanning grey objects. Number of worker threads can be set with + * a runtime option. + * + * Worker threads use heap chunks as the unit of work. Chunks to be scanned are stored in the synchronized + * {@link ChunkBuffer}. Worker threads pop chunks from the buffer and scan them for references to live + * objects to be promoted. For each object being promoted, they speculatively allocate memory for its copy + * in the to-space, then compete to install forwarding pointer in the original object. The winning thread + * proceeds to copy object data, losing threads retract the speculatively allocated memory. + * + * Each worker thread allocates memory in its own thread local "allocation chunk" for speed. As allocation + * chunks become filled up, they are pushed to {@link ChunkBuffer}. This pop-scan-push cycle continues until + * the chunk buffer becomes empty. At this point, worker threads are parked and the GC routine continues on + * the main GC thread. + */ public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; From d31e9d775b13fac39a198b341d0bbf3728d95f4a Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 27 Mar 2023 18:53:48 +0300 Subject: [PATCH 097/125] Workaround for @Uninterruptible visitors --- .../core/genscavenge/GreyToBlackObjRefVisitor.java | 10 ++++++++-- .../core/genscavenge/GreyToBlackObjectVisitor.java | 8 +++++++- .../oracle/svm/core/genscavenge/ObjectHeaderImpl.java | 1 + .../src/com/oracle/svm/core/code/CodeInfoEncoder.java | 1 + .../oracle/svm/core/heap/ObjectReferenceVisitor.java | 6 ++++++ .../src/com/oracle/svm/core/heap/ObjectVisitor.java | 11 ++++++++++- 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 367599924ea9..d13e9d9e4edd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -61,13 +61,19 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceInline(objRef, 0, compressed, holderObject); + return visitObjectReferenceUninterruptibly(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { + return visitObjectReferenceUninterruptibly(objRef, innerOffset, compressed, holderObject); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean visitObjectReferenceUninterruptibly(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index bc432b866261..329c626fb69d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -56,8 +56,14 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectInline(Object o) { + return visitObjectUninterruptibly(o); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean visitObjectUninterruptibly(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index f8d4d56ebdac..d9e43accf79e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -210,6 +210,7 @@ public boolean hasOptionalIdentityHashField(Word header) { return header.and(IDHASH_STATE_BITS).equal(inFieldState); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setIdentityHashInField(Object o) { assert VMOperation.isGCInProgress(); VMError.guarantee(!hasFixedIdentityHashField()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java index bde7d510f25d..00672fbea363 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java @@ -701,6 +701,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { int derivedOffset = NumUtil.safeToInt(objRef.rawValue()); result.markReferenceAtOffset(derivedOffset, derivedOffset - innerOffset, compressed); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java index f51f41d7a382..c424075096b4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; @@ -44,12 +45,17 @@ public interface ObjectReferenceVisitor { boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject); /** + * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) + * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with + * calleeMustBe=false, so that it can call interruptible implementations. + * * @param innerOffset If the reference is a {@linkplain CodeReferenceMapDecoder derived * reference}, a positive integer that must be subtracted from the address to which * the object reference points in order to get the start of the referenced object. */ @AlwaysInline("GC performance") @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Some implementations allocate.") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { VMError.guarantee(innerOffset == 0, "visitor does not support derived references"); return visitObjectReference(objRef, compressed, holderObject); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java index ac706669506e..816e276b6d86 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + /** * Supply a closure to be applied to Objects. * @@ -38,8 +40,15 @@ public interface ObjectVisitor { @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") boolean visitObject(Object o); - /** Like visitObject(Object), but inlined for performance. */ + /** + * Like visitObject(Object), but inlined for performance. + * + * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) + * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with + * calleeMustBe=false, so that it can call interruptible implementations. + */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectInline(Object o) { return visitObject(o); } From 65e7ad2ef3fb8069508327b275cf8a63dec7dde7 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 29 Mar 2023 11:58:57 +0300 Subject: [PATCH 098/125] Fixed crash due to chunk space being null --- .../oracle/svm/core/genscavenge/Space.java | 126 ++++++++---------- 1 file changed, 52 insertions(+), 74 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ea73c12e2000..1fd482e2742b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -46,7 +46,6 @@ import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; -import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; @@ -257,8 +256,13 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { accounting.reset(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Must not interact with garbage collections.") void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { + appendAlignedHeapChunk(aChunk, null); + } + + @Uninterruptible(reason = "Must not interact with garbage collections.") + void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk, Space originalSpace) { /* * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. @@ -266,14 +270,26 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion."); } - appendAlignedHeapChunkUninterruptibly(aChunk); - accounting.noteAlignedHeapChunk(); - } + HeapChunk.setSpace(aChunk, this); + + if (originalSpace != null) { + assert VMOperation.isGCInProgress() : "Should only be called by the collector."; + AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); + AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + originalSpace.setFirstAlignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + originalSpace.setLastAlignedHeapChunk(chunkPrev); + } + originalSpace.accounting.unnoteAlignedHeapChunk(); + } - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeader aChunk) { AlignedHeapChunk.AlignedHeader oldLast = getLastAlignedHeapChunk(); - HeapChunk.setSpace(aChunk, this); HeapChunk.setPrevious(aChunk, oldLast); HeapChunk.setNext(aChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { @@ -283,36 +299,16 @@ private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeade if (getFirstAlignedHeapChunk().isNull()) { setFirstAlignedHeapChunk(aChunk); } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { - assert VMOperation.isGCInProgress() : "Should only be called by the collector."; - extractAlignedHeapChunkUninterruptibly(aChunk); - accounting.unnoteAlignedHeapChunk(); + accounting.noteAlignedHeapChunk(); } @Uninterruptible(reason = "Must not interact with garbage collections.") - private void extractAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeader aChunk) { - AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); - AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - setFirstAlignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - setLastAlignedHeapChunk(chunkPrev); - } - HeapChunk.setNext(aChunk, WordFactory.nullPointer()); - HeapChunk.setPrevious(aChunk, WordFactory.nullPointer()); - HeapChunk.setSpace(aChunk, null); + void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { + appendUnalignedHeapChunk(uChunk, null); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { + @Uninterruptible(reason = "Must not interact with garbage collections.") + void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk, Space originalSpace) { /* * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. @@ -320,14 +316,26 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion."); } - appendUnalignedHeapChunkUninterruptibly(uChunk); - accounting.noteUnalignedHeapChunk(uChunk); - } + HeapChunk.setSpace(uChunk, this); + + if (originalSpace != null) { + assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; + UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); + UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + originalSpace.setFirstUnalignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + originalSpace.setLastUnalignedHeapChunk(chunkPrev); + } + originalSpace.accounting.unnoteUnalignedHeapChunk(uChunk); + } - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.UnalignedHeader uChunk) { UnalignedHeapChunk.UnalignedHeader oldLast = getLastUnalignedHeapChunk(); - HeapChunk.setSpace(uChunk, this); HeapChunk.setPrevious(uChunk, oldLast); HeapChunk.setNext(uChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { @@ -337,33 +345,7 @@ private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.Unaligne if (getFirstUnalignedHeapChunk().isNull()) { setFirstUnalignedHeapChunk(uChunk); } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { - assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; - extractUnalignedHeapChunkUninterruptibly(uChunk); - accounting.unnoteUnalignedHeapChunk(uChunk); - } - - @Uninterruptible(reason = "Must not interact with garbage collections.") - private void extractUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.UnalignedHeader uChunk) { - UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); - UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - setFirstUnalignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - setLastUnalignedHeapChunk(chunkPrev); - } - /* Reset the fields that the result chunk keeps for Space. */ - HeapChunk.setNext(uChunk, WordFactory.nullPointer()); - HeapChunk.setPrevious(uChunk, WordFactory.nullPointer()); - HeapChunk.setSpace(uChunk, null); + accounting.noteUnalignedHeapChunk(uChunk); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -547,8 +529,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { - originalSpace.extractAlignedHeapChunk(chunk); - appendAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk, originalSpace); } finally { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); @@ -577,8 +558,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); } try { - originalSpace.extractUnalignedHeapChunk(chunk); - appendUnalignedHeapChunk(chunk); + appendUnalignedHeapChunk(chunk, originalSpace); } finally { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); @@ -621,15 +601,13 @@ void absorb(Space src) { AlignedHeapChunk.AlignedHeader aChunk = src.getFirstAlignedHeapChunk(); while (aChunk.isNonNull()) { AlignedHeapChunk.AlignedHeader next = HeapChunk.getNext(aChunk); - src.extractAlignedHeapChunk(aChunk); - appendAlignedHeapChunk(aChunk); + appendAlignedHeapChunk(aChunk, src); aChunk = next; } UnalignedHeapChunk.UnalignedHeader uChunk = src.getFirstUnalignedHeapChunk(); while (uChunk.isNonNull()) { UnalignedHeapChunk.UnalignedHeader next = HeapChunk.getNext(uChunk); - src.extractUnalignedHeapChunk(uChunk); - appendUnalignedHeapChunk(uChunk); + appendUnalignedHeapChunk(uChunk, src); uChunk = next; } } From 5b47410fdbb5b45e49e3cdc80b19d03665c6e92f Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 12:19:04 +0200 Subject: [PATCH 099/125] Fix object forwarding. --- .../core/genscavenge/ObjectHeaderImpl.java | 88 ++++++++----------- .../oracle/svm/core/genscavenge/Space.java | 16 ++-- 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index d9e43accf79e..f5e16c2c8702 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -66,7 +66,6 @@ public final class ObjectHeaderImpl extends ObjectHeader { private static final UnsignedWord UNALIGNED_BIT = WordFactory.unsigned(0b00001); private static final UnsignedWord REMEMBERED_SET_BIT = WordFactory.unsigned(0b00010); private static final UnsignedWord FORWARDED_BIT = WordFactory.unsigned(0b00100); - private static final UnsignedWord SHIFT_FORWARD_HEADER = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); /** * Optional: per-object identity hash code state to avoid a fixed field, initially implicitly @@ -96,7 +95,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { } else { VMError.guarantee(ReferenceAccess.singleton().haveCompressedReferences(), "Ensures hubs (at the start of the image heap) remain addressable"); numReservedBits = numMinimumReservedBits + 2; - VMError.guarantee(numReservedBits <= numAlignmentBits || ReferenceAccess.singleton().getCompressEncoding().hasShift(), + VMError.guarantee(numReservedBits <= numAlignmentBits || hasShift(), "With no shift, forwarding references are stored directly in the header (with 64-bit, must be) and we cannot use non-alignment header bits"); } numReservedExtraBits = numReservedBits - numAlignmentBits; @@ -342,8 +341,7 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro VMError.guarantee((header >>> numReservedExtraBits) == hubOffsetFromHeapBase, "Hub is too far from heap base for encoding in object header"); assert (header & reservedBitsMask) == 0 : "Object header bits must be zero initially"; if (HeapImpl.usesImageHeapCardMarking()) { - if (obj.getPartition() instanceof ChunkedImageHeapPartition) { - ChunkedImageHeapPartition partition = (ChunkedImageHeapPartition) obj.getPartition(); + if (obj.getPartition() instanceof ChunkedImageHeapPartition partition) { if (partition.isWritable()) { header |= REMEMBERED_SET_BIT.rawValue(); } @@ -413,9 +411,7 @@ Object getForwardedObject(Pointer ptr) { Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TODO (chaeubl): fix this - // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - if (false) { + if (hasShift()) { // References compressed with shift have no bits to spare, so the forwarding // reference is stored separately, after the object header ObjectLayout layout = ConfigurationValues.getObjectLayout(); @@ -435,59 +431,48 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); - UnsignedWord forwardHeader; - if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TODO (chaeubl): fix this - // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - if (false) { - // Compression with a shift uses all bits of a reference, so store the forwarding - // pointer in the location following the hub pointer. - forwardHeader = SHIFT_FORWARD_HEADER; - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); - } else { - forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); - } - } else { - forwardHeader = Word.objectToUntrackedPointer(copy); - } - assert getHeaderBitsFromHeader(forwardHeader).equal(0); - writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); + UnsignedWord forwardHeader = getForwardHeader(copy); + ObjectAccess.writeLong(original, getHubOffset(), forwardHeader.rawValue()); assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); } + /** + * The original header are the 8 bytes at the hub offset (regardless if compressed references + * are used or not). + */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + Object installForwardingPointerParallel(Object original, long eightHeaderBytes, Object copy) { + UnsignedWord forwardHeader = getForwardHeader(copy); + /* Try installing the new header. */ + Pointer originalPtr = Word.objectToUntrackedPointer(original); + long value = originalPtr.compareAndSwapLong(getHubOffset(), eightHeaderBytes, forwardHeader.rawValue(), LocationIdentity.ANY_LOCATION); + assert isPointerToForwardedObject(originalPtr); + if (value != eightHeaderBytes) { + return getForwardedObject(originalPtr, WordFactory.unsigned(value)); + } + return copy; + } + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - Object installForwardingPointerParallel(Object original, UnsignedWord originalHeader, Object copy) { - // create forwarding header - UnsignedWord forwardHeader; - boolean hasShift = false; + private UnsignedWord getForwardHeader(Object copy) { + UnsignedWord result; if (ReferenceAccess.singleton().haveCompressedReferences()) { - // TODO (chaeubl): fix this - // if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { - if (false) { - forwardHeader = SHIFT_FORWARD_HEADER; - hasShift = true; + UnsignedWord compressedCopy = ReferenceAccess.singleton().getCompressedRepresentation(copy); + if (hasShift()) { + // Compression with a shift uses all bits of a reference, so store the forwarding + // pointer in the location following the hub pointer. + result = compressedCopy.shiftLeft(32).or(WordFactory.unsigned(0x00000000e0e0e0e0L)); } else { - forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); + result = compressedCopy; } } else { - forwardHeader = Word.objectToUntrackedPointer(copy); + result = Word.objectToUntrackedPointer(copy); } - assert getHeaderBitsFromHeader(forwardHeader).equal(0); - forwardHeader = forwardHeader.or(FORWARDED_BIT); - // try installing the new header - Pointer originalPtr = Word.objectToUntrackedPointer(original); - UnsignedWord witness = originalPtr.compareAndSwapWord(getHubOffset(), originalHeader, forwardHeader, LocationIdentity.ANY_LOCATION); - assert isPointerToForwardedObject(originalPtr); - if (witness.notEqual(originalHeader)) { - return getForwardedObject(originalPtr, witness); - } else if (hasShift) { - // Compression with a shift uses all bits of a reference, so store the forwarding - // pointer in the location following the hub pointer. - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); - } - return copy; + assert getHeaderBitsFromHeader(result).equal(0); + return result.or(FORWARDED_BIT); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -507,6 +492,11 @@ static int getReferenceSize() { return ConfigurationValues.getObjectLayout().getReferenceSize(); } + @Fold + static boolean hasShift() { + return getReferenceSize() == 4; + } + @Fold static boolean hasFixedIdentityHashField() { return ConfigurationValues.getObjectLayout().hasFixedIdentityHashField(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 8ebf281d4d62..c234f9a10de7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -413,9 +413,14 @@ Object copyAlignedObjectParallel(Object original) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); + /* + * Always read 8 bytes at the hub offset so that we can install the forwarding header with + * cmpxchng. + */ Pointer originalMemory = Word.objectToUntrackedPointer(original); int hubOffset = ObjectHeaderImpl.getHubOffset(); - Word originalHeader = originalMemory.readWord(hubOffset); + long eightHeaderBytes = originalMemory.readLong(hubOffset); + Word originalHeader = ObjectHeaderImpl.hasShift() ? WordFactory.unsigned(eightHeaderBytes & 0xFFFFFFFFL) : WordFactory.unsigned(eightHeaderBytes); ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { return ohi.getForwardedObject(originalMemory, originalHeader); @@ -441,15 +446,16 @@ Object copyAlignedObjectParallel(Object original) { // Install forwarding pointer into the original header Object copy = copyMemory.toObject(); - Object forward = ohi.installForwardingPointerParallel(original, originalHeader, copy); + Object forward = ohi.installForwardingPointerParallel(original, eightHeaderBytes, copy); if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original header - copyMemory.writeWord(hubOffset, originalHeader); + // We have won the race, now we must copy the object bits. First install the original + // header + copyMemory.writeLong(hubOffset, eightHeaderBytes); // Copy the rest of original object if (hubOffset > 0) { UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } - int offset = hubOffset + ConfigurationValues.getObjectLayout().getReferenceSize(); + int offset = hubOffset + Long.BYTES; UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { From 7c1aa1388eb8ab8ecf23d4d71a0a52c228b81f07 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 4 Apr 2023 14:58:20 +0300 Subject: [PATCH 100/125] Use at most 8 worker threads --- .../oracle/svm/core/genscavenge/parallel/ParallelGC.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 2b63dd122cd0..de8930108f92 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -61,6 +61,7 @@ import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.Jvm; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; @@ -90,6 +91,8 @@ public class ParallelGC { public static final int UNALIGNED_BIT = 0x01; + private static final int MAX_WORKER_THREADS = 8; + private final CEntryPointLiteral gcWorkerRunFunc; public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); @@ -207,15 +210,13 @@ public void initialize() { private static int getWorkerCount() { int index = IsolateArgumentParser.getOptionIndex(SubstrateOptions.ParallelGCThreads); int setting = IsolateArgumentParser.getIntOptionValue(index); - int count = setting > 0 ? setting : getDefaultWorkerCount(); - return count; + return setting > 0 ? setting : getDefaultWorkerCount(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getDefaultWorkerCount() { - // Adapted from Hotspot, see WorkerPolicy::nof_parallel_worker_threads() int cpus = Jvm.JVM_ActiveProcessorCount(); - return cpus <= 8 ? cpus : 8 + (cpus - 8) * 5 / 8; + return UninterruptibleUtils.Math.min(cpus, MAX_WORKER_THREADS); } @Uninterruptible(reason = "Heap base is not set up yet.") From 91eec0936cc1b8381d6db2a513295dda67f67e2c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 2 May 2023 13:18:14 +0200 Subject: [PATCH 101/125] Cleanups. --- .../core/genscavenge/AlignedHeapChunk.java | 12 +-- .../GenScavengeMemoryPoolMXBeans.java | 2 +- .../genscavenge/GreyToBlackObjRefVisitor.java | 5 +- .../genscavenge/GreyToBlackObjectVisitor.java | 8 +- .../svm/core/genscavenge/HeapChunk.java | 32 ++---- .../core/genscavenge/HeapChunkProvider.java | 11 +-- .../oracle/svm/core/genscavenge/HeapImpl.java | 2 +- .../core/genscavenge/JfrGCEventSupport.java | 2 +- .../svm/core/genscavenge/OldGeneration.java | 4 +- .../core/genscavenge/PinnedObjectImpl.java | 2 +- .../ReferenceObjectProcessing.java | 2 +- .../SerialAndEpsilonGCOptions.java | 25 ++--- .../svm/core/genscavenge/SerialGCOptions.java | 87 ++++++++-------- .../oracle/svm/core/genscavenge/Space.java | 20 ++-- .../core/genscavenge/UnalignedHeapChunk.java | 7 +- ...silonGC.java => UseSerialOrEpsilonGC.java} | 5 +- .../genscavenge/graal/BarrierSnippets.java | 2 +- .../graal/GenScavengeGCFeature.java | 6 +- .../genscavenge/parallel/ChunkBuffer.java | 14 +-- .../core/genscavenge/parallel/ParallelGC.java | 99 ++++++++++--------- .../src/com/oracle/svm/core/MemoryWalker.java | 15 --- .../com/oracle/svm/core/SubstrateOptions.java | 13 +-- .../oracle/svm/core/code/CodeInfoEncoder.java | 1 - .../svm/core/heap/ObjectReferenceVisitor.java | 6 -- .../oracle/svm/core/heap/ObjectVisitor.java | 7 -- .../oracle/svm/core/heap/OutOfMemoryUtil.java | 1 + .../heap/Target_java_lang_ref_Reference.java | 6 +- .../com/oracle/svm/core/thread/VMThreads.java | 4 +- .../UninterruptibleAnnotationChecker.java | 5 + 29 files changed, 179 insertions(+), 226 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/{UseMarkAndCopyOrEpsilonGC.java => UseSerialOrEpsilonGC.java} (90%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 3fd36cec86eb..da7be2af2225 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -114,13 +114,12 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { return result; } - /** Retract the latest allocation. Used by parallel collector. */ + /** Retract the latest allocation. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static Pointer retractAllocation(AlignedHeader that, UnsignedWord size) { + static void retractAllocation(AlignedHeader that, UnsignedWord size) { Pointer newTop = HeapChunk.getTopPointer(that).subtract(size); assert newTop.aboveOrEqual(getObjectsStart(that)); HeapChunk.setTopPointer(that, newTop); - return newTop; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -167,7 +166,7 @@ static MemoryWalker.HeapChunkAccess getMemoryWal } /** Methods for a {@link MemoryWalker} to access an aligned heap chunk. */ - @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseSerialOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) @@ -178,10 +177,5 @@ static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessIm public boolean isAligned(AlignedHeapChunk.AlignedHeader heapChunk) { return true; } - - @Override - public UnsignedWord getAllocationStart(AlignedHeapChunk.AlignedHeader heapChunk) { - return getObjectsStart(heapChunk); - } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java index 1ca18aea7770..50ac37182285 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java @@ -46,7 +46,7 @@ public class GenScavengeMemoryPoolMXBeans { @Platforms(Platform.HOSTED_ONLY.class) public static MemoryPoolMXBean[] createMemoryPoolMXBeans() { - if (SubstrateOptions.useMarkAndCopyGC()) { + if (SubstrateOptions.useSerialOrParallelGC()) { mxBeans = new AbstractMemoryPoolMXBean[]{ new EdenMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER), new SurvivorMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER), diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 3786cff0c8d2..45e3adf3f64c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -36,6 +36,7 @@ import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; /** @@ -67,7 +68,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { return visitObjectReferenceUninterruptibly(objRef, innerOffset, compressed, holderObject); } @@ -103,6 +104,7 @@ private boolean visitObjectReferenceUninterruptibly(Pointer objRef, int innerOff counters.noteForwardedReferent(); // Update the reference to point to the forwarded Object. Object obj = ohi.getForwardedObject(p, header); + assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); Object offsetObj = (innerOffset == 0) ? obj : Word.objectToUntrackedPointer(obj).add(innerOffset).toObject(); ReferenceAccess.singleton().writeObjectAt(objRef, offsetObj, compressed); RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); @@ -115,6 +117,7 @@ private boolean visitObjectReferenceUninterruptibly(Pointer objRef, int innerOff if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); + assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(copy).rawValue(); Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); } else { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 329c626fb69d..bc432b866261 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -55,15 +55,9 @@ public boolean visitObject(Object o) { } @Override - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) - public boolean visitObjectInline(Object o) { - return visitObjectUninterruptibly(o); - } - @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private boolean visitObjectUninterruptibly(Object o) { + public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index f395ef97877d..2f588f856895 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -175,9 +175,9 @@ public interface Header> extends HeaderPadding { public static void initialize(Header chunk, Pointer objectsStart, UnsignedWord chunkSize) { HeapChunk.setEndOffset(chunk, chunkSize); HeapChunk.setTopPointer(chunk, objectsStart); - HeapChunk.setSpace(chunk, null); - HeapChunk.setNext(chunk, WordFactory.nullPointer()); - HeapChunk.setPrevious(chunk, WordFactory.nullPointer()); + chunk.setSpace(null); + chunk.setOffsetToNextChunk(WordFactory.zero()); + chunk.setOffsetToPreviousChunk(WordFactory.zero()); /* * The epoch is obviously not random, but cheap to use, and we cannot use a random number @@ -227,6 +227,10 @@ public static void setEndOffset(Header that, UnsignedWord newEnd) { that.setEndOffset(newEnd); } + /** + * If the parallel GC is used, then this method is racy. So, it may return null or an outdated + * value if it is called for an unaligned heap chunk that is in the middle of being promoted. + */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Space getSpace(Header that) { return that.getSpace(); @@ -234,6 +238,7 @@ public static Space getSpace(Header that) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setSpace(Header that, Space newSpace) { + assert newSpace == null || that.getSpace() == null : "heap chunk must be removed from its current space before it can be registered with a new space"; that.setSpace(newSpace); } @@ -367,26 +372,5 @@ public UnsignedWord getStart(T heapChunk) { public UnsignedWord getSize(T heapChunk) { return HeapChunk.getEndOffset(heapChunk); } - - @Override - public UnsignedWord getAllocationEnd(T heapChunk) { - return HeapChunk.getTopPointer(heapChunk); - } - - @Override - public String getRegion(T heapChunk) { - /* This method knows too much about spaces, especially the "free" space. */ - Space space = getSpace(heapChunk); - String result; - if (space == null) { - result = "free"; - } else if (space.isYoungSpace()) { - result = "young"; - } else { - result = "old"; - } - return result; - } - } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 222379576e71..f94172566a49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -95,11 +95,6 @@ public UnsignedWord getBytesInUnusedChunks() { /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { - return produceAlignedChunk(true); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - AlignedHeader produceAlignedChunk(boolean reportOutOfMemory) { UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); AlignedHeader result = popUnusedAlignedChunk(); if (result.isNull()) { @@ -107,11 +102,7 @@ AlignedHeader produceAlignedChunk(boolean reportOutOfMemory) { noteFirstAllocationTime(); result = (AlignedHeader) CommittedMemoryProvider.get().allocateAlignedChunk(chunkSize, HeapParameters.getAlignedHeapChunkAlignment()); if (result.isNull()) { - if (reportOutOfMemory) { - throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); - } else { - throw ALIGNED_OUT_OF_MEMORY_ERROR; - } + throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); } AlignedHeapChunk.initialize(result, chunkSize); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 5a7ba7756f97..bc04d19fe68a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -906,7 +906,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev } } -@TargetClass(value = java.lang.Runtime.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) +@TargetClass(value = java.lang.Runtime.class, onlyWith = UseSerialOrEpsilonGC.class) @SuppressWarnings("static-method") final class Target_java_lang_Runtime { @Substitute diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 9c9cc1447966..54965f7a338f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -143,7 +143,7 @@ private int popPhase() { class JfrGCEventFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.useMarkAndCopyGC(); + return SubstrateOptions.useSerialOrParallelGC(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index ed7c84496061..8c77d261b9b5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -166,9 +166,9 @@ UnsignedWord getChunkBytes() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") - AlignedHeapChunk.AlignedHeader requestAlignedChunk(boolean reportOutOfMemory) { + AlignedHeapChunk.AlignedHeader requestAlignedChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; - AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(reportOutOfMemory); + AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index b83e866848ea..9f85d0afa7bb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -43,7 +43,7 @@ /** Support for pinning objects to a memory address with {@link PinnedObject}. */ final class PinnedObjectImpl implements PinnedObject { - @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseMarkAndCopyOrEpsilonGC.class) + @AutomaticallyRegisteredImageSingleton(value = PinnedObjectSupport.class, onlyWith = UseSerialOrEpsilonGC.class) static class PinnedObjectSupportImpl implements PinnedObjectSupport { @Override public PinnedObject create(Object object) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 2abe7e0ed2f1..96b1f294677f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -240,6 +240,6 @@ private static boolean maybeUpdateForwardedReference(Reference dr, Pointer re private static boolean willSurviveThisCollection(Object obj) { HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); Space space = HeapChunk.getSpace(chunk); - return !space.isFromSpace(); + return space != null && !space.isFromSpace(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java index 046f2d033f3b..f7c9c444ba8d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java @@ -36,16 +36,17 @@ import com.oracle.svm.core.util.InterruptImageBuilding; import com.oracle.svm.core.util.UserError; -/** Common options that can be specified for both the serial and the epsilon GC. */ +/** Options that can be specified for the serial, the parallel, and the epsilon GC. */ +// TODO (chaeubl): rename public final class SerialAndEpsilonGCOptions { @Option(help = "The maximum heap size as percent of physical memory. Serial, parallel, and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final RuntimeOptionKey MaximumHeapSizePercent = new NotifyGCRuntimeOptionKey<>(80, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); @Option(help = "The maximum size of the young generation as a percentage of the maximum heap size. Serial, parallel, and epsilon GC only.", type = OptionType.User) // - public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final RuntimeOptionKey MaximumYoungGenerationSizePercent = new NotifyGCRuntimeOptionKey<>(10, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); @Option(help = "The size of an aligned chunk. Serial, parallel, and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(512 * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly) { + public static final HostedOptionKey AlignedHeapChunkSize = new HostedOptionKey<>(512 * 1024L, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly) { @Override protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { int multiple = 4096; @@ -58,28 +59,28 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV * fit in an aligned chunk. */ @Option(help = "The size at or above which an array will be allocated in its own unaligned chunk. Serial, parallel, and epsilon GC only.", type = OptionType.Expert) // - public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final HostedOptionKey LargeArrayThreshold = new HostedOptionKey<>(0L, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); @Option(help = "Fill unused memory chunks with a sentinel value. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final HostedOptionKey ZapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); @Option(help = "Before use, fill memory chunks with a sentinel value. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final HostedOptionKey ZapProducedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); @Option(help = "After use, Fill memory chunks with a sentinel value. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final HostedOptionKey ZapConsumedHeapChunks = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); @Option(help = "Bytes that can be allocated before (re-)querying the physical memory size. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final HostedOptionKey AllocationBeforePhysicalMemorySize = new HostedOptionKey<>(1L * 1024L * 1024L, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); @Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial, parallel, and epsilon GC only.", type = OptionType.Debug) // - public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::markAndCopyOrEpsilonGCOnly); + public static final HostedOptionKey HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::serialOrParallelOrEpsilonGCOnly); private SerialAndEpsilonGCOptions() { } - private static void markAndCopyOrEpsilonGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.useMarkAndCopyOrEpsilonGC()) { + private static void serialOrParallelOrEpsilonGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useSerialOrParallelOrEpsilonGC()) { throw new InterruptImageBuilding( "The option '" + optionKey.getName() + "' can only be used together with the serial ('--gc=serial'), parallel ('--gc=parallel'), or the epsilon garbage collector ('--gc=epsilon')."); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java index 4f470b026343..005bfaf05e6c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java @@ -36,19 +36,20 @@ import com.oracle.svm.core.util.InterruptImageBuilding; import com.oracle.svm.core.util.UserError; -/** Options that are only valid for the serial GC (and not for the epsilon GC). */ +/** Options that can be specified for the serial and the parallel GC. */ +// TODO (chaeubl): rename this class public final class SerialGCOptions { - @Option(help = "The garbage collection policy, either Adaptive (default) or BySpaceAndTime. Serial GC only.", type = OptionType.User)// - public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::markAndCopyGCOnly); + @Option(help = "The garbage collection policy, either Adaptive (default) or BySpaceAndTime. Serial and parallel GC only.", type = OptionType.User)// + public static final HostedOptionKey InitialCollectionPolicy = new HostedOptionKey<>("Adaptive", SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Percentage of total collection time that should be spent on young generation collections. Serial GC with collection policy 'BySpaceAndTime' only.", type = OptionType.User)// - public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Percentage of total collection time that should be spent on young generation collections. Serial and parallel GC only, if the collection policy 'BySpaceAndTime' is used.", type = OptionType.User)// + public static final RuntimeOptionKey PercentTimeInIncrementalCollection = new RuntimeOptionKey<>(50, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "The maximum free bytes reserved for allocations, in bytes (0 for automatic according to GC policy). Serial GC only.", type = OptionType.User)// - public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "The maximum free bytes reserved for allocations, in bytes (0 for automatic according to GC policy). Serial and parallel GC only.", type = OptionType.User)// + public static final RuntimeOptionKey MaxHeapFree = new RuntimeOptionKey<>(0L, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Maximum number of survivor spaces. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly) { + @Option(help = "Maximum number of survivor spaces. Serial and parallel GC only.", type = OptionType.Expert) // + public static final HostedOptionKey MaxSurvivorSpaces = new HostedOptionKey<>(null, SerialGCOptions::serialOrParallelGCOnly) { @Override public Integer getValueOrDefault(UnmodifiableEconomicMap, Object> values) { Integer value = (Integer) values.get(this); @@ -63,60 +64,60 @@ public Integer getValue(OptionValues values) { } }; - @Option(help = "Determines if a full GC collects the young generation separately or together with the old generation. Serial GC only.", type = OptionType.Expert) // - public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Determines if a full GC collects the young generation separately or together with the old generation. Serial and parallel GC only.", type = OptionType.Expert) // + public static final RuntimeOptionKey CollectYoungGenerationSeparately = new RuntimeOptionKey<>(null, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Enables card marking for image heap objects, which arranges them in chunks. Automatically enabled when supported. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Enables card marking for image heap objects, which arranges them in chunks. Automatically enabled when supported. Serial and parallel GC only.", type = OptionType.Expert) // + public static final HostedOptionKey ImageHeapCardMarking = new HostedOptionKey<>(null, SerialGCOptions::serialOrParallelGCOnly); @Option(help = "This number of milliseconds multiplied by the free heap memory in MByte is the time span " + - "for which a soft reference will keep its referent alive after its last access. Serial GC only.", type = OptionType.Expert) // - public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::markAndCopyGCOnly); + "for which a soft reference will keep its referent alive after its last access. Serial and parallel GC only.", type = OptionType.Expert) // + public static final HostedOptionKey SoftRefLRUPolicyMSPerMB = new HostedOptionKey<>(1000, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Print the shape of the heap before and after each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Print the shape of the heap before and after each collection, if +VerboseGC. Serial and parallel GC only.", type = OptionType.Debug)// + public static final RuntimeOptionKey PrintHeapShape = new RuntimeOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Print summary GC information after application main method returns. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Print summary GC information after application main method returns. Serial and parallel GC only.", type = OptionType.Debug)// + public static final RuntimeOptionKey PrintGCSummary = new RuntimeOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Print a time stamp at each collection, if +PrintGC or +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Print a time stamp at each collection, if +PrintGC or +VerboseGC. Serial and parallel GC only.", type = OptionType.Debug)// + public static final RuntimeOptionKey PrintGCTimeStamps = new RuntimeOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Print the time for each of the phases of each collection, if +VerboseGC. Serial GC only.", type = OptionType.Debug)// - public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Print the time for each of the phases of each collection, if +VerboseGC. Serial and parallel GC only.", type = OptionType.Debug)// + public static final RuntimeOptionKey PrintGCTimes = new RuntimeOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Instrument write barriers with counters. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Instrument write barriers with counters. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Verify the heap before doing a garbage collection if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyBeforeGC = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Verify the heap before doing a garbage collection if VerifyHeap is enabled. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyBeforeGC = new HostedOptionKey<>(true, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Verify the heap after doing a garbage collection if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyAfterGC = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Verify the heap after doing a garbage collection if VerifyHeap is enabled. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyAfterGC = new HostedOptionKey<>(true, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Verify the remembered set if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Verify the remembered set if VerifyHeap is enabled. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Verify all object references if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Verify all object references if VerifyHeap is enabled. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyReferences = new HostedOptionKey<>(true, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Verify that object references point into valid heap chunks if VerifyHeap is enabled. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyReferencesPointIntoValidChunk = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Verify that object references point into valid heap chunks if VerifyHeap is enabled. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyReferencesPointIntoValidChunk = new HostedOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Verify write barriers. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Verify write barriers. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Trace heap chunks during collections, if +VerboseGC and +PrintHeapShape. Serial GC only.", type = OptionType.Debug) // - public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Trace heap chunks during collections, if +VerboseGC and +PrintHeapShape. Serial and parallel GC only.", type = OptionType.Debug) // + public static final RuntimeOptionKey TraceHeapChunks = new RuntimeOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); - @Option(help = "Develop demographics of the object references visited. Serial GC only.", type = OptionType.Debug)// - public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::markAndCopyGCOnly); + @Option(help = "Develop demographics of the object references visited. Serial and parallel GC only.", type = OptionType.Debug)// + public static final HostedOptionKey GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::serialOrParallelGCOnly); private SerialGCOptions() { } - private static void markAndCopyGCOnly(OptionKey optionKey) { - if (!SubstrateOptions.useMarkAndCopyGC()) { + private static void serialOrParallelGCOnly(OptionKey optionKey) { + if (!SubstrateOptions.useSerialOrParallelGC()) { throw new InterruptImageBuilding("The option '" + optionKey.getName() + "' can only be used together with the serial ('--gc=serial') or parallel garbage collector ('--gc=parallel')."); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 36ffd097ea5f..bcec821287a8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -209,21 +209,19 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { return allocateInNewChunkParallel(oldChunk, objectSize); } - /** - * Retract the latest allocation. Used by parallel collector. - */ + /** Retract the latest allocation. */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static Pointer retractAllocation(UnsignedWord objectSize) { + private static void retractAllocation(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); assert oldChunk.isNonNull(); - return AlignedHeapChunk.retractAllocation(oldChunk, objectSize); + AlignedHeapChunk.retractAllocation(oldChunk, objectSize); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(true); + AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); if (newChunk.isNonNull()) { return AlignedHeapChunk.allocateMemory(newChunk, objectSize); } @@ -236,7 +234,7 @@ private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChu ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); try { ParallelGC.singleton().pushAllocChunk(oldChunk); - newChunk = requestAlignedHeapChunk(false); + newChunk = requestAlignedHeapChunk(); } finally { ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } @@ -540,7 +538,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina appendAlignedHeapChunk(chunk, originalSpace); } finally { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGC.singleton().push(HeapChunk.asPointer(chunk)); + ParallelGC.singleton().push(chunk); if (ParallelGC.isInParallelPhase()) { ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } @@ -569,7 +567,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o appendUnalignedHeapChunk(chunk, originalSpace); } finally { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGC.singleton().push(HeapChunk.asPointer(chunk).or(ParallelGC.UNALIGNED_BIT)); + ParallelGC.singleton().push(chunk); if (ParallelGC.isInParallelPhase()) { ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); } @@ -587,13 +585,13 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk(boolean reportOutOfMemory) { + private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { assert isSurvivorSpace(); chunk = HeapImpl.getHeapImpl().getYoungGeneration().requestAlignedSurvivorChunk(); } else { - chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(reportOutOfMemory); + chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(); } if (chunk.isNonNull()) { appendAlignedHeapChunk(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 1117387771ca..8911d44cda00 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -165,7 +165,7 @@ public static MemoryWalker.HeapChunkAccess g return ImageSingletons.lookup(UnalignedHeapChunk.MemoryWalkerAccessImpl.class); } - @AutomaticallyRegisteredImageSingleton(onlyWith = UseMarkAndCopyOrEpsilonGC.class) + @AutomaticallyRegisteredImageSingleton(onlyWith = UseSerialOrEpsilonGC.class) static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessImpl { @Platforms(Platform.HOSTED_ONLY.class) @@ -176,10 +176,5 @@ static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessIm public boolean isAligned(UnalignedHeapChunk.UnalignedHeader heapChunk) { return false; } - - @Override - public UnsignedWord getAllocationStart(UnalignedHeapChunk.UnalignedHeader heapChunk) { - return getObjectStart(heapChunk); - } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java similarity index 90% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java index 7ab0a5f6d284..48be7c373ede 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseMarkAndCopyOrEpsilonGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UseSerialOrEpsilonGC.java @@ -32,9 +32,10 @@ import com.oracle.svm.core.SubstrateOptions; @Platforms(Platform.HOSTED_ONLY.class) -public class UseMarkAndCopyOrEpsilonGC implements BooleanSupplier { +// TODO (chaeubl): rename +public class UseSerialOrEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.useMarkAndCopyOrEpsilonGC(); + return SubstrateOptions.useSerialOrParallelOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index 32baec3caa55..90eee97f3691 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -205,7 +205,7 @@ class BarrierSnippetCounters { class BarrierSnippetCountersFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return SubstrateOptions.useMarkAndCopyGC() && SubstrateOptions.useRememberedSet(); + return SubstrateOptions.useSerialOrParallelGC() && SubstrateOptions.useRememberedSet(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index 07d5f45c3d98..df035266567e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Map; -import com.oracle.svm.core.genscavenge.UseMarkAndCopyOrEpsilonGC; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; @@ -47,6 +46,7 @@ import com.oracle.svm.core.genscavenge.ImageHeapInfo; import com.oracle.svm.core.genscavenge.IncrementalGarbageCollectorMXBean; import com.oracle.svm.core.genscavenge.LinearImageHeapLayouter; +import com.oracle.svm.core.genscavenge.UseSerialOrEpsilonGC; import com.oracle.svm.core.genscavenge.jvmstat.EpsilonGCPerfData; import com.oracle.svm.core.genscavenge.jvmstat.SerialGCPerfData; import com.oracle.svm.core.genscavenge.remset.CardTableBasedRememberedSet; @@ -72,7 +72,7 @@ class GenScavengeGCFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return new UseMarkAndCopyOrEpsilonGC().getAsBoolean(); + return new UseSerialOrEpsilonGC().getAsBoolean(); } @Override @@ -161,7 +161,7 @@ private static RememberedSet createRememberedSet() { } private static PerfDataHolder createPerfData() { - if (SubstrateOptions.useMarkAndCopyGC()) { + if (SubstrateOptions.useSerialOrParallelGC()) { return new SerialGCPerfData(); } else { assert SubstrateOptions.UseEpsilonGC.getValue(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java index 8e335c5d872d..dad08688b81e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java @@ -36,10 +36,12 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.VMError; /** - * Synchronized buffer that stores "grey" heap chunks to be scanned. + * Synchronized buffer that stores pointers into "grey" heap chunks that need to be scanned. Note + * that the pointers don't necessarily point to the beginning of a chunk. */ public class ChunkBuffer { private static final int INITIAL_SIZE = 1024 * wordSize(); @@ -65,7 +67,7 @@ public void initialize() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void push(Pointer ptr) { - assert !ParallelGC.isInParallelPhase() || ParallelGC.mutex.hasOwner(); + assert !ParallelGC.singleton().isInParallelPhase() && VMThreads.ownsThreadMutex() || ParallelGC.singleton().getMutex().hasOwner(); if (top >= size) { size *= 2; assert top < size; @@ -78,8 +80,8 @@ void push(Pointer ptr) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Pointer pop() { - assert ParallelGC.isInParallelPhase(); - ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); + assert ParallelGC.singleton().isInParallelPhase(); + ParallelGC.singleton().getMutex().lockNoTransitionUnspecifiedOwner(); try { if (top > 0) { top -= wordSize(); @@ -88,13 +90,13 @@ Pointer pop() { return WordFactory.nullPointer(); } } finally { - ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); + ParallelGC.singleton().getMutex().unlockNoTransitionUnspecifiedOwner(); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isEmpty() { - assert !ParallelGC.isInParallelPhase(); + assert !ParallelGC.singleton().isInParallelPhase(); return top == 0; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 648d2e2634e4..0f6ce107b68b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -46,7 +46,6 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.IsolateArgumentParser; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; @@ -64,7 +63,6 @@ import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.log.Log; import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandle; import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandlePointer; @@ -72,33 +70,31 @@ import com.oracle.svm.core.util.VMError; /** - * A garbage collector that tries to shorten GC pause by executing work on multiple "worker threads". - * Currently the only phase supported is scanning grey objects. Number of worker threads can be set with - * a runtime option. + * A garbage collector that tries to shorten GC pauses by using multiple "worker threads". + * Currently, the only phase supported is scanning grey objects during a full GC. The number of + * worker threads can be set with a runtime option (see {@link SubstrateOptions#ParallelGCThreads}). * - * Worker threads use heap chunks as the unit of work. Chunks to be scanned are stored in the synchronized - * {@link ChunkBuffer}. Worker threads pop chunks from the buffer and scan them for references to live - * objects to be promoted. For each object being promoted, they speculatively allocate memory for its copy - * in the to-space, then compete to install forwarding pointer in the original object. The winning thread - * proceeds to copy object data, losing threads retract the speculatively allocated memory. + * Worker threads use heap chunks as the unit of work. Chunks to be scanned are stored in the + * synchronized {@link ChunkBuffer}. Worker threads pop chunks from the buffer and scan them for + * references to live objects to be promoted. When promoting an aligned chunk object, they + * speculatively allocate memory for its copy in the to-space, then compete to install forwarding + * pointer in the original object. The winning thread proceeds to copy object data, losing threads + * retract the speculatively allocated memory. * - * Each worker thread allocates memory in its own thread local "allocation chunk" for speed. As allocation - * chunks become filled up, they are pushed to {@link ChunkBuffer}. This pop-scan-push cycle continues until - * the chunk buffer becomes empty. At this point, worker threads are parked and the GC routine continues on - * the main GC thread. + * Each worker thread allocates memory in its own thread local "allocation chunk" for speed. As + * allocation chunks become filled up, they are pushed to {@link ChunkBuffer}. This pop-scan-push + * cycle continues until the chunk buffer becomes empty. At this point, worker threads are parked + * and the GC routine continues on the main GC thread. */ public class ParallelGC { - - public static final int UNALIGNED_BIT = 0x01; - + private static final int UNALIGNED_BIT = 0x01; private static final int MAX_WORKER_THREADS = 8; - private final CEntryPointLiteral gcWorkerRunFunc; - - public static final VMMutex mutex = new VMMutex("ParallelGCImpl"); + private final VMMutex mutex = new VMMutex("parallelGC"); private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); private final ChunkBuffer buffer = new ChunkBuffer(); + private final CEntryPointLiteral gcWorkerRunFunc = CEntryPointLiteral.create(ParallelGC.class, "gcWorkerRun", GCWorkerThreadState.class); private boolean initialized; private OSThreadHandlePointer workerThreads; @@ -110,9 +106,9 @@ public class ParallelGC { private volatile boolean shutdown; private volatile Throwable error; + // TEMP (chaeubl): finish reviewing ChunkBuffer and ParallelGC. @Platforms(Platform.HOSTED_ONLY.class) public ParallelGC() { - gcWorkerRunFunc = CEntryPointLiteral.create(ParallelGC.class, "gcWorkerRun", GCWorkerThreadState.class); } @Fold @@ -126,8 +122,13 @@ public static boolean isEnabled() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean isInParallelPhase() { - return singleton().inParallelPhase; + public boolean isInParallelPhase() { + return inParallelPhase; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public VMMutex getMutex() { + return mutex; } @Uninterruptible(reason = "Called from a GC worker thread.") @@ -145,25 +146,34 @@ public void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { } @Uninterruptible(reason = "Called from a GC worker thread.") - public void push(Pointer ptr) { + public void push(AlignedHeapChunk.AlignedHeader aChunk) { + push(HeapChunk.asPointer(aChunk)); + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + public void push(UnalignedHeapChunk.UnalignedHeader uChunk) { + push(HeapChunk.asPointer(uChunk).or(ParallelGC.UNALIGNED_BIT)); + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + private void push(Pointer ptr) { assert ptr.isNonNull(); - if (buffer != null) { - buffer.push(ptr); - if (inParallelPhase) { - parPhase.signal(); - } + buffer.push(ptr); + if (inParallelPhase) { + parPhase.signal(); } } @Uninterruptible(reason = "Called from a GC worker thread.") public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { - assert isEnabled() && GCImpl.getGCImpl().isCompleteCollection(); + assert GCImpl.getGCImpl().isCompleteCollection(); GCWorkerThreadState state = getWorkerThreadState(); if (chunk.notEqual(state.getScannedChunk())) { UnsignedWord scanOffset = state.getAllocChunkScanOffset(); assert scanOffset.aboveThan(0); if (chunk.getTopOffset().aboveThan(scanOffset)) { - push(HeapChunk.asPointer(chunk).add(scanOffset)); + Pointer ptrIntoChunk = HeapChunk.asPointer(chunk).add(scanOffset); + push(ptrIntoChunk); } } } @@ -176,7 +186,6 @@ private GCWorkerThreadState getWorkerThreadState() { return workerStates.addressOf(numWorkerThreads); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void initialize() { if (initialized) { return; @@ -206,15 +215,14 @@ public void initialize() { } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getWorkerCount() { - int index = IsolateArgumentParser.getOptionIndex(SubstrateOptions.ParallelGCThreads); - int setting = IsolateArgumentParser.getIntOptionValue(index); + int setting = SubstrateOptions.ParallelGCThreads.getValue(); return setting > 0 ? setting : getDefaultWorkerCount(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getDefaultWorkerCount() { + // TEMP (chaeubl): this is incorrect if the container support is present. int cpus = Jvm.JVM_ActiveProcessorCount(); return UninterruptibleUtils.Math.min(cpus, MAX_WORKER_THREADS); } @@ -273,19 +281,22 @@ private void work(GCWorkerThreadState state) { public void waitForIdle() { GCWorkerThreadState state = getWorkerThreadState(); assert state.getAllocChunk().isNonNull(); - push(HeapChunk.asPointer(state.getAllocChunk())); + push(state.getAllocChunk()); mutex.lockNoTransitionUnspecifiedOwner(); try { while (busyWorkerThreads > 0) { - seqPhase.blockNoTransitionUnspecifiedOwner(); // wait for worker threads to become ready + /* Wait for worker threads to become ready. */ + seqPhase.blockNoTransitionUnspecifiedOwner(); } inParallelPhase = true; - parPhase.broadcast(); // let worker threads run + /* Let worker threads run. */ + parPhase.broadcast(); while (inParallelPhase) { - seqPhase.blockNoTransitionUnspecifiedOwner(); // wait for them to become idle + /* Wait for them to become idle. */ + seqPhase.blockNoTransitionUnspecifiedOwner(); } } finally { mutex.unlockNoTransitionUnspecifiedOwner(); @@ -294,7 +305,7 @@ public void waitForIdle() { haltOnError(); assert buffer.isEmpty(); - // Reset thread local allocation chunks. + /* Reset thread local allocation chunks. */ state.setAllocChunk(WordFactory.nullPointer()); for (int i = 0; i < numWorkerThreads; i++) { workerStates.addressOf(i).setAllocChunk(WordFactory.nullPointer()); @@ -329,14 +340,14 @@ private void scanAllocChunk() { HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); state.setScannedChunk(WordFactory.nullPointer()); if (state.getAllocChunk().equal(allocChunk)) { - // remember top offset so that we don't scan the same objects again + /* Remember top offset so that we don't scan the same objects again. */ state.setAllocChunkScanOffset(allocChunk.getTopOffset()); } } } @Uninterruptible(reason = "Called from a GC worker thread.") - private boolean allocChunkNeedsScanning(GCWorkerThreadState state) { + private static boolean allocChunkNeedsScanning(GCWorkerThreadState state) { AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } @@ -360,7 +371,7 @@ public void tearDown() { buffer.release(); - // signal the worker threads so that they can shut down + /* Signal the worker threads so that they can shut down. */ shutdown = true; parPhase.broadcast(); for (int i = 0; i < numWorkerThreads; i++) { @@ -417,7 +428,7 @@ public static void enter(GCWorkerThreadState state) { private static class UseParallelGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return SubstrateOptions.UseParallelGC.getValue(); + return ParallelGC.isEnabled(); } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java index 85104becd0a4..980c7db17c4c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java @@ -80,21 +80,6 @@ public interface HeapChunkAccess { /** Return the size of the heap chunk. */ UnsignedWord getSize(T heapChunk); - /** Return the address where allocation starts within the heap chunk. */ - UnsignedWord getAllocationStart(T heapChunk); - - /** - * Return the address where allocation has ended within the heap chunk. This is the first - * address past the end of allocated space within the heap chunk. - */ - UnsignedWord getAllocationEnd(T heapChunk); - - /** - * Return the name of the region that contains the heap chunk. E.g., "young", "old", "free", - * etc. - */ - String getRegion(T heapChunk); - /** Return true if the heap chunk is an aligned heap chunk, else false. */ boolean isAligned(T heapChunk); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 6f5fab8c4379..de4e97b002de 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -326,23 +326,24 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }; - @Option(help = "Number of worker threads used by ParallelGC.", type = OptionType.User)// + @Option(help = "Number of GC worker threads. Parallel and G1 GC only.", type = OptionType.User)// public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); private static void requireMultiThreading(OptionKey optionKey) { - if (!SubstrateOptions.MultiThreaded.getValue()) { - throw new InterruptImageBuilding( - "The option '" + optionKey.getName() + "' requires the option MultiThreaded to be set."); + if (!MultiThreaded.getValue()) { + throw new InterruptImageBuilding(String.format("The option %s requires the option %s to be set.", + SubstrateOptionsParser.commandArgument(optionKey, "+"), + SubstrateOptionsParser.commandArgument(MultiThreaded, "+"))); } } @Fold - public static boolean useMarkAndCopyGC() { + public static boolean useSerialOrParallelGC() { return UseSerialGC.getValue() || UseParallelGC.getValue(); } @Fold - public static boolean useMarkAndCopyOrEpsilonGC() { + public static boolean useSerialOrParallelOrEpsilonGC() { return UseSerialGC.getValue() || UseParallelGC.getValue() || UseEpsilonGC.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java index 3d51b40980a7..976939121899 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java @@ -701,7 +701,6 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { int derivedOffset = NumUtil.safeToInt(objRef.rawValue()); result.markReferenceAtOffset(derivedOffset, derivedOffset - innerOffset, compressed); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java index c424075096b4..f51f41d7a382 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.heap; -import com.oracle.svm.core.Uninterruptible; import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; @@ -45,17 +44,12 @@ public interface ObjectReferenceVisitor { boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject); /** - * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) - * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with - * calleeMustBe=false, so that it can call interruptible implementations. - * * @param innerOffset If the reference is a {@linkplain CodeReferenceMapDecoder derived * reference}, a positive integer that must be subtracted from the address to which * the object reference points in order to get the start of the referenced object. */ @AlwaysInline("GC performance") @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Some implementations allocate.") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { VMError.guarantee(innerOffset == 0, "visitor does not support derived references"); return visitObjectReference(objRef, compressed, holderObject); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java index 816e276b6d86..7882d214733c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.heap; -import com.oracle.svm.core.Uninterruptible; - /** * Supply a closure to be applied to Objects. * @@ -42,13 +40,8 @@ public interface ObjectVisitor { /** * Like visitObject(Object), but inlined for performance. - * - * This method is invoked from both uninterruptible (parallel GC) and regular (HeapDumpWriter, HeapVerifier etc) - * contexts. It is marked @Uninterruptible, so that uninterruptible invocations are possible, but with - * calleeMustBe=false, so that it can call interruptible implementations. */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) default boolean visitObjectInline(Object o) { return visitObject(o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 2daabb05d6ec..629a1ecbc81d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -42,6 +42,7 @@ public static OutOfMemoryError heapSizeExceeded() { @Uninterruptible(reason = "Not uninterruptible but it doesn't matter for the callers.", calleeMustBe = false) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { + // TEMP (chaeubl): fix this if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { if (LibC.isSupported()) { Log.log().string("Terminating due to java.lang.OutOfMemoryError: ").string(JDKUtils.getRawMessage(error)).newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java index aa534abb0a90..30c820d71901 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Target_java_lang_ref_Reference.java @@ -98,7 +98,7 @@ public final class Target_java_lang_ref_Reference { @SuppressWarnings("unused") // @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) // - @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotMarkAndCopyOrEpsilonGC.class) // + @ExcludeFromReferenceMap(reason = "Some GCs process this field manually.", onlyIf = NotSerialNotParallelNotEpsilonGC.class) // transient Target_java_lang_ref_Reference discovered; @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Custom, declClass = ComputeQueueValue.class) // @@ -226,9 +226,9 @@ public Object transform(Object receiver, Object originalValue) { } @Platforms(Platform.HOSTED_ONLY.class) -class NotMarkAndCopyOrEpsilonGC implements BooleanSupplier { +class NotSerialNotParallelNotEpsilonGC implements BooleanSupplier { @Override public boolean getAsBoolean() { - return !SubstrateOptions.useMarkAndCopyOrEpsilonGC(); + return !SubstrateOptions.useSerialOrParallelOrEpsilonGC(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 555493b3ea70..ed7a9e0cf667 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -896,7 +896,7 @@ public static int getSafepointBehaviorVolatile(IsolateThread vmThread) { * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * thread will also actively prevent the VM from reaching a safepoint (regardless of its * thread status). - * + * * NOTE: Be careful with this method and make sure that this thread does not allocate any * Java objects as this could result deadlocks. This method will only prevent safepoints * reliably if it is called from a thread with {@link StatusSupport#STATUS_IN_JAVA}. @@ -910,7 +910,7 @@ public static void preventSafepoints() { /** * Marks the thread as crashed. This method may only be used in places where it is not * possible to safely detach a thread. - * + * * Changes the safepoint behavior so that this thread won't freeze at a safepoint. The * safepoint handling will ignore the thread so that the VM can reach a safepoint regardless * of the status of this thread. diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java index 15e58be1f46f..4d26a75781ec 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java @@ -127,6 +127,11 @@ private void checkSpecifiedOptions(HostedMethod method, Uninterruptible annotati violations.add("Method " + method.format("%H.%n(%p)") + " uses an unspecific reason but is annotated with 'callerMustBe = true'. Please document in the reason why the callers need to be uninterruptible."); } + + if (!annotation.calleeMustBe()) { + violations.add("Method " + method.format("%H.%n(%p)") + + " uses an unspecific reason but is annotated with 'calleeMustBe = false'. Please document in the reason why it is safe to execute interruptible code."); + } } else if (isSimilarToUnspecificReason(annotation.reason())) { violations.add("Method " + method.format("%H.%n(%p)") + " uses a reason that is similar to the unspecific reason '" + Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE + "'. " + "If the method has an inherent reason for being uninterruptible, besides being called from uninterruptible code, then please improve the reason. " + From cc7b1070e2596937a3536d53b8c84f2f60871332 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 2 May 2023 14:36:27 +0200 Subject: [PATCH 102/125] Fix counter initialization. --- .../genscavenge/GreyToBlackObjRefVisitor.java | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 45e3adf3f64c..535e14ed0e17 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.jdk.UninterruptibleUtils; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -37,6 +36,7 @@ import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; import com.oracle.svm.core.log.Log; /** @@ -63,19 +63,13 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceUninterruptibly(objRef, 0, compressed, holderObject); + return visitObjectReferenceInline(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - return visitObjectReferenceUninterruptibly(objRef, innerOffset, compressed, holderObject); - } - - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private boolean visitObjectReferenceUninterruptibly(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); counters.noteObjRef(); @@ -160,19 +154,17 @@ public interface Counters extends AutoCloseable { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteUnmodifiedReference(); - void toLog(); - void reset(); } public static class RealCounters implements Counters { - private UninterruptibleUtils.AtomicLong objRef; - private UninterruptibleUtils.AtomicLong nullObjRef; - private UninterruptibleUtils.AtomicLong nullReferent; - private UninterruptibleUtils.AtomicLong forwardedReferent; - private UninterruptibleUtils.AtomicLong nonHeapReferent; - private UninterruptibleUtils.AtomicLong copiedReferent; - private UninterruptibleUtils.AtomicLong unmodifiedReference; + private final AtomicLong objRef = new AtomicLong(0); + private final AtomicLong nullObjRef = new AtomicLong(0); + private final AtomicLong nullReferent = new AtomicLong(0); + private final AtomicLong forwardedReferent = new AtomicLong(0); + private final AtomicLong nonHeapReferent = new AtomicLong(0); + private final AtomicLong copiedReferent = new AtomicLong(0); + private final AtomicLong unmodifiedReference = new AtomicLong(0); RealCounters() { reset(); @@ -237,8 +229,7 @@ public void noteUnmodifiedReference() { unmodifiedReference.incrementAndGet(); } - @Override - public void toLog() { + private void toLog() { Log log = Log.log(); log.string("[GreyToBlackObjRefVisitor.counters:"); log.string(" objRef: ").signed(objRef.get()); @@ -296,10 +287,6 @@ public void noteCopiedReferent() { public void noteUnmodifiedReference() { } - @Override - public void toLog() { - } - @Override public void reset() { } From c4957c6ecf0b5671dd5862f527e95cf5f978f21b Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 2 May 2023 17:48:13 +0200 Subject: [PATCH 103/125] Fix an overflow. --- .../com/oracle/svm/core/IsolateArgumentParser.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java index 9e0bbf3e0905..65b3ae40c224 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java @@ -55,7 +55,7 @@ @AutomaticallyRegisteredImageSingleton public class IsolateArgumentParser { private static final RuntimeOptionKey[] OPTIONS = {SubstrateGCOptions.MinHeapSize, SubstrateGCOptions.MaxHeapSize, SubstrateGCOptions.MaxNewSize, - SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData, SubstrateOptions.ParallelGCThreads}; + SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling, SubstrateOptions.ConcealedOptions.UsePerfData}; private static final int OPTION_COUNT = OPTIONS.length; private static final CGlobalData OPTION_NAMES = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNames); private static final CGlobalData OPTION_NAME_POSITIONS = CGlobalDataFactory.createBytes(IsolateArgumentParser::createOptionNamePosition); @@ -63,10 +63,10 @@ public class IsolateArgumentParser { private static final CGlobalData HOSTED_VALUES = CGlobalDataFactory.createBytes(IsolateArgumentParser::createHostedValues); private static final long[] PARSED_OPTION_VALUES = new long[OPTION_COUNT]; - private static final int K = 1024; - private static final int M = K * K; - private static final int G = K * M; - private static final int T = K * G; + private static final long K = 1024; + private static final long M = K * K; + private static final long G = K * M; + private static final long T = K * G; @Platforms(Platform.HOSTED_ONLY.class) public IsolateArgumentParser() { @@ -346,7 +346,7 @@ private static boolean atojulong(CCharPointer s, CLongPointer result) { return false; } - int modifier; + long modifier; switch (tail.read()) { case 'T': case 't': @@ -371,7 +371,7 @@ private static boolean atojulong(CCharPointer s, CLongPointer result) { return false; } - UnsignedWord value = n.multiply(modifier); + UnsignedWord value = n.multiply(WordFactory.unsigned(modifier)); if (checkForOverflow(value, n, modifier)) { return false; } From ad940e93669a6233c62051ae2626da3f30ddd402 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 3 May 2023 17:06:50 +0200 Subject: [PATCH 104/125] Fix a race that could result in an incorrect remembered set. --- .../oracle/svm/core/genscavenge/GCImpl.java | 21 +- .../svm/core/genscavenge/Generation.java | 44 -- .../svm/core/genscavenge/OldGeneration.java | 18 +- .../oracle/svm/core/genscavenge/Space.java | 406 ++++++++++-------- .../genscavenge/ThreadLocalAllocation.java | 4 +- .../svm/core/genscavenge/YoungGeneration.java | 17 +- .../com/oracle/svm/core/thread/VMThreads.java | 10 + 7 files changed, 269 insertions(+), 251 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 2cfd09e16ccb..a170d9b3ebfa 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -199,7 +199,7 @@ private void collectOperation(CollectionVMOperationData data) { assert VMOperation.isGCInProgress() : "Collection should be a VMOperation."; assert getCollectionEpoch().equal(data.getRequestingEpoch()); - if (SubstrateOptions.UseParallelGC.getValue()) { + if (ParallelGC.isEnabled()) { ParallelGC.singleton().initialize(); } @@ -1101,20 +1101,23 @@ private static void scanGreyObjectsLoop() { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @SuppressWarnings("static-method") Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header); Header originalChunk = getChunk(original, isAligned); + + /* If the parallel GC is used, then the space may be outdated or null. */ Space originalSpace = HeapChunk.getSpace(originalChunk); - if (!originalSpace.isFromSpace()) { + assert originalSpace != null || ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase(); + if (originalSpace == null || !originalSpace.isFromSpace()) { + /* Object was already promoted or is currently being promoted. */ return original; } Object result = null; if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) { if (isAligned) { - result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + result = heap.getYoungGeneration().promoteAlignedObject(original, originalSpace); } else { result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); } @@ -1122,11 +1125,13 @@ Object promoteObject(Object original, UnsignedWord header) { accounting.onSurvivorOverflowed(); } } - if (result == null) { // complete collection, tenuring age reached, or survivor space full + + /* Complete collection, tenuring age reached, or survivor space full. */ + if (result == null) { if (isAligned) { - result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace); + result = heap.getOldGeneration().promoteAlignedObject(original, originalSpace); } else { - result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace); + result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk); } assert result != null : "promotion failure in old generation must have been handled"; } @@ -1160,7 +1165,7 @@ private void promotePinnedObject(PinnedObjectImpl pinned) { } } if (!promoted) { - heap.getOldGeneration().promoteChunk(originalChunk, isAligned, originalSpace); + heap.getOldGeneration().promoteChunk(originalChunk, isAligned); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java index e487a4c7f1d2..d5559bd77e52 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java @@ -27,8 +27,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; @@ -55,46 +53,4 @@ public String getName() { /** Report some statistics about the Generation to a Log. */ public abstract Log report(Log log, boolean traceHeapChunks); - - /** - * Promote an Object to this Generation, typically by copying and leaving a forwarding pointer - * to the new Object in place of the original Object. If the object cannot be promoted due to - * insufficient capacity, returns {@code null}. - * - * This turns an Object from white to grey: the object is in this Generation, but has not yet - * had its interior pointers visited. - * - * @return a reference to the promoted object, which is different to the original reference if - * promotion was done by copying, or {@code null} if there was insufficient capacity in - * this generation. - */ - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace); - - /** - * Promote an Object to this Generation, typically by HeapChunk motion. If the object cannot be - * promoted due to insufficient capacity, returns {@code null}. - * - * This turns an Object from white to grey: the object is in this Generation, but has not yet - * had its interior pointers visited. - * - * @return a reference to the promoted object, which is the same as the original if the object - * was promoted through HeapChunk motion, or {@code null} if there was insufficient - * capacity in this generation. - */ - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace); - - /** - * Promote a HeapChunk from its original space to the appropriate space in this generation if - * there is sufficient capacity. - * - * This turns all the Objects in the chunk from white to grey: the objects are in the target - * Space, but have not yet had their interior pointers visited. - * - * @return true on success, false if the there was insufficient capacity. - */ - protected abstract boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 8c77d261b9b5..ca5f7e0a7abe 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -74,31 +74,25 @@ public boolean walkObjects(ObjectVisitor visitor) { /** Promote an Object to ToSpace if it is not already in ToSpace. */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @Override - public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { + Object promoteAlignedObject(Object original, Space originalSpace) { assert originalSpace.isFromSpace(); return getToSpace().promoteAlignedObject(original, originalSpace); } @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @Override - protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { - assert originalSpace.isFromSpace(); - getToSpace().promoteUnalignedHeapChunk(originalChunk, originalSpace); + Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk) { + getToSpace().promoteUnalignedHeapChunk(originalChunk); return original; } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { - assert originalSpace.isFromSpace(); + void promoteChunk(HeapChunk.Header originalChunk, boolean isAligned) { if (isAligned) { - getToSpace().promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace); + getToSpace().promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk); } else { - getToSpace().promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace); + getToSpace().promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk); } - return true; } void releaseSpaces(ChunkReleaser chunkReleaser) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index bcec821287a8..ab557de22b26 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -180,14 +180,19 @@ private Pointer allocateMemory(UnsignedWord objectSize) { if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { return allocateMemoryParallel(objectSize); } - Pointer result = WordFactory.nullPointer(); + return allocateMemorySerial(objectSize); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer allocateMemorySerial(UnsignedWord objectSize) { /* Fast-path: try allocating in the last chunk. */ AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); if (oldChunk.isNonNull()) { - result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); - } - if (result.isNonNull()) { - return result; + Pointer result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); + if (result.isNonNull()) { + return result; + } } /* Slow-path: try allocating a new chunk for the requested memory. */ return allocateInNewChunk(objectSize); @@ -196,24 +201,43 @@ private Pointer allocateMemory(UnsignedWord objectSize) { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemoryParallel(UnsignedWord objectSize) { - Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the thread local allocation chunk. */ AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); if (oldChunk.isNonNull()) { - result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); - } - if (result.isNonNull()) { - return result; + Pointer result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); + if (result.isNonNull()) { + return result; + } } /* Slow-path: try allocating a new chunk for the requested memory. */ return allocateInNewChunkParallel(oldChunk, objectSize); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { + AlignedHeapChunk.AlignedHeader newChunk; + ParallelGC.singleton().getMutex().lockNoTransitionUnspecifiedOwner(); + try { + if (oldChunk.isNonNull()) { + ParallelGC.singleton().pushAllocChunk(oldChunk); + } + newChunk = requestAlignedHeapChunk(); + } finally { + ParallelGC.singleton().getMutex().unlockNoTransitionUnspecifiedOwner(); + } + + ParallelGC.singleton().setAllocationChunk(newChunk); + if (newChunk.isNonNull()) { + return AlignedHeapChunk.allocateMemory(newChunk, objectSize); + } + return WordFactory.nullPointer(); + } + /** Retract the latest allocation. */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static void retractAllocation(UnsignedWord objectSize) { - assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); + private static void retractAllocationParallel(UnsignedWord objectSize) { + assert ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase(); AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); assert oldChunk.isNonNull(); AlignedHeapChunk.retractAllocation(oldChunk, objectSize); @@ -229,22 +253,6 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { - AlignedHeapChunk.AlignedHeader newChunk; - ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); - try { - ParallelGC.singleton().pushAllocChunk(oldChunk); - newChunk = requestAlignedHeapChunk(); - } finally { - ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); - } - if (newChunk.isNonNull()) { - ParallelGC.singleton().setAllocationChunk(newChunk); - return AlignedHeapChunk.allocateMemory(newChunk, objectSize); - } - return WordFactory.nullPointer(); - } - public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); @@ -256,39 +264,17 @@ public void releaseChunks(ChunkReleaser chunkReleaser) { accounting.reset(); } - @Uninterruptible(reason = "Must not interact with garbage collections.") - void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { - appendAlignedHeapChunk(aChunk, null); - } - @Uninterruptible(reason = "Must not interact with garbage collections.") void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk, Space originalSpace) { - /* - * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not - * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. - */ - if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { - VMThreads.guaranteeOwnsThreadMutex("Trying to append an aligned heap chunk but no mutual exclusion.", true); - } - HeapChunk.setSpace(aChunk, this); + assert verifyMutualExclusionForAppendChunk() : "Trying to append an aligned heap chunk but no mutual exclusion."; + assert HeapChunk.getSpace(aChunk) == originalSpace; + assert this != originalSpace; if (originalSpace != null) { - assert VMOperation.isGCInProgress() : "Should only be called by the collector."; - AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); - AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - originalSpace.setFirstAlignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - originalSpace.setLastAlignedHeapChunk(chunkPrev); - } - originalSpace.accounting.unnoteAlignedHeapChunk(); + originalSpace.extractAlignedHeapChunk(aChunk); } + HeapChunk.setSpace(aChunk, this); AlignedHeapChunk.AlignedHeader oldLast = getLastAlignedHeapChunk(); HeapChunk.setPrevious(aChunk, oldLast); HeapChunk.setNext(aChunk, WordFactory.nullPointer()); @@ -302,39 +288,17 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk, Space origina accounting.noteAlignedHeapChunk(); } - @Uninterruptible(reason = "Must not interact with garbage collections.") - void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { - appendUnalignedHeapChunk(uChunk, null); - } - @Uninterruptible(reason = "Must not interact with garbage collections.") void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk, Space originalSpace) { - /* - * This method is used from {@link PosixJavaThreads#detachThread(VMThread)}, so it can not - * guarantee that it is inside a VMOperation, only that there is some mutual exclusion. - */ - if (SubstrateOptions.MultiThreaded.getValue() && !(SubstrateOptions.UseParallelGC.getValue() && VMOperation.isGCInProgress())) { - VMThreads.guaranteeOwnsThreadMutex("Trying to append an unaligned chunk but no mutual exclusion.", true); - } - HeapChunk.setSpace(uChunk, this); + assert verifyMutualExclusionForAppendChunk() : "Trying to append an aligned heap chunk but no mutual exclusion."; + assert HeapChunk.getSpace(uChunk) == originalSpace; + assert this != originalSpace; if (originalSpace != null) { - assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; - UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); - UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); - if (chunkPrev.isNonNull()) { - HeapChunk.setNext(chunkPrev, chunkNext); - } else { - originalSpace.setFirstUnalignedHeapChunk(chunkNext); - } - if (chunkNext.isNonNull()) { - HeapChunk.setPrevious(chunkNext, chunkPrev); - } else { - originalSpace.setLastUnalignedHeapChunk(chunkPrev); - } - originalSpace.accounting.unnoteUnalignedHeapChunk(uChunk); + originalSpace.extractUnalignedHeapChunk(uChunk); } + HeapChunk.setSpace(uChunk, this); UnalignedHeapChunk.UnalignedHeader oldLast = getLastUnalignedHeapChunk(); HeapChunk.setPrevious(uChunk, oldLast); HeapChunk.setNext(uChunk, WordFactory.nullPointer()); @@ -348,6 +312,58 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk, Space o accounting.noteUnalignedHeapChunk(uChunk); } + @Uninterruptible(reason = "Must not interact with garbage collections.") + private void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { + assert VMOperation.isGCInProgress(); + + AlignedHeapChunk.AlignedHeader chunkNext = HeapChunk.getNext(aChunk); + AlignedHeapChunk.AlignedHeader chunkPrev = HeapChunk.getPrevious(aChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + setFirstAlignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + setLastAlignedHeapChunk(chunkPrev); + } + HeapChunk.setNext(aChunk, WordFactory.nullPointer()); + HeapChunk.setPrevious(aChunk, WordFactory.nullPointer()); + HeapChunk.setSpace(aChunk, null); + accounting.unnoteAlignedHeapChunk(); + } + + @Uninterruptible(reason = "Must not interact with garbage collections.") + private void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { + assert VMOperation.isGCInProgress(); + + UnalignedHeapChunk.UnalignedHeader chunkNext = HeapChunk.getNext(uChunk); + UnalignedHeapChunk.UnalignedHeader chunkPrev = HeapChunk.getPrevious(uChunk); + if (chunkPrev.isNonNull()) { + HeapChunk.setNext(chunkPrev, chunkNext); + } else { + setFirstUnalignedHeapChunk(chunkNext); + } + if (chunkNext.isNonNull()) { + HeapChunk.setPrevious(chunkNext, chunkPrev); + } else { + setLastUnalignedHeapChunk(chunkPrev); + } + /* Reset the fields that the result chunk keeps for Space. */ + HeapChunk.setNext(uChunk, WordFactory.nullPointer()); + HeapChunk.setPrevious(uChunk, WordFactory.nullPointer()); + HeapChunk.setSpace(uChunk, null); + accounting.unnoteUnalignedHeapChunk(uChunk); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean verifyMutualExclusionForAppendChunk() { + return !SubstrateOptions.MultiThreaded.getValue() || + VMThreads.ownsThreadMutex(true) || + ParallelGC.isEnabled() && VMOperation.isGCInProgress() && ParallelGC.singleton().isInParallelPhase() && ParallelGC.singleton().getMutex().isOwner(); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public AlignedHeapChunk.AlignedHeader getFirstAlignedHeapChunk() { return firstAlignedHeapChunk; @@ -395,22 +411,65 @@ Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); - if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { + if (ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase()) { return copyAlignedObjectParallel(original); } + return copyAlignedObjectSerial(original); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Object copyAlignedObjectSerial(Object originalObj) { + assert VMOperation.isGCInProgress(); + assert ObjectHeaderImpl.isAlignedObject(originalObj); + + UnsignedWord originalSize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, false); + UnsignedWord copySize = originalSize; + boolean addIdentityHashField = false; + if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) { + Word header = ObjectHeader.readHeaderFromObject(originalObj); + if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(header))) { + addIdentityHashField = true; + copySize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, true); + } + } - Object copy = copyAlignedObject(original); - if (copy != null) { - ObjectHeaderImpl.getObjectHeaderImpl().installForwardingPointer(original, copy); + Pointer copyMemory = allocateMemory(copySize); + if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { + return null; } + + /* + * This does a direct memory copy, without regard to whether the copied data contains object + * references. That's okay, because all references in the copy are visited and overwritten + * later on anyways (the card table is also updated at that point if necessary). + */ + Pointer originalMemory = Word.objectToUntrackedPointer(originalObj); + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, originalSize); + + Object copy = copyMemory.toObject(); + if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { + // Must do first: ensures correct object size below and in other places + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalObj); + int offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); + ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); + ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); + } + if (isOldSpace()) { + // If the object was promoted to the old gen, we need to take care of the remembered + // set bit and the first object table (even when promoting from old to old). + AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); + RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); + } + + ObjectHeaderImpl.getObjectHeaderImpl().installForwardingPointer(originalObj, copy); return copy; } @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - Object copyAlignedObjectParallel(Object original) { + private Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); - assert ParallelGC.isEnabled() && ParallelGC.isInParallelPhase(); assert ObjectHeaderImpl.isAlignedObject(original); /* @@ -426,8 +485,10 @@ Object copyAlignedObjectParallel(Object original) { return ohi.getForwardedObject(originalMemory, originalHeader); } - // We need forwarding pointer to point somewhere, so we speculatively allocate memory here. - // If another thread copies the object first, we retract the allocation later. + /* + * We need the forwarding pointer to point somewhere, so we speculatively allocate memory + * here. If another thread copies the object first, we retract the allocation later. + */ UnsignedWord originalSize = LayoutEncoding.getSizeFromHeader(original, originalHeader, false); UnsignedWord copySize = originalSize; boolean addIdentityHashField = false; @@ -444,79 +505,30 @@ Object copyAlignedObjectParallel(Object original) { return null; } - // Install forwarding pointer into the original header + /* Install forwarding pointer into the original header. */ Object copy = copyMemory.toObject(); Object forward = ohi.installForwardingPointerParallel(original, eightHeaderBytes, copy); - if (forward == copy) { - // We have won the race, now we must copy the object bits. First install the original - // header - copyMemory.writeLong(hubOffset, eightHeaderBytes); - // Copy the rest of original object - if (hubOffset > 0) { - UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); - } - int offset = hubOffset + Long.BYTES; - UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); - - if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { - int value = IdentityHashCodeSupport.computeHashCodeFromAddress(original); - offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); - ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); - ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); - } - - if (isOldSpace()) { - // If the object was promoted to the old gen, we need to take care of the remembered - // set bit and the first object table (even when promoting from old to old). - AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy); - RememberedSet.get().enableRememberedSetForObject(copyChunk, copy); - } - return copy; - } else { - // Retract speculatively allocated memory - retractAllocation(originalSize); + if (forward != copy) { + /* We lost the race. Retract speculatively allocated memory. */ + retractAllocationParallel(originalSize); return forward; } - } - - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private Object copyAlignedObject(Object originalObj) { - assert VMOperation.isGCInProgress(); - assert ObjectHeaderImpl.isAlignedObject(originalObj); - - UnsignedWord originalSize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, false); - UnsignedWord copySize = originalSize; - boolean addIdentityHashField = false; - if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) { - Word header = ObjectHeader.readHeaderFromObject(originalObj); - if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(header))) { - addIdentityHashField = true; - copySize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, true); - } - } - Pointer copyMemory = allocateMemory(copySize); - if (probability(VERY_SLOW_PATH_PROBABILITY, copyMemory.isNull())) { - return null; + /* We have won the race. Install the object header and copy the rest of the object. */ + copyMemory.writeLong(hubOffset, eightHeaderBytes); + if (hubOffset > 0) { + UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } + int offset = hubOffset + Long.BYTES; + UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); - /* - * This does a direct memory copy, without regard to whether the copied data contains object - * references. That's okay, because all references in the copy are visited and overwritten - * later on anyways (the card table is also updated at that point if necessary). - */ - Pointer originalMemory = Word.objectToUntrackedPointer(originalObj); - UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, originalSize); - - Object copy = copyMemory.toObject(); if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { - // Must do first: ensures correct object size below and in other places - int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalObj); - int offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(original); + offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); } + if (isOldSpace()) { // If the object was promoted to the old gen, we need to take care of the remembered // set bit and the first object table (even when promoting from old to old). @@ -528,23 +540,45 @@ private Object copyAlignedObject(Object originalObj) { /** Promote an AlignedHeapChunk by moving it to this space. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { - assert this != originalSpace && originalSpace.isFromSpace(); + void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk) { + if (ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase()) { + promoteAlignedHeapChunkParallel(chunk); + } else { + promoteAlignedHeapChunkSerial(chunk); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void promoteAlignedHeapChunkSerial(AlignedHeapChunk.AlignedHeader chunk) { + Space originalSpace = HeapChunk.getSpace(chunk); + promoteAlignedHeapChunk0(chunk, originalSpace); - if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(chunk); } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void promoteAlignedHeapChunkParallel(AlignedHeapChunk.AlignedHeader chunk) { + ParallelGC.singleton().getMutex().lockNoTransitionUnspecifiedOwner(); try { - appendAlignedHeapChunk(chunk, originalSpace); - } finally { - if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGC.singleton().push(chunk); - if (ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); - } + Space originalSpace = HeapChunk.getSpace(chunk); + if (!originalSpace.isFromSpace()) { + /* The chunk was already promoted in the meanwhile. */ + return; } + promoteAlignedHeapChunk0(chunk, originalSpace); + ParallelGC.singleton().push(chunk); + } finally { + ParallelGC.singleton().getMutex().unlockNoTransitionUnspecifiedOwner(); } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void promoteAlignedHeapChunk0(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { + assert originalSpace.isFromSpace(); + appendAlignedHeapChunk(chunk, originalSpace); if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { RememberedSet.get().enableRememberedSetForChunk(chunk); @@ -557,23 +591,45 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina /** Promote an UnalignedHeapChunk by moving it to this Space. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { - assert this != originalSpace && originalSpace.isFromSpace(); + void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { + if (ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase()) { + promoteUnalignedHeapChunkParallel(chunk); + } else { + promoteUnalignedHeapChunkSerial(chunk); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void promoteUnalignedHeapChunkSerial(UnalignedHeapChunk.UnalignedHeader chunk) { + Space originalSpace = HeapChunk.getSpace(chunk); + promoteUnalignedHeapChunk0(chunk, originalSpace); - if (ParallelGC.isEnabled() && ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.lockNoTransitionUnspecifiedOwner(); + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(chunk); } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void promoteUnalignedHeapChunkParallel(UnalignedHeapChunk.UnalignedHeader chunk) { + ParallelGC.singleton().getMutex().lockNoTransitionUnspecifiedOwner(); try { - appendUnalignedHeapChunk(chunk, originalSpace); - } finally { - if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGC.singleton().push(chunk); - if (ParallelGC.isInParallelPhase()) { - ParallelGC.mutex.unlockNoTransitionUnspecifiedOwner(); - } + Space originalSpace = HeapChunk.getSpace(chunk); + if (!originalSpace.isFromSpace()) { + /* The chunk was already promoted in the meanwhile. */ + return; } + promoteUnalignedHeapChunk0(chunk, originalSpace); + ParallelGC.singleton().push(chunk); + } finally { + ParallelGC.singleton().getMutex().unlockNoTransitionUnspecifiedOwner(); } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void promoteUnalignedHeapChunk0(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { + assert originalSpace.isFromSpace(); + appendUnalignedHeapChunk(chunk, originalSpace); if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { RememberedSet.get().enableRememberedSetForChunk(chunk); @@ -594,7 +650,7 @@ private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { chunk = HeapImpl.getHeapImpl().getOldGeneration().requestAlignedChunk(); } if (chunk.isNonNull()) { - appendAlignedHeapChunk(chunk); + appendAlignedHeapChunk(chunk, null); } return chunk; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index e42ad9153ec2..15293ed0ea39 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -475,14 +475,14 @@ private static void retireTlabToEden(IsolateThread thread) { while (alignedChunk.isNonNull()) { AlignedHeader next = HeapChunk.getNext(alignedChunk); HeapChunk.setNext(alignedChunk, WordFactory.nullPointer()); - eden.appendAlignedHeapChunk(alignedChunk); + eden.appendAlignedHeapChunk(alignedChunk, null); alignedChunk = next; } while (unalignedChunk.isNonNull()) { UnalignedHeader next = HeapChunk.getNext(unalignedChunk); HeapChunk.setNext(unalignedChunk, WordFactory.nullPointer()); - eden.appendUnalignedHeapChunk(unalignedChunk); + eden.appendUnalignedHeapChunk(unalignedChunk, null); unalignedChunk = next; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index d7f337a6a0b4..e378e2e60c6f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -253,10 +253,9 @@ public boolean contains(Object object) { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @Override - protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { - assert originalSpace.isFromSpace(); + Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); + assert originalSpace.isFromSpace(); assert originalSpace.getAge() < maxSurvivorSpaces; // The object might fit in an existing chunk in the survivor space. If it doesn't, we get @@ -269,8 +268,7 @@ protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedH @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @Override - protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { + Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); assert originalSpace.getAge() < maxSurvivorSpaces; if (!unalignedChunkFitsInSurvivors(originalChunk)) { @@ -279,13 +277,12 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal int age = originalSpace.getNextAgeForPromotion(); Space toSpace = getSurvivorToSpaceAt(age - 1); - toSpace.promoteUnalignedHeapChunk(originalChunk, originalSpace); + toSpace.promoteUnalignedHeapChunk(originalChunk); return original; } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { + boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { assert originalSpace.isFromSpace(); assert originalSpace.getAge() < maxSurvivorSpaces; if (!fitsInSurvivors(originalChunk, isAligned)) { @@ -295,9 +292,9 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig int age = originalSpace.getNextAgeForPromotion(); Space toSpace = getSurvivorToSpaceAt(age - 1); if (isAligned) { - toSpace.promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk, originalSpace); + toSpace.promoteAlignedHeapChunk((AlignedHeapChunk.AlignedHeader) originalChunk); } else { - toSpace.promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk, originalSpace); + toSpace.promoteUnalignedHeapChunk((UnalignedHeapChunk.UnalignedHeader) originalChunk); } return true; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index ed7a9e0cf667..05864e077f83 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -617,6 +617,16 @@ public static void guaranteeOwnsThreadMutex(String message, boolean allowUnspeci THREAD_MUTEX.guaranteeIsOwner(message, allowUnspecifiedOwner); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean ownsThreadMutex() { + return THREAD_MUTEX.isOwner(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean ownsThreadMutex(boolean allowUnspecifiedOwner) { + return THREAD_MUTEX.isOwner(allowUnspecifiedOwner); + } + public static boolean printLocationInfo(Log log, UnsignedWord value, boolean allowUnsafeOperations) { for (IsolateThread thread = firstThreadUnsafe(); thread.isNonNull(); thread = nextThread(thread)) { if (thread.equal(value)) { From b338cb34c40587916d84bf75dd2561112f5f042c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 4 May 2023 16:22:24 +0200 Subject: [PATCH 105/125] Support unattached threads in VMError.shouldNotReachHere(). --- .../src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java index bd7b2bb953a2..7a0391c20f85 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VMErrorSubstitutions.java @@ -27,7 +27,9 @@ import static com.oracle.svm.core.heap.RestrictHeapAccess.Access.NO_ALLOCATION; import org.graalvm.compiler.nodes.UnreachableNode; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Isolate; import org.graalvm.nativeimage.LogHandler; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.function.CodePointer; @@ -35,9 +37,11 @@ import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateDiagnostics; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; +import com.oracle.svm.core.c.function.CEntryPointActions; import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.snippets.KnownIntrinsics; @@ -135,6 +139,10 @@ public class VMErrorSubstitutions { static RuntimeException shouldNotReachHere(CodePointer callerIP, String msg, Throwable ex) { ThreadStackPrinter.printBacktrace(); + if (SubstrateOptions.SpawnIsolates.getValue() && CurrentIsolate.getCurrentThread().isNull()) { + CEntryPointActions.enterAttachThreadFromCrashHandler((Isolate) KnownIntrinsics.heapBase()); + } + SafepointBehavior.preventSafepoints(); StackOverflowCheck.singleton().disableStackOverflowChecksForFatalError(); From c698f8029debf4984cc417dfaabb02ec10350100 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 4 May 2023 16:22:56 +0200 Subject: [PATCH 106/125] Various cleanups and fixes for the parallel GC. --- .../oracle/svm/core/genscavenge/Space.java | 10 +- .../genscavenge/jvmstat/SerialGCPerfData.java | 6 +- .../{ChunkBuffer.java => ChunkQueue.java} | 41 +-- .../core/genscavenge/parallel/ParallelGC.java | 292 ++++++++++-------- .../oracle/svm/core/heap/OutOfMemoryUtil.java | 13 +- .../svm/core/thread/PlatformThreads.java | 5 + 6 files changed, 205 insertions(+), 162 deletions(-) rename substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/{ChunkBuffer.java => ChunkQueue.java} (74%) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index ab557de22b26..3f47f3ae7558 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -210,17 +210,15 @@ private Pointer allocateMemoryParallel(UnsignedWord objectSize) { } } /* Slow-path: try allocating a new chunk for the requested memory. */ - return allocateInNewChunkParallel(oldChunk, objectSize); + return allocateInNewChunkParallel(objectSize); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private Pointer allocateInNewChunkParallel(AlignedHeapChunk.AlignedHeader oldChunk, UnsignedWord objectSize) { + private Pointer allocateInNewChunkParallel(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk; ParallelGC.singleton().getMutex().lockNoTransitionUnspecifiedOwner(); try { - if (oldChunk.isNonNull()) { - ParallelGC.singleton().pushAllocChunk(oldChunk); - } + ParallelGC.singleton().pushAllocChunk(); newChunk = requestAlignedHeapChunk(); } finally { ParallelGC.singleton().getMutex().unlockNoTransitionUnspecifiedOwner(); @@ -361,7 +359,7 @@ private void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk private static boolean verifyMutualExclusionForAppendChunk() { return !SubstrateOptions.MultiThreaded.getValue() || VMThreads.ownsThreadMutex(true) || - ParallelGC.isEnabled() && VMOperation.isGCInProgress() && ParallelGC.singleton().isInParallelPhase() && ParallelGC.singleton().getMutex().isOwner(); + ParallelGC.isEnabled() && VMOperation.isGCInProgress() && ParallelGC.singleton().isInParallelPhase() && ParallelGC.singleton().getMutex().isOwner(true); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/jvmstat/SerialGCPerfData.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/jvmstat/SerialGCPerfData.java index 4fe4410401eb..b9e72d73cc38 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/jvmstat/SerialGCPerfData.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/jvmstat/SerialGCPerfData.java @@ -34,6 +34,7 @@ import com.oracle.svm.core.genscavenge.HeapAccounting; import com.oracle.svm.core.genscavenge.HeapImpl; import com.oracle.svm.core.genscavenge.HeapParameters; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.jvmstat.PerfDataHolder; import com.oracle.svm.core.jvmstat.PerfLongConstant; import com.oracle.svm.core.jvmstat.PerfLongCounter; @@ -45,6 +46,7 @@ /** * Performance data for our serial GC. */ +// TODO (chaeubl): rename this class public class SerialGCPerfData implements PerfDataHolder { private final PerfDataGCPolicy gcPolicy; private final PerfDataCollector youngCollector; @@ -78,7 +80,9 @@ public void allocate() { gcPolicy.allocate(); youngCollector.allocate("Serial young collection pauses"); - oldCollector.allocate("Serial full collection pauses"); + + String oldCollectorName = ParallelGC.isEnabled() ? "Parallel" : "Serial"; + oldCollector.allocate(oldCollectorName + " full collection pauses"); youngGen.allocate("young"); youngGen.spaces[0].allocate("eden"); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkQueue.java similarity index 74% rename from substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java rename to substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkQueue.java index dad08688b81e..b0c76e4d3c0c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkBuffer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ChunkQueue.java @@ -40,14 +40,15 @@ import com.oracle.svm.core.util.VMError; /** - * Synchronized buffer that stores pointers into "grey" heap chunks that need to be scanned. Note - * that the pointers don't necessarily point to the beginning of a chunk. + * A queue that stores pointers into "grey" heap chunks that need to be scanned. Note that the + * pointers don't necessarily point to the beginning of a chunk. GC workers threads may only access + * the queue if they hold {@link ParallelGC#getMutex()}. */ -public class ChunkBuffer { +public class ChunkQueue { private static final int INITIAL_SIZE = 1024 * wordSize(); private Pointer buffer; - private int size = INITIAL_SIZE; + private int size; private int top; @Fold @@ -56,23 +57,26 @@ static int wordSize() { } @Platforms(Platform.HOSTED_ONLY.class) - ChunkBuffer() { + ChunkQueue() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void initialize() { + assert top == 0 && size == 0 && buffer.isNull(); + top = 0; + size = INITIAL_SIZE; buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(size)); - VMError.guarantee(buffer.isNonNull()); + VMError.guarantee(buffer.isNonNull(), "Failed to allocate native memory for the ChunkBuffer."); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void push(Pointer ptr) { - assert !ParallelGC.singleton().isInParallelPhase() && VMThreads.ownsThreadMutex() || ParallelGC.singleton().getMutex().hasOwner(); + assert !ParallelGC.singleton().isInParallelPhase() && VMThreads.ownsThreadMutex() || ParallelGC.singleton().isInParallelPhase() && ParallelGC.singleton().getMutex().isOwner(true); if (top >= size) { size *= 2; assert top < size; buffer = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(buffer, WordFactory.unsigned(size)); - VMError.guarantee(buffer.isNonNull()); + VMError.guarantee(buffer.isNonNull(), "Failed to allocate native memory for the ChunkBuffer."); } buffer.writeWord(top, ptr); top += wordSize(); @@ -80,18 +84,12 @@ void push(Pointer ptr) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Pointer pop() { - assert ParallelGC.singleton().isInParallelPhase(); - ParallelGC.singleton().getMutex().lockNoTransitionUnspecifiedOwner(); - try { - if (top > 0) { - top -= wordSize(); - return buffer.readWord(top); - } else { - return WordFactory.nullPointer(); - } - } finally { - ParallelGC.singleton().getMutex().unlockNoTransitionUnspecifiedOwner(); + assert ParallelGC.singleton().isInParallelPhase() && ParallelGC.singleton().getMutex().isOwner(true); + if (top > 0) { + top -= wordSize(); + return buffer.readWord(top); } + return WordFactory.nullPointer(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -101,7 +99,10 @@ boolean isEmpty() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void release() { + void teardown() { ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + buffer = WordFactory.nullPointer(); + size = 0; + top = 0; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 0f6ce107b68b..24b8895593ed 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -58,7 +58,6 @@ import com.oracle.svm.core.genscavenge.UnalignedHeapChunk; import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; -import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.Jvm; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.locks.VMCondition; @@ -70,43 +69,45 @@ import com.oracle.svm.core.util.VMError; /** - * A garbage collector that tries to shorten GC pauses by using multiple "worker threads". - * Currently, the only phase supported is scanning grey objects during a full GC. The number of - * worker threads can be set with a runtime option (see {@link SubstrateOptions#ParallelGCThreads}). - * + * A garbage collector that tries to shorten GC pauses by using multiple worker threads. Currently, + * the only phase supported is scanning grey objects during a full GC. The number of worker threads + * can be set with a runtime option (see {@link SubstrateOptions#ParallelGCThreads}). + *

+ * The GC worker threads are unattached threads that are started lazily and that call AOT-compiled + * code. So, they don't have an {@link org.graalvm.nativeimage.IsolateThread} data structure and + * don't participate in the safepoint handling. + *

* Worker threads use heap chunks as the unit of work. Chunks to be scanned are stored in the - * synchronized {@link ChunkBuffer}. Worker threads pop chunks from the buffer and scan them for - * references to live objects to be promoted. When promoting an aligned chunk object, they - * speculatively allocate memory for its copy in the to-space, then compete to install forwarding - * pointer in the original object. The winning thread proceeds to copy object data, losing threads - * retract the speculatively allocated memory. - * - * Each worker thread allocates memory in its own thread local "allocation chunk" for speed. As - * allocation chunks become filled up, they are pushed to {@link ChunkBuffer}. This pop-scan-push + * {@link ChunkQueue}. Worker threads pop chunks from the queue and scan them for references to live + * objects to be promoted. When promoting an aligned chunk object, they speculatively allocate + * memory for its copy in the to-space, then compete to install forwarding pointer in the original + * object. The winning thread proceeds to copy object data, losing threads retract the speculatively + * allocated memory. + *

+ * Each worker thread allocates memory in its own thread local allocation chunk for speed. As + * allocation chunks become filled up, they are pushed to {@link ChunkQueue}. This pop-scan-push * cycle continues until the chunk buffer becomes empty. At this point, worker threads are parked * and the GC routine continues on the main GC thread. */ public class ParallelGC { - private static final int UNALIGNED_BIT = 0x01; + private static final int UNALIGNED_BIT = 0b01; private static final int MAX_WORKER_THREADS = 8; private final VMMutex mutex = new VMMutex("parallelGC"); private final VMCondition seqPhase = new VMCondition(mutex); private final VMCondition parPhase = new VMCondition(mutex); - private final ChunkBuffer buffer = new ChunkBuffer(); + private final ChunkQueue chunkQueue = new ChunkQueue(); private final CEntryPointLiteral gcWorkerRunFunc = CEntryPointLiteral.create(ParallelGC.class, "gcWorkerRun", GCWorkerThreadState.class); private boolean initialized; - private OSThreadHandlePointer workerThreads; + private ThreadLocalKey workerStateTL; private GCWorkerThreadState workerStates; + private OSThreadHandlePointer workerThreads; private int numWorkerThreads; private int busyWorkerThreads; - private ThreadLocalKey workerStateTL; private volatile boolean inParallelPhase; private volatile boolean shutdown; - private volatile Throwable error; - // TEMP (chaeubl): finish reviewing ChunkBuffer and ParallelGC. @Platforms(Platform.HOSTED_ONLY.class) public ParallelGC() { } @@ -121,12 +122,12 @@ public static boolean isEnabled() { return SubstrateOptions.UseParallelGC.getValue(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Called from a GC worker thread.") public boolean isInParallelPhase() { return inParallelPhase; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Called from a GC worker thread.") public VMMutex getMutex() { return mutex; } @@ -158,27 +159,35 @@ public void push(UnalignedHeapChunk.UnalignedHeader uChunk) { @Uninterruptible(reason = "Called from a GC worker thread.") private void push(Pointer ptr) { assert ptr.isNonNull(); - buffer.push(ptr); + chunkQueue.push(ptr); if (inParallelPhase) { + assert mutex.isOwner(true); parPhase.signal(); } } @Uninterruptible(reason = "Called from a GC worker thread.") - public void pushAllocChunk(AlignedHeapChunk.AlignedHeader chunk) { + public void pushAllocChunk() { assert GCImpl.getGCImpl().isCompleteCollection(); GCWorkerThreadState state = getWorkerThreadState(); - if (chunk.notEqual(state.getScannedChunk())) { - UnsignedWord scanOffset = state.getAllocChunkScanOffset(); - assert scanOffset.aboveThan(0); - if (chunk.getTopOffset().aboveThan(scanOffset)) { - Pointer ptrIntoChunk = HeapChunk.asPointer(chunk).add(scanOffset); - push(ptrIntoChunk); - } + AlignedHeapChunk.AlignedHeader chunk = state.getAllocChunk(); + if (chunk.isNull() || chunk.equal(state.getScannedChunk())) { + /* + * Scanning (and therefore enqueueing) is not necessary if we are already in the middle + * of scanning the chunk, or if we don't have a chunk. + */ + return; + } + + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); + assert scanOffset.aboveThan(0); + if (chunk.getTopOffset().aboveThan(scanOffset)) { + Pointer ptrIntoChunk = HeapChunk.asPointer(chunk).add(scanOffset); + push(ptrIntoChunk); } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = "Called from a GC worker thread.") private GCWorkerThreadState getWorkerThreadState() { if (CurrentIsolate.getCurrentThread().isNull()) { return PlatformThreads.singleton().getUnmanagedThreadLocalValue(workerStateTL); @@ -192,10 +201,10 @@ public void initialize() { } initialized = true; - buffer.initialize(); + inParallelPhase = true; + chunkQueue.initialize(); workerStateTL = PlatformThreads.singleton().createUnmanagedThreadLocal(); - numWorkerThreads = getWorkerCount(); busyWorkerThreads = numWorkerThreads; @@ -207,12 +216,43 @@ public void initialize() { workerStates.addressOf(i).setIsolate(CurrentIsolate.getIsolate()); } + /* Start the worker threads and wait until they are in a well-defined state. */ workerThreads = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(OSThreadHandlePointer.class).multiply(numWorkerThreads)); VMError.guarantee(workerThreads.isNonNull()); for (int i = 0; i < numWorkerThreads; i++) { OSThreadHandle thread = PlatformThreads.singleton().startThreadUnmanaged(gcWorkerRunFunc.getFunctionPointer(), workerStates.addressOf(i), 0); workerThreads.write(i, thread); } + + waitUntilWorkerThreadsFinish(); + } + + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + if (!initialized) { + return; + } + + initialized = false; + + chunkQueue.teardown(); + + /* Signal the worker threads so that they can shut down. */ + shutdown = true; + parPhase.broadcast(); + for (int i = 0; i < numWorkerThreads; i++) { + OSThreadHandle thread = workerThreads.read(i); + PlatformThreads.singleton().joinThreadUnmanaged(thread); + } + assert busyWorkerThreads == 0; + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); + workerThreads = WordFactory.nullPointer(); + + PlatformThreads.singleton().deleteUnmanagedThreadLocal(workerStateTL); + workerStateTL = WordFactory.nullPointer(); + + numWorkerThreads = 0; } private static int getWorkerCount() { @@ -220,9 +260,8 @@ private static int getWorkerCount() { return setting > 0 ? setting : getDefaultWorkerCount(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getDefaultWorkerCount() { - // TEMP (chaeubl): this is incorrect if the container support is present. + /* This does not take the container support into account. */ int cpus = Jvm.JVM_ActiveProcessorCount(); return UninterruptibleUtils.Math.min(cpus, MAX_WORKER_THREADS); } @@ -242,95 +281,56 @@ private static void gcWorkerRun(GCWorkerThreadState state) { @Uninterruptible(reason = "Called from a GC worker thread.") private void work(GCWorkerThreadState state) { PlatformThreads.singleton().setUnmanagedThreadLocalValue(workerStateTL, state); - - while (true) { - Pointer ptr; - while (!inParallelPhase || error != null || (ptr = buffer.pop()).isNull() && !allocChunkNeedsScanning(state)) { - mutex.lockNoTransitionUnspecifiedOwner(); - try { - if (--busyWorkerThreads == 0) { - inParallelPhase = false; - seqPhase.signal(); - } - parPhase.blockNoTransitionUnspecifiedOwner(); - if (shutdown) { - return; - } - ++busyWorkerThreads; - } finally { - mutex.unlockNoTransitionUnspecifiedOwner(); - } - } - - try { - do { - scanChunk(ptr); - } while ((ptr = buffer.pop()).isNonNull()); - scanAllocChunk(); - } catch (Throwable e) { - error = e; - shutdown = true; - } + try { + work0(state); + } catch (Throwable e) { + VMError.shouldNotReachHere(e); } } - /** - * Start parallel phase and wait until all chunks have been processed. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void waitForIdle() { - GCWorkerThreadState state = getWorkerThreadState(); - assert state.getAllocChunk().isNonNull(); - push(state.getAllocChunk()); - - mutex.lockNoTransitionUnspecifiedOwner(); - try { - while (busyWorkerThreads > 0) { - /* Wait for worker threads to become ready. */ - seqPhase.blockNoTransitionUnspecifiedOwner(); + @Uninterruptible(reason = "Called from a GC worker thread.") + private void work0(GCWorkerThreadState state) { + while (!shutdown) { + Pointer ptr; + mutex.lockNoTransitionUnspecifiedOwner(); + try { + ptr = chunkQueue.pop(); + /* Block if there is no local/global work. */ + if (ptr.isNull() && !allocChunkNeedsScanning(state)) { + decrementBusyWorkers(); + do { + parPhase.blockNoTransitionUnspecifiedOwner(); + } while (!inParallelPhase); + incrementBusyWorkers(); + } + } finally { + mutex.unlockNoTransitionUnspecifiedOwner(); } - inParallelPhase = true; - /* Let worker threads run. */ - parPhase.broadcast(); - - while (inParallelPhase) { - /* Wait for them to become idle. */ - seqPhase.blockNoTransitionUnspecifiedOwner(); + if (ptr.isNonNull()) { + scanChunk(ptr); + } else { + scanAllocChunk(state); } - } finally { - mutex.unlockNoTransitionUnspecifiedOwner(); - } - - haltOnError(); - - assert buffer.isEmpty(); - /* Reset thread local allocation chunks. */ - state.setAllocChunk(WordFactory.nullPointer()); - for (int i = 0; i < numWorkerThreads; i++) { - workerStates.addressOf(i).setAllocChunk(WordFactory.nullPointer()); } } @Uninterruptible(reason = "Called from a GC worker thread.") private static void scanChunk(Pointer ptr) { - if (ptr.isNonNull()) { - if (ptr.and(UNALIGNED_BIT).notEqual(0)) { - UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); - } else { - Pointer start = ptr; - AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); - if (chunk.equal(ptr)) { - start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); - } - HeapChunk.walkObjectsFromInline(chunk, start, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); + if (ptr.and(UNALIGNED_BIT).notEqual(0)) { + UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); + } else { + Pointer start = ptr; + AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); + if (chunk.equal(ptr)) { + start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); } + HeapChunk.walkObjectsFromInline(chunk, start, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); } } @Uninterruptible(reason = "Called from a GC worker thread.") - private void scanAllocChunk() { - GCWorkerThreadState state = getWorkerThreadState(); + private static void scanAllocChunk(GCWorkerThreadState state) { if (allocChunkNeedsScanning(state)) { AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); UnsignedWord scanOffset = state.getAllocChunkScanOffset(); @@ -352,38 +352,65 @@ private static boolean allocChunkNeedsScanning(GCWorkerThreadState state) { return allocChunk.isNonNull() && allocChunk.getTopOffset().aboveThan(state.getAllocChunkScanOffset()); } + @Uninterruptible(reason = "Called from a GC worker thread.") + private void incrementBusyWorkers() { + assert mutex.isOwner(true); + ++busyWorkerThreads; + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + private void decrementBusyWorkers() { + assert mutex.isOwner(true); + if (--busyWorkerThreads == 0) { + inParallelPhase = false; + seqPhase.signal(); + } + } + /** - * Make sure errors are thrown on main GC thread. + * Start parallel phase and wait until all chunks have been processed. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void haltOnError() { - if (error instanceof OutOfMemoryError oome) { - throw OutOfMemoryUtil.reportOutOfMemoryError(oome); - } else if (error instanceof Error) { - throw VMError.shouldNotReachHere(error); - } - } + public void waitForIdle() { + assert getWorkerThreadState().getAllocChunk().isNonNull(); + pushAllocChunk(); - @Uninterruptible(reason = "Tear-down in progress.") - public void tearDown() { - if (initialized) { - initialized = false; + mutex.lockNoTransitionUnspecifiedOwner(); + try { + /* Let worker threads run. */ + inParallelPhase = true; + parPhase.broadcast(); - buffer.release(); + waitUntilWorkerThreadsFinish0(); + } finally { + mutex.unlockNoTransitionUnspecifiedOwner(); + } - /* Signal the worker threads so that they can shut down. */ - shutdown = true; - parPhase.broadcast(); - for (int i = 0; i < numWorkerThreads; i++) { - OSThreadHandle thread = workerThreads.read(i); - PlatformThreads.singleton().joinThreadUnmanaged(thread, WordFactory.nullPointer()); - } + assert chunkQueue.isEmpty(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); - workerThreads = WordFactory.nullPointer(); + /* Reset all thread local states. */ + for (int i = 0; i < numWorkerThreads + 1; i++) { + GCWorkerThreadState state = workerStates.addressOf(i); + state.setAllocChunk(WordFactory.nullPointer()); + assert state.getScannedChunk().isNull(); + } + } - PlatformThreads.singleton().deleteUnmanagedThreadLocal(workerStateTL); - workerStateTL = WordFactory.nullPointer(); + @Uninterruptible(reason = "Called from a GC worker thread.") + private void waitUntilWorkerThreadsFinish() { + mutex.lockNoTransitionUnspecifiedOwner(); + try { + waitUntilWorkerThreadsFinish0(); + } finally { + mutex.unlockNoTransitionUnspecifiedOwner(); + } + } + + @Uninterruptible(reason = "Called from a GC worker thread.") + private void waitUntilWorkerThreadsFinish0() { + /* Wait for them to become idle. */ + while (inParallelPhase) { + seqPhase.blockNoTransitionUnspecifiedOwner(); } } @@ -437,7 +464,6 @@ public boolean getAsBoolean() { @AutomaticallyRegisteredFeature() @SuppressWarnings("unused") class ParallelGCFeature implements InternalFeature { - @Override public boolean isInConfiguration(IsInConfigurationAccess access) { return ParallelGC.isEnabled(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 629a1ecbc81d..2cc8ae34abc9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.JDKUtils; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.VMError; public class OutOfMemoryUtil { @@ -39,10 +40,18 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } - @Uninterruptible(reason = "Not uninterruptible but it doesn't matter for the callers.", calleeMustBe = false) + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { - // TEMP (chaeubl): fix this + if (VMOperation.isGCInProgress()) { + /* An OutOfMemoryError during a GC is always a fatal error. */ + throw VMError.shouldNotReachHere(error); + } + throw reportOutOfMemoryError0(error); + } + + @Uninterruptible(reason = "Not uninterruptible but it doesn't matter for the callers.", calleeMustBe = false) + private static OutOfMemoryError reportOutOfMemoryError0(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { if (LibC.isSupported()) { Log.log().string("Terminating due to java.lang.OutOfMemoryError: ").string(JDKUtils.getRawMessage(error)).newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index f2cb98519e78..09620cb7e7a9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -556,6 +556,11 @@ public OSThreadHandle startThreadUnmanaged(CFunctionPointer threadRoutine, Point throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.startThreadUnmanaged directly."); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean joinThreadUnmanaged(OSThreadHandle threadHandle) { + return joinThreadUnmanaged(threadHandle, WordFactory.nullPointer()); + } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer threadExitStatus) { From 2e11172c5c0371dab0a2a98259e1f0774fc7d7d5 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 5 May 2023 10:19:00 +0200 Subject: [PATCH 107/125] Throw an error if the parallel GC is used on other platforms than Linux and macOS. --- .../com/oracle/svm/core/genscavenge/parallel/ParallelGC.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 24b8895593ed..d9f15a42b7b8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -66,6 +66,7 @@ import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandle; import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandlePointer; import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; +import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; /** @@ -471,6 +472,9 @@ public boolean isInConfiguration(IsInConfigurationAccess access) { @Override public void afterRegistration(AfterRegistrationAccess access) { + UserError.guarantee(Platform.includedIn(Platform.LINUX.class) || Platform.includedIn(Platform.DARWIN.class), + "The parallel garbage collector ('--gc=parallel') is currently only supported on Linux and macOS."); + ImageSingletons.add(ParallelGC.class, new ParallelGC()); } } From 24051ebc10b163de643b30d824eb78b793def61c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 5 May 2023 15:56:57 +0200 Subject: [PATCH 108/125] Style fix. --- .../src/com/oracle/svm/core/hub/LayoutEncoding.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 7ff84b76f4b2..9293cd63d4be 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -355,7 +355,7 @@ public static UnsignedWord getSizeFromHeader(Object obj, Word header, boolean ad DynamicHub hub = oh.dynamicHubFromObjectHeader(header); int encoding = hub.getLayoutEncoding(); boolean withOptionalIdHashField = addOptionalIdHashField || - (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && oh.hasOptionalIdentityHashField(header)); + (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && oh.hasOptionalIdentityHashField(header)); return getSizeFromEncoding(obj, hub, encoding, withOptionalIdHashField); } From 6ea15c53aa5c64ab774ef7d952b1522e8cf8aed0 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Fri, 5 May 2023 16:10:05 +0200 Subject: [PATCH 109/125] Verify that SpawnIsolates is enabled if the ParallelGC is used. --- .../oracle/svm/core/genscavenge/parallel/ParallelGC.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index d9f15a42b7b8..fb29cd397506 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -62,6 +62,7 @@ import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.option.SubstrateOptionKey; import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandle; import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandlePointer; @@ -474,7 +475,13 @@ public boolean isInConfiguration(IsInConfigurationAccess access) { public void afterRegistration(AfterRegistrationAccess access) { UserError.guarantee(Platform.includedIn(Platform.LINUX.class) || Platform.includedIn(Platform.DARWIN.class), "The parallel garbage collector ('--gc=parallel') is currently only supported on Linux and macOS."); + verifyOptionEnabled(SubstrateOptions.SpawnIsolates); ImageSingletons.add(ParallelGC.class, new ParallelGC()); } + + private static void verifyOptionEnabled(SubstrateOptionKey option) { + String optionMustBeEnabledFmt = "When using the parallel garbage collector ('--gc=parallel'), please note that option '%s' must be enabled."; + UserError.guarantee(option.getValue(), optionMustBeEnabledFmt, option.getName()); + } } From 101726a1b565ce62d2cc981248a414d60ff447a9 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 8 May 2023 12:49:14 +0200 Subject: [PATCH 110/125] Fix parallel GC teardown. --- .../com/oracle/svm/core/genscavenge/parallel/ParallelGC.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index fb29cd397506..c32f867beae4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -142,7 +142,6 @@ public AlignedHeapChunk.AlignedHeader getAllocationChunk() { @Uninterruptible(reason = "Called from a GC worker thread.") public void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { - assert chunk.isNonNull(); GCWorkerThreadState state = getWorkerThreadState(); state.setAllocChunk(chunk); state.setAllocChunkScanOffset(AlignedHeapChunk.getObjectsStartOffset()); @@ -240,13 +239,15 @@ public void tearDown() { chunkQueue.teardown(); /* Signal the worker threads so that they can shut down. */ + inParallelPhase = true; shutdown = true; parPhase.broadcast(); for (int i = 0; i < numWorkerThreads; i++) { OSThreadHandle thread = workerThreads.read(i); PlatformThreads.singleton().joinThreadUnmanaged(thread); } - assert busyWorkerThreads == 0; + inParallelPhase = false; + busyWorkerThreads = 0; ImageSingletons.lookup(UnmanagedMemorySupport.class).free(workerThreads); workerThreads = WordFactory.nullPointer(); From 393bb27ccc0f3048276b4ece977b0b38098d2f04 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 8 May 2023 12:59:59 +0200 Subject: [PATCH 111/125] Fix identity hashcode computation. Cleanups. --- .../svm/core/genscavenge/HeapChunk.java | 4 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 2 +- .../svm/core/genscavenge/OldGeneration.java | 6 ++ .../oracle/svm/core/genscavenge/Space.java | 83 ++++++++----------- .../IdentityHashCodeSupport.java | 6 +- 5 files changed, 49 insertions(+), 52 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 2f588f856895..41ea872863c0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -263,8 +263,8 @@ public static > void setNext(Header that, T newNext) { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getIdentityHashSalt(Header that) { - return that.getIdentityHashSalt(IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION); + public static long getIdentityHashSalt(Header that) { + return that.getIdentityHashSalt(IdentityHashCodeSupport.IDENTITY_HASHCODE_SALT_LOCATION).rawValue(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index bc04d19fe68a..06ae4743aacb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -710,7 +710,7 @@ public long getIdentityHashSalt(Object obj) { assert !isInImageHeap(obj) : "Image heap objects have identity hash code fields"; } HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); - return HeapChunk.getIdentityHashSalt(chunk).rawValue(); + return HeapChunk.getIdentityHashSalt(chunk); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index ca5f7e0a7abe..01339d9f7b4c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -35,6 +35,7 @@ import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; @@ -101,11 +102,16 @@ void releaseSpaces(ChunkReleaser chunkReleaser) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void prepareForPromotion() { + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + return; + } toGreyObjectsWalker.setScanStart(getToSpace()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean scanGreyObjects() { + assert !ParallelGC.isEnabled() || !GCImpl.getGCImpl().isCompleteCollection(); + if (!toGreyObjectsWalker.haveGreyObjects()) { return false; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 3f47f3ae7558..a7e259dec09b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -97,13 +97,16 @@ public String getName() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isEmpty() { - return (getFirstAlignedHeapChunk().isNull() && getFirstUnalignedHeapChunk().isNull()); + return firstAlignedHeapChunk.isNull() && firstUnalignedHeapChunk.isNull(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void tearDown() { - HeapChunkProvider.freeAlignedChunkList(getFirstAlignedHeapChunk()); - HeapChunkProvider.freeUnalignedChunkList(getFirstUnalignedHeapChunk()); + HeapChunkProvider.freeAlignedChunkList(firstAlignedHeapChunk); + firstAlignedHeapChunk = WordFactory.nullPointer(); + + HeapChunkProvider.freeUnalignedChunkList(firstUnalignedHeapChunk); + firstUnalignedHeapChunk = WordFactory.nullPointer(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -142,14 +145,14 @@ boolean isFromSpace() { } public boolean walkObjects(ObjectVisitor visitor) { - AlignedHeapChunk.AlignedHeader aChunk = getFirstAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader aChunk = firstAlignedHeapChunk; while (aChunk.isNonNull()) { if (!AlignedHeapChunk.walkObjects(aChunk, visitor)) { return false; } aChunk = HeapChunk.getNext(aChunk); } - UnalignedHeapChunk.UnalignedHeader uChunk = getFirstUnalignedHeapChunk(); + UnalignedHeapChunk.UnalignedHeader uChunk = firstUnalignedHeapChunk; while (uChunk.isNonNull()) { if (!UnalignedHeapChunk.walkObjects(uChunk, visitor)) { return false; @@ -164,8 +167,8 @@ public Log report(Log log, boolean traceHeapChunks) { log.string(getName()).string(":").indent(true); accounting.report(log); if (traceHeapChunks) { - HeapChunkLogging.logChunks(log, getFirstAlignedHeapChunk()); - HeapChunkLogging.logChunks(log, getFirstUnalignedHeapChunk()); + HeapChunkLogging.logChunks(log, firstAlignedHeapChunk); + HeapChunkLogging.logChunks(log, firstUnalignedHeapChunk); } log.redent(false); return log; @@ -186,8 +189,10 @@ private Pointer allocateMemory(UnsignedWord objectSize) { @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemorySerial(UnsignedWord objectSize) { + assert !ParallelGC.isEnabled() || !GCImpl.getGCImpl().isCompleteCollection(); + /* Fast-path: try allocating in the last chunk. */ - AlignedHeapChunk.AlignedHeader oldChunk = getLastAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader oldChunk = lastAlignedHeapChunk; if (oldChunk.isNonNull()) { Pointer result = AlignedHeapChunk.allocateMemory(oldChunk, objectSize); if (result.isNonNull()) { @@ -273,15 +278,15 @@ void appendAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk, Space origina } HeapChunk.setSpace(aChunk, this); - AlignedHeapChunk.AlignedHeader oldLast = getLastAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader oldLast = lastAlignedHeapChunk; HeapChunk.setPrevious(aChunk, oldLast); HeapChunk.setNext(aChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { HeapChunk.setNext(oldLast, aChunk); } - setLastAlignedHeapChunk(aChunk); - if (getFirstAlignedHeapChunk().isNull()) { - setFirstAlignedHeapChunk(aChunk); + lastAlignedHeapChunk = aChunk; + if (firstAlignedHeapChunk.isNull()) { + firstAlignedHeapChunk = aChunk; } accounting.noteAlignedHeapChunk(); } @@ -297,15 +302,15 @@ void appendUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk, Space o } HeapChunk.setSpace(uChunk, this); - UnalignedHeapChunk.UnalignedHeader oldLast = getLastUnalignedHeapChunk(); + UnalignedHeapChunk.UnalignedHeader oldLast = lastUnalignedHeapChunk; HeapChunk.setPrevious(uChunk, oldLast); HeapChunk.setNext(uChunk, WordFactory.nullPointer()); if (oldLast.isNonNull()) { HeapChunk.setNext(oldLast, uChunk); } - setLastUnalignedHeapChunk(uChunk); - if (getFirstUnalignedHeapChunk().isNull()) { - setFirstUnalignedHeapChunk(uChunk); + lastUnalignedHeapChunk = uChunk; + if (firstUnalignedHeapChunk.isNull()) { + firstUnalignedHeapChunk = uChunk; } accounting.noteUnalignedHeapChunk(uChunk); } @@ -319,12 +324,12 @@ private void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { if (chunkPrev.isNonNull()) { HeapChunk.setNext(chunkPrev, chunkNext); } else { - setFirstAlignedHeapChunk(chunkNext); + firstAlignedHeapChunk = chunkNext; } if (chunkNext.isNonNull()) { HeapChunk.setPrevious(chunkNext, chunkPrev); } else { - setLastAlignedHeapChunk(chunkPrev); + lastAlignedHeapChunk = chunkPrev; } HeapChunk.setNext(aChunk, WordFactory.nullPointer()); HeapChunk.setPrevious(aChunk, WordFactory.nullPointer()); @@ -341,12 +346,12 @@ private void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk if (chunkPrev.isNonNull()) { HeapChunk.setNext(chunkPrev, chunkNext); } else { - setFirstUnalignedHeapChunk(chunkNext); + firstUnalignedHeapChunk = chunkNext; } if (chunkNext.isNonNull()) { HeapChunk.setPrevious(chunkNext, chunkPrev); } else { - setLastUnalignedHeapChunk(chunkPrev); + lastUnalignedHeapChunk = chunkPrev; } /* Reset the fields that the result chunk keeps for Space. */ HeapChunk.setNext(uChunk, WordFactory.nullPointer()); @@ -367,41 +372,21 @@ public AlignedHeapChunk.AlignedHeader getFirstAlignedHeapChunk() { return firstAlignedHeapChunk; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void setFirstAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk) { - firstAlignedHeapChunk = chunk; - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeapChunk.AlignedHeader getLastAlignedHeapChunk() { return lastAlignedHeapChunk; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void setLastAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk) { - lastAlignedHeapChunk = chunk; - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnalignedHeapChunk.UnalignedHeader getFirstUnalignedHeapChunk() { return firstUnalignedHeapChunk; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void setFirstUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { - this.firstUnalignedHeapChunk = chunk; - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) UnalignedHeapChunk.UnalignedHeader getLastUnalignedHeapChunk() { return lastUnalignedHeapChunk; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void setLastUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { - lastUnalignedHeapChunk = chunk; - } - /** Promote an aligned Object to this Space. */ @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -442,13 +427,14 @@ private Object copyAlignedObjectSerial(Object originalObj) { * references. That's okay, because all references in the copy are visited and overwritten * later on anyways (the card table is also updated at that point if necessary). */ - Pointer originalMemory = Word.objectToUntrackedPointer(originalObj); + Word originalMemory = Word.objectToUntrackedPointer(originalObj); UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, originalSize); Object copy = copyMemory.toObject(); if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { // Must do first: ensures correct object size below and in other places - int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalObj); + AlignedHeapChunk.AlignedHeader originalChunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(originalMemory); + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalMemory, HeapChunk.getIdentityHashSalt(originalChunk)); int offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); @@ -474,7 +460,7 @@ private Object copyAlignedObjectParallel(Object original) { * Always read 8 bytes at the hub offset so that we can install the forwarding header with * cmpxchng. */ - Pointer originalMemory = Word.objectToUntrackedPointer(original); + Word originalMemory = Word.objectToUntrackedPointer(original); int hubOffset = ObjectHeader.getHubOffset(); long eightHeaderBytes = originalMemory.readLong(hubOffset); Word originalHeader = ObjectHeaderImpl.hasShift() ? WordFactory.unsigned(eightHeaderBytes & 0xFFFFFFFFL) : WordFactory.unsigned(eightHeaderBytes); @@ -521,7 +507,8 @@ private Object copyAlignedObjectParallel(Object original) { UnmanagedMemoryUtil.copyLongsForward(originalMemory.add(offset), copyMemory.add(offset), originalSize.subtract(offset)); if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { - int value = IdentityHashCodeSupport.computeHashCodeFromAddress(original); + AlignedHeapChunk.AlignedHeader originalChunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(originalMemory); + int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalMemory, HeapChunk.getIdentityHashSalt(originalChunk)); offset = LayoutEncoding.getOptionalIdentityHashOffset(copy); ObjectAccess.writeInt(copy, offset, value, IdentityHashCodeSupport.IDENTITY_HASHCODE_LOCATION); ObjectHeaderImpl.getObjectHeaderImpl().setIdentityHashInField(copy); @@ -675,12 +662,12 @@ void absorb(Space src) { boolean walkHeapChunks(MemoryWalker.Visitor visitor) { boolean continueVisiting = true; - AlignedHeapChunk.AlignedHeader aChunk = getFirstAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader aChunk = firstAlignedHeapChunk; while (continueVisiting && aChunk.isNonNull()) { continueVisiting = visitor.visitHeapChunk(aChunk, AlignedHeapChunk.getMemoryWalkerAccess()); aChunk = HeapChunk.getNext(aChunk); } - UnalignedHeapChunk.UnalignedHeader uChunk = getFirstUnalignedHeapChunk(); + UnalignedHeapChunk.UnalignedHeader uChunk = firstUnalignedHeapChunk; while (continueVisiting && uChunk.isNonNull()) { continueVisiting = visitor.visitHeapChunk(uChunk, UnalignedHeapChunk.getMemoryWalkerAccess()); uChunk = HeapChunk.getNext(uChunk); @@ -710,7 +697,7 @@ UnsignedWord computeObjectBytes() { private UnsignedWord computeAlignedObjectBytes() { UnsignedWord result = WordFactory.zero(); - AlignedHeapChunk.AlignedHeader aChunk = getFirstAlignedHeapChunk(); + AlignedHeapChunk.AlignedHeader aChunk = firstAlignedHeapChunk; while (aChunk.isNonNull()) { UnsignedWord allocatedBytes = HeapChunk.getTopOffset(aChunk).subtract(AlignedHeapChunk.getObjectsStartOffset()); result = result.add(allocatedBytes); @@ -721,7 +708,7 @@ private UnsignedWord computeAlignedObjectBytes() { private UnsignedWord computeUnalignedObjectBytes() { UnsignedWord result = WordFactory.zero(); - UnalignedHeapChunk.UnalignedHeader uChunk = getFirstUnalignedHeapChunk(); + UnalignedHeapChunk.UnalignedHeader uChunk = firstUnalignedHeapChunk; while (uChunk.isNonNull()) { UnsignedWord allocatedBytes = HeapChunk.getTopOffset(uChunk).subtract(UnalignedHeapChunk.getObjectStartOffset()); result = result.add(allocatedBytes); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java index 32ffd9585564..edebb2f74d13 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/identityhashcode/IdentityHashCodeSupport.java @@ -85,8 +85,12 @@ public static int generateIdentityHashCode(Object obj) { @Uninterruptible(reason = "Prevent a GC interfering with the object's identity hash state.") public static int computeHashCodeFromAddress(Object obj) { - Word address = Word.objectToUntrackedPointer(obj); long salt = Heap.getHeap().getIdentityHashSalt(obj); + return computeHashCodeFromAddress(Word.objectToUntrackedPointer(obj), salt); + } + + @Uninterruptible(reason = "Prevent a GC interfering with the object's identity hash state.") + public static int computeHashCodeFromAddress(Word address, long salt) { SignedWord salted = WordFactory.signed(salt).xor(address); int hash = mix32(salted.rawValue()) >>> 1; // shift: ensure positive, same as on HotSpot return (hash == 0) ? 1 : hash; // ensure nonzero From 657f774f2716882da9ab36a1c7ee7e911af859d3 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 8 May 2023 14:00:09 +0200 Subject: [PATCH 112/125] Fix validation of UseParallelGC option. --- .../src/com/oracle/svm/core/SubstrateOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index de4e97b002de..e2e907292761 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -329,8 +329,8 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @Option(help = "Number of GC worker threads. Parallel and G1 GC only.", type = OptionType.User)// public static final RuntimeOptionKey ParallelGCThreads = new RuntimeOptionKey<>(0, Immutable); - private static void requireMultiThreading(OptionKey optionKey) { - if (!MultiThreaded.getValue()) { + private static void requireMultiThreading(HostedOptionKey optionKey) { + if (optionKey.getValue() && !MultiThreaded.getValue()) { throw new InterruptImageBuilding(String.format("The option %s requires the option %s to be set.", SubstrateOptionsParser.commandArgument(optionKey, "+"), SubstrateOptionsParser.commandArgument(MultiThreaded, "+"))); From 91e3177aeea2c75c4cf62dd38efd83d5825be189 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 8 May 2023 16:30:58 +0200 Subject: [PATCH 113/125] Avoid that unattached threads are interpreted as the error handling thread. --- .../src/com/oracle/svm/core/SubstrateDiagnostics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index 0d9e14f7d4ec..fc135cb12d53 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -124,7 +124,7 @@ public static boolean isFatalErrorHandlingInProgress() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isFatalErrorHandlingThread() { - return fatalErrorState().diagnosticThread.get() == CurrentIsolate.getCurrentThread(); + return CurrentIsolate.getCurrentThread().isNonNull() && fatalErrorState().diagnosticThread.get() == CurrentIsolate.getCurrentThread(); } public static int maxInvocations() { From 35013d35e5dbfd51018cdfd06d55cb83bc40e6b1 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 8 May 2023 16:31:43 +0200 Subject: [PATCH 114/125] Skip the parallel GC phase if there isn't any work. --- .../core/genscavenge/parallel/ParallelGC.java | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index c32f867beae4..07bbc9fb01d7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -143,13 +143,15 @@ public AlignedHeapChunk.AlignedHeader getAllocationChunk() { @Uninterruptible(reason = "Called from a GC worker thread.") public void setAllocationChunk(AlignedHeapChunk.AlignedHeader chunk) { GCWorkerThreadState state = getWorkerThreadState(); + assert state.getAllocChunk().isNull() && state.getAllocChunkScanOffset().equal(0); + state.setAllocChunk(chunk); state.setAllocChunkScanOffset(AlignedHeapChunk.getObjectsStartOffset()); } @Uninterruptible(reason = "Called from a GC worker thread.") public void push(AlignedHeapChunk.AlignedHeader aChunk) { - push(HeapChunk.asPointer(aChunk)); + push(AlignedHeapChunk.getObjectsStart(aChunk)); } @Uninterruptible(reason = "Called from a GC worker thread.") @@ -170,22 +172,24 @@ private void push(Pointer ptr) { @Uninterruptible(reason = "Called from a GC worker thread.") public void pushAllocChunk() { assert GCImpl.getGCImpl().isCompleteCollection(); + + /* + * Scanning (and therefore enqueueing) is only necessary if there are any not yet scanned in + * the chunk. + */ GCWorkerThreadState state = getWorkerThreadState(); AlignedHeapChunk.AlignedHeader chunk = state.getAllocChunk(); - if (chunk.isNull() || chunk.equal(state.getScannedChunk())) { - /* - * Scanning (and therefore enqueueing) is not necessary if we are already in the middle - * of scanning the chunk, or if we don't have a chunk. - */ - return; + if (chunk.isNonNull() && !chunk.equal(state.getScannedChunk())) { + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); + assert scanOffset.aboveThan(0); + if (chunk.getTopOffset().aboveThan(scanOffset)) { + Pointer ptrIntoChunk = HeapChunk.asPointer(chunk).add(scanOffset); + push(ptrIntoChunk); + } } - UnsignedWord scanOffset = state.getAllocChunkScanOffset(); - assert scanOffset.aboveThan(0); - if (chunk.getTopOffset().aboveThan(scanOffset)) { - Pointer ptrIntoChunk = HeapChunk.asPointer(chunk).add(scanOffset); - push(ptrIntoChunk); - } + state.setAllocChunk(WordFactory.nullPointer()); + state.setAllocChunkScanOffset(WordFactory.zero()); } @Uninterruptible(reason = "Called from a GC worker thread.") @@ -323,12 +327,8 @@ private static void scanChunk(Pointer ptr) { if (ptr.and(UNALIGNED_BIT).notEqual(0)) { UnalignedHeapChunk.walkObjectsInline((UnalignedHeapChunk.UnalignedHeader) ptr.and(~UNALIGNED_BIT), GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); } else { - Pointer start = ptr; AlignedHeapChunk.AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptr); - if (chunk.equal(ptr)) { - start = ptr.add(AlignedHeapChunk.getObjectsStartOffset()); - } - HeapChunk.walkObjectsFromInline(chunk, start, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); + HeapChunk.walkObjectsFromInline(chunk, ptr, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); } } @@ -375,18 +375,19 @@ private void decrementBusyWorkers() { */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void waitForIdle() { - assert getWorkerThreadState().getAllocChunk().isNonNull(); pushAllocChunk(); - mutex.lockNoTransitionUnspecifiedOwner(); - try { - /* Let worker threads run. */ - inParallelPhase = true; - parPhase.broadcast(); + if (!chunkQueue.isEmpty()) { + mutex.lockNoTransitionUnspecifiedOwner(); + try { + /* Let worker threads run. */ + inParallelPhase = true; + parPhase.broadcast(); - waitUntilWorkerThreadsFinish0(); - } finally { - mutex.unlockNoTransitionUnspecifiedOwner(); + waitUntilWorkerThreadsFinish0(); + } finally { + mutex.unlockNoTransitionUnspecifiedOwner(); + } } assert chunkQueue.isEmpty(); @@ -395,6 +396,7 @@ public void waitForIdle() { for (int i = 0; i < numWorkerThreads + 1; i++) { GCWorkerThreadState state = workerStates.addressOf(i); state.setAllocChunk(WordFactory.nullPointer()); + state.setAllocChunkScanOffset(WordFactory.zero()); assert state.getScannedChunk().isNull(); } } From 31bb51d7cff3ff55f266edc02892c4ceb6b12343 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 8 May 2023 18:11:38 +0200 Subject: [PATCH 115/125] Cleanups. --- .../core/genscavenge/parallel/ParallelGC.java | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 07bbc9fb01d7..0613c8da0685 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -181,7 +181,7 @@ public void pushAllocChunk() { AlignedHeapChunk.AlignedHeader chunk = state.getAllocChunk(); if (chunk.isNonNull() && !chunk.equal(state.getScannedChunk())) { UnsignedWord scanOffset = state.getAllocChunkScanOffset(); - assert scanOffset.aboveThan(0); + assert scanOffset.aboveOrEqual(AlignedHeapChunk.getObjectsStartOffset()); if (chunk.getTopOffset().aboveThan(scanOffset)) { Pointer ptrIntoChunk = HeapChunk.asPointer(chunk).add(scanOffset); push(ptrIntoChunk); @@ -246,6 +246,7 @@ public void tearDown() { inParallelPhase = true; shutdown = true; parPhase.broadcast(); + for (int i = 0; i < numWorkerThreads; i++) { OSThreadHandle thread = workerThreads.read(i); PlatformThreads.singleton().joinThreadUnmanaged(thread); @@ -334,18 +335,20 @@ private static void scanChunk(Pointer ptr) { @Uninterruptible(reason = "Called from a GC worker thread.") private static void scanAllocChunk(GCWorkerThreadState state) { - if (allocChunkNeedsScanning(state)) { - AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); - UnsignedWord scanOffset = state.getAllocChunkScanOffset(); - assert scanOffset.aboveThan(0); - Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); - state.setScannedChunk(allocChunk); - HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); - state.setScannedChunk(WordFactory.nullPointer()); - if (state.getAllocChunk().equal(allocChunk)) { - /* Remember top offset so that we don't scan the same objects again. */ - state.setAllocChunkScanOffset(allocChunk.getTopOffset()); - } + if (!allocChunkNeedsScanning(state)) { + return; + } + + AlignedHeapChunk.AlignedHeader allocChunk = state.getAllocChunk(); + UnsignedWord scanOffset = state.getAllocChunkScanOffset(); + assert scanOffset.aboveOrEqual(AlignedHeapChunk.getObjectsStartOffset()); + Pointer scanPointer = HeapChunk.asPointer(allocChunk).add(scanOffset); + state.setScannedChunk(allocChunk); + HeapChunk.walkObjectsFromInline(allocChunk, scanPointer, GCImpl.getGCImpl().getGreyToBlackObjectVisitor()); + state.setScannedChunk(WordFactory.nullPointer()); + if (allocChunk.equal(state.getAllocChunk())) { + /* Remember top offset so that we don't scan the same objects again. */ + state.setAllocChunkScanOffset(allocChunk.getTopOffset()); } } @@ -391,6 +394,8 @@ public void waitForIdle() { } assert chunkQueue.isEmpty(); + assert !inParallelPhase; + assert busyWorkerThreads == 0; /* Reset all thread local states. */ for (int i = 0; i < numWorkerThreads + 1; i++) { @@ -413,7 +418,6 @@ private void waitUntilWorkerThreadsFinish() { @Uninterruptible(reason = "Called from a GC worker thread.") private void waitUntilWorkerThreadsFinish0() { - /* Wait for them to become idle. */ while (inParallelPhase) { seqPhase.blockNoTransitionUnspecifiedOwner(); } From 0c5564217b531f860ab1413cd120c0d594033d4f Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 11 May 2023 17:21:21 +0300 Subject: [PATCH 116/125] Fixed race in reference processing --- .../svm/core/genscavenge/HeapChunk.java | 1 + .../ReferenceObjectProcessing.java | 28 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 41ea872863c0..abea3fbbb30a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -350,6 +350,7 @@ public static HeapChunk.Header getEnclosingHeapChunk(Object obj) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static HeapChunk.Header getEnclosingHeapChunk(Pointer ptrToObj, UnsignedWord header) { if (ObjectHeaderImpl.isAlignedHeader(header)) { return AlignedHeapChunk.getEnclosingChunkFromObjectPointer(ptrToObj); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 96b1f294677f..4b6960e1aa09 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -114,17 +114,18 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { // Referents in the image heap cannot be moved or reclaimed, no need to look closer. return; } - if (maybeUpdateForwardedReference(dr, referentAddr)) { + + UnsignedWord referentHeader = ObjectHeader.readHeaderFromPointer(referentAddr); + if (maybeUpdateForwardedReference(dr, referentAddr, referentHeader)) { // Some other object had a strong reference to the referent, so the referent was already // promoted. The call above updated the reference object so that it now points to the // promoted object. return; } - Object refObject = referentAddr.toObject(); - if (willSurviveThisCollection(refObject)) { + if (willSurviveThisCollection(referentAddr, referentHeader)) { // Referent is in a to-space. So, this is either an object that got promoted without // being moved or an object in the old gen. - RememberedSet.get().dirtyCardIfNecessary(dr, refObject); + RememberedSet.get().dirtyCardIfNecessary(dr, referentAddr.toObject()); return; } if (!softReferencesAreWeak && dr instanceof SoftReference) { @@ -205,12 +206,13 @@ private static boolean processRememberedRef(Reference dr) { Pointer refPointer = ReferenceInternals.getReferentPointer(dr); assert refPointer.isNonNull() : "Referent is null: should not have been discovered"; assert !HeapImpl.getHeapImpl().isInImageHeap(refPointer) : "Image heap referent: should not have been discovered"; - if (maybeUpdateForwardedReference(dr, refPointer)) { + + UnsignedWord refHeader = ObjectHeader.readHeaderFromPointer(refPointer); + if (maybeUpdateForwardedReference(dr, refPointer, refHeader)) { return true; } - Object refObject = refPointer.toObject(); - if (willSurviveThisCollection(refObject)) { - RememberedSet.get().dirtyCardIfNecessary(dr, refObject); + if (willSurviveThisCollection(refPointer, refHeader)) { + RememberedSet.get().dirtyCardIfNecessary(dr, refPointer.toObject()); return true; } /* @@ -225,11 +227,9 @@ private static boolean processRememberedRef(Reference dr) { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr) { - ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); - UnsignedWord header = ObjectHeader.readHeaderFromPointer(referentAddr); + private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr, UnsignedWord header) { if (ObjectHeaderImpl.isForwardedHeader(header)) { - Object forwardedObj = ohi.getForwardedObject(referentAddr); + Object forwardedObj = ObjectHeaderImpl.getObjectHeaderImpl().getForwardedObject(referentAddr); ReferenceInternals.setReferent(dr, forwardedObj); return true; } @@ -237,8 +237,8 @@ private static boolean maybeUpdateForwardedReference(Reference dr, Pointer re } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean willSurviveThisCollection(Object obj) { - HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); + private static boolean willSurviveThisCollection(Pointer ptr, UnsignedWord header) { + HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(ptr, header); Space space = HeapChunk.getSpace(chunk); return space != null && !space.isFromSpace(); } From 383484af20a84239d352d458b02435e9f10ad770 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 11 May 2023 17:21:57 +0300 Subject: [PATCH 117/125] Fixed racy assertion in GreyToBlackObjRefVisitor --- .../svm/core/genscavenge/GreyToBlackObjRefVisitor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 535e14ed0e17..52ff8b135682 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -31,6 +31,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.genscavenge.parallel.ParallelGC; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.ObjectReferenceVisitor; @@ -98,7 +99,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole counters.noteForwardedReferent(); // Update the reference to point to the forwarded Object. Object obj = ohi.getForwardedObject(p, header); - assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); + assert ParallelGC.singleton().isInParallelPhase() || innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); Object offsetObj = (innerOffset == 0) ? obj : Word.objectToUntrackedPointer(obj).add(innerOffset).toObject(); ReferenceAccess.singleton().writeObjectAt(objRef, offsetObj, compressed); RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); @@ -111,7 +112,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); - assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(copy).rawValue(); + assert ParallelGC.singleton().isInParallelPhase() || innerOffset < LayoutEncoding.getSizeFromObjectInGC(copy).rawValue(); Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); } else { From 1e02a09ac269e405fc4e941ba3d64bc219448398 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 15 May 2023 17:17:54 +0300 Subject: [PATCH 118/125] Fixed assertion that broke SerialGC --- .../svm/core/genscavenge/GreyToBlackObjRefVisitor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 52ff8b135682..fae6561ab1c2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -99,7 +99,8 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole counters.noteForwardedReferent(); // Update the reference to point to the forwarded Object. Object obj = ohi.getForwardedObject(p, header); - assert ParallelGC.singleton().isInParallelPhase() || innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); + assert ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase() || + innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); Object offsetObj = (innerOffset == 0) ? obj : Word.objectToUntrackedPointer(obj).add(innerOffset).toObject(); ReferenceAccess.singleton().writeObjectAt(objRef, offsetObj, compressed); RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); @@ -112,7 +113,8 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); - assert ParallelGC.singleton().isInParallelPhase() || innerOffset < LayoutEncoding.getSizeFromObjectInGC(copy).rawValue(); + assert ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase() || + innerOffset < LayoutEncoding.getSizeFromObjectInGC(copy).rawValue(); Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); } else { From c38406cc9903ee1f1205730a9575417bb6bcd955 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 16 May 2023 16:48:25 +0200 Subject: [PATCH 119/125] Added some documentation. --- .../svm/core/genscavenge/ReferenceObjectProcessing.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 4b6960e1aa09..5ed48967c559 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -115,6 +115,10 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { return; } + /* + * The parallel GC may modify the object header at any time, so we only read the object + * header once. + */ UnsignedWord referentHeader = ObjectHeader.readHeaderFromPointer(referentAddr); if (maybeUpdateForwardedReference(dr, referentAddr, referentHeader)) { // Some other object had a strong reference to the referent, so the referent was already From 59c56b07484fc319c4a8fd51173e17856a65d684 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 19 Jun 2023 11:24:11 +0300 Subject: [PATCH 120/125] Fixed some races --- .../oracle/svm/core/genscavenge/ObjectHeaderImpl.java | 10 +++++----- .../core/genscavenge/ReferenceObjectProcessing.java | 2 +- .../src/com/oracle/svm/core/genscavenge/Space.java | 10 ++++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 4f3cfeb1a2e6..60b821f02a93 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -343,6 +343,11 @@ public static void setRememberedSetBit(Object o) { writeHeaderToObject(o, newHeader); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static long setRememberedSetBit(long headerBytes) { + return headerBytes | REMEMBERED_SET_BIT.rawValue(); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean hasRememberedSet(UnsignedWord header) { return header.and(REMEMBERED_SET_BIT).notEqual(0); @@ -359,11 +364,6 @@ public static boolean isForwardedHeader(UnsignedWord header) { return header.and(FORWARDED_BIT).notEqual(0); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - Object getForwardedObject(Pointer ptr) { - return getForwardedObject(ptr, readHeaderFromPointer(ptr)); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 5ed48967c559..269c2d3abe1b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -233,7 +233,7 @@ private static boolean processRememberedRef(Reference dr) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr, UnsignedWord header) { if (ObjectHeaderImpl.isForwardedHeader(header)) { - Object forwardedObj = ObjectHeaderImpl.getObjectHeaderImpl().getForwardedObject(referentAddr); + Object forwardedObj = ObjectHeaderImpl.getObjectHeaderImpl().getForwardedObject(referentAddr, header); ReferenceInternals.setReferent(dr, forwardedObj); return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index a7e259dec09b..5b0f52bc7a54 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -489,6 +489,13 @@ private Object copyAlignedObjectParallel(Object original) { return null; } + /* + * It's important that we set the RS bit before everything else because + * YoungGeneration.contains() checks it. + */ + long copyHeaderBytes = isOldSpace() ? ObjectHeaderImpl.setRememberedSetBit(eightHeaderBytes) : eightHeaderBytes; + copyMemory.writeLong(hubOffset, copyHeaderBytes); + /* Install forwarding pointer into the original header. */ Object copy = copyMemory.toObject(); Object forward = ohi.installForwardingPointerParallel(original, eightHeaderBytes, copy); @@ -498,8 +505,7 @@ private Object copyAlignedObjectParallel(Object original) { return forward; } - /* We have won the race. Install the object header and copy the rest of the object. */ - copyMemory.writeLong(hubOffset, eightHeaderBytes); + /* We have won the race. Copy the rest of the object. */ if (hubOffset > 0) { UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, WordFactory.unsigned(hubOffset)); } From e43b3e982db2ac759844a9181d42fdc448d1d870 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 29 Jun 2023 14:01:20 +0200 Subject: [PATCH 121/125] Fix an issue where the parallel GC could destroy the array length. --- .../oracle/svm/core/genscavenge/Space.java | 16 ++++++++---- .../core/genscavenge/parallel/ParallelGC.java | 4 +-- .../oracle/svm/core/hub/LayoutEncoding.java | 26 +++++++++++++++++-- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 5b0f52bc7a54..611dd53925f4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -454,16 +454,21 @@ private Object copyAlignedObjectSerial(Object originalObj) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObjectParallel(Object original) { assert VMOperation.isGCInProgress(); - assert ObjectHeaderImpl.isAlignedObject(original); /* - * Always read 8 bytes at the hub offset so that we can install the forwarding header with - * cmpxchng. + * The GC worker thread doesn't own the object yet, so the 8 bytes starting at the hub + * offset can be changed at any time (if another GC worker thread forwards the object). Note + * that those bytes may also include data such as the array length. + * + * So, we read the 8 bytes at the hub offset once and then extract all necessary data from + * those bytes. This is necessary to avoid races. */ Word originalMemory = Word.objectToUntrackedPointer(original); int hubOffset = ObjectHeader.getHubOffset(); long eightHeaderBytes = originalMemory.readLong(hubOffset); Word originalHeader = ObjectHeaderImpl.hasShift() ? WordFactory.unsigned(eightHeaderBytes & 0xFFFFFFFFL) : WordFactory.unsigned(eightHeaderBytes); + assert ObjectHeaderImpl.isAlignedHeader(originalHeader); + ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); if (ObjectHeaderImpl.isForwardedHeader(originalHeader)) { return ohi.getForwardedObject(originalMemory, originalHeader); @@ -473,13 +478,13 @@ private Object copyAlignedObjectParallel(Object original) { * We need the forwarding pointer to point somewhere, so we speculatively allocate memory * here. If another thread copies the object first, we retract the allocation later. */ - UnsignedWord originalSize = LayoutEncoding.getSizeFromHeader(original, originalHeader, false); + UnsignedWord originalSize = LayoutEncoding.getSizeFromHeader(original, originalHeader, eightHeaderBytes, false); UnsignedWord copySize = originalSize; boolean addIdentityHashField = false; if (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField()) { if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(originalHeader))) { addIdentityHashField = true; - copySize = LayoutEncoding.getSizeFromHeader(original, originalHeader, true); + copySize = LayoutEncoding.getSizeFromHeader(original, originalHeader, eightHeaderBytes, true); } } @@ -568,6 +573,7 @@ private void promoteAlignedHeapChunkParallel(AlignedHeapChunk.AlignedHeader chun @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void promoteAlignedHeapChunk0(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert originalSpace.isFromSpace(); + assert !this.isFromSpace(); appendAlignedHeapChunk(chunk, originalSpace); if (this.isOldSpace()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 0613c8da0685..09ca8a1d2833 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -174,8 +174,8 @@ public void pushAllocChunk() { assert GCImpl.getGCImpl().isCompleteCollection(); /* - * Scanning (and therefore enqueueing) is only necessary if there are any not yet scanned in - * the chunk. + * Scanning (and therefore enqueueing) is only necessary if there are any not yet scanned + * objects in the chunk. */ GCWorkerThreadState state = getWorkerThreadState(); AlignedHeapChunk.AlignedHeader chunk = state.getAllocChunk(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index 9293cd63d4be..bf723046aa21 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -350,13 +350,35 @@ public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOp @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getSizeFromHeader(Object obj, Word header, boolean addOptionalIdHashField) { + public static UnsignedWord getSizeFromHeader(Object obj, Word header, long eightHeaderBytes, boolean addOptionalIdHashField) { ObjectHeader oh = Heap.getHeap().getObjectHeader(); DynamicHub hub = oh.dynamicHubFromObjectHeader(header); int encoding = hub.getLayoutEncoding(); boolean withOptionalIdHashField = addOptionalIdHashField || (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && oh.hasOptionalIdentityHashField(header)); - return getSizeFromEncoding(obj, hub, encoding, withOptionalIdHashField); + + if (isArrayLike(encoding)) { + int arrayLength = getArrayLengthFromHeader(obj, eightHeaderBytes); + return getArraySize(encoding, arrayLength, withOptionalIdHashField); + } else { + return getPureInstanceSize(hub, withOptionalIdHashField); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static int getArrayLengthFromHeader(Object obj, long eightHeaderBytes) { + ObjectLayout ol = ConfigurationValues.getObjectLayout(); + assert ol.getArrayLengthOffset() >= 4; + if (ol.getArrayLengthOffset() == 4) { + /* + * If the array length is located within the first 8 bytes, then we need to extract it + * from the already read header data. + */ + int result = (int) (eightHeaderBytes >>> 32); + assert result >= 0; + return result; + } + return ArrayLengthNode.arrayLength(obj); } @AlwaysInline("Actual inlining decided by callers.") From 80f27d4013bd0ef8c88ea03e7b8bbf5bc0764826 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 29 Jun 2023 14:01:41 +0200 Subject: [PATCH 122/125] Fix the object size used in retractAllocation(). --- .../src/com/oracle/svm/core/genscavenge/Space.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 611dd53925f4..d49c8ecd5d80 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -241,9 +241,9 @@ private Pointer allocateInNewChunkParallel(UnsignedWord objectSize) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void retractAllocationParallel(UnsignedWord objectSize) { assert ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase(); - AlignedHeapChunk.AlignedHeader oldChunk = ParallelGC.singleton().getAllocationChunk(); - assert oldChunk.isNonNull(); - AlignedHeapChunk.retractAllocation(oldChunk, objectSize); + AlignedHeapChunk.AlignedHeader chunk = ParallelGC.singleton().getAllocationChunk(); + assert chunk.isNonNull(); + AlignedHeapChunk.retractAllocation(chunk, objectSize); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -506,7 +506,7 @@ private Object copyAlignedObjectParallel(Object original) { Object forward = ohi.installForwardingPointerParallel(original, eightHeaderBytes, copy); if (forward != copy) { /* We lost the race. Retract speculatively allocated memory. */ - retractAllocationParallel(originalSize); + retractAllocationParallel(copySize); return forward; } From 2047127e257a1cad75e24261f6257e6bece8d058 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 29 Jun 2023 14:01:54 +0200 Subject: [PATCH 123/125] Support the parallel GC on Windows. --- .../com/oracle/svm/core/genscavenge/parallel/ParallelGC.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index 09ca8a1d2833..d295e128e53e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -480,8 +480,6 @@ public boolean isInConfiguration(IsInConfigurationAccess access) { @Override public void afterRegistration(AfterRegistrationAccess access) { - UserError.guarantee(Platform.includedIn(Platform.LINUX.class) || Platform.includedIn(Platform.DARWIN.class), - "The parallel garbage collector ('--gc=parallel') is currently only supported on Linux and macOS."); verifyOptionEnabled(SubstrateOptions.SpawnIsolates); ImageSingletons.add(ParallelGC.class, new ParallelGC()); From 9710da3cbb45947c7c001b3c855023f86040a752 Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 13 Jul 2023 11:12:53 +0300 Subject: [PATCH 124/125] Enable RememberedSet before promoting unaligned chunk --- .../oracle/svm/core/genscavenge/Space.java | 41 +++---------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index d49c8ecd5d80..d14603da7207 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -537,41 +537,9 @@ private Object copyAlignedObjectParallel(Object original) { /** Promote an AlignedHeapChunk by moving it to this space. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk) { - if (ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase()) { - promoteAlignedHeapChunkParallel(chunk); - } else { - promoteAlignedHeapChunkSerial(chunk); - } - } + assert !(ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase()); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void promoteAlignedHeapChunkSerial(AlignedHeapChunk.AlignedHeader chunk) { Space originalSpace = HeapChunk.getSpace(chunk); - promoteAlignedHeapChunk0(chunk, originalSpace); - - if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { - ParallelGC.singleton().push(chunk); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void promoteAlignedHeapChunkParallel(AlignedHeapChunk.AlignedHeader chunk) { - ParallelGC.singleton().getMutex().lockNoTransitionUnspecifiedOwner(); - try { - Space originalSpace = HeapChunk.getSpace(chunk); - if (!originalSpace.isFromSpace()) { - /* The chunk was already promoted in the meanwhile. */ - return; - } - promoteAlignedHeapChunk0(chunk, originalSpace); - ParallelGC.singleton().push(chunk); - } finally { - ParallelGC.singleton().getMutex().unlockNoTransitionUnspecifiedOwner(); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void promoteAlignedHeapChunk0(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert originalSpace.isFromSpace(); assert !this.isFromSpace(); @@ -584,6 +552,10 @@ private void promoteAlignedHeapChunk0(AlignedHeapChunk.AlignedHeader chunk, Spac RememberedSet.get().clearRememberedSet(chunk); } } + + if (ParallelGC.isEnabled() && GCImpl.getGCImpl().isCompleteCollection()) { + ParallelGC.singleton().push(chunk); + } } /** Promote an UnalignedHeapChunk by moving it to this Space. */ @@ -625,8 +597,6 @@ private void promoteUnalignedHeapChunkParallel(UnalignedHeapChunk.UnalignedHeade @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void promoteUnalignedHeapChunk0(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert originalSpace.isFromSpace(); - - appendUnalignedHeapChunk(chunk, originalSpace); if (this.isOldSpace()) { if (originalSpace.isYoungSpace()) { RememberedSet.get().enableRememberedSetForChunk(chunk); @@ -635,6 +605,7 @@ private void promoteUnalignedHeapChunk0(UnalignedHeapChunk.UnalignedHeader chunk RememberedSet.get().clearRememberedSet(chunk); } } + appendUnalignedHeapChunk(chunk, originalSpace); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) From 3e57babbb3bf0ce0c6aa797b0b6e2dc6caad2d86 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 17 Jul 2023 15:47:22 +0300 Subject: [PATCH 125/125] Log number of worker threads --- .../svm/core/genscavenge/parallel/ParallelGC.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java index d295e128e53e..63531f206db9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/parallel/ParallelGC.java @@ -47,6 +47,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointOptions; @@ -62,6 +63,7 @@ import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.log.Log; import com.oracle.svm.core.option.SubstrateOptionKey; import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.thread.PlatformThreads.OSThreadHandle; @@ -265,7 +267,9 @@ public void tearDown() { private static int getWorkerCount() { int setting = SubstrateOptions.ParallelGCThreads.getValue(); - return setting > 0 ? setting : getDefaultWorkerCount(); + int workerCount = setting > 0 ? setting : getDefaultWorkerCount(); + verboseGCLog().string("[Number of ParallelGC threads: ").unsigned(workerCount).string("]").newline(); + return workerCount; } private static int getDefaultWorkerCount() { @@ -423,6 +427,10 @@ private void waitUntilWorkerThreadsFinish0() { } } + private static Log verboseGCLog() { + return SubstrateGCOptions.VerboseGC.getValue() ? Log.log() : Log.noopLog(); + } + @RawStructure private interface GCWorkerThreadState extends PointerBase { @RawField